Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add Weaviate vector storage #2451 (#2581)

Lucas-CX1ed730fa0460
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.py
index 940b07184..0ff8d6767 100644
--- a/camel/storages/__init__.py
+++ b/camel/storages/__init__.py
@@ -31,6 +31,7 @@ from .vectordb_storages.milvus import MilvusStorage
from .vectordb_storages.oceanbase import OceanBaseStorage
from .vectordb_storages.qdrant import QdrantStorage
from .vectordb_storages.tidb import TiDBStorage
+from .vectordb_storages.weaviate import WeaviateStorage
__all__ = [
'BaseKeyValueStorage',
@@ -50,4 +51,5 @@ __all__ = [
'NebulaGraph',
'Mem0Storage',
'OceanBaseStorage',
+ 'WeaviateStorage',
]
diff --git a/camel/storages/vectordb_storages/__init__.py b/camel/storages/vectordb_storages/__init__.py
index 3b8b020a8..4f04e6a7f 100644
--- a/camel/storages/vectordb_storages/__init__.py
+++ b/camel/storages/vectordb_storages/__init__.py
@@ -24,6 +24,7 @@ from .milvus import MilvusStorage
from .oceanbase import OceanBaseStorage
from .qdrant import QdrantStorage
from .tidb import TiDBStorage
+from .weaviate import WeaviateStorage
__all__ = [
'BaseVectorStorage',
@@ -34,6 +35,7 @@ __all__ = [
"TiDBStorage",
'FaissStorage',
'OceanBaseStorage',
+ 'WeaviateStorage',
'VectorRecord',
'VectorDBStatus',
]
diff --git a/camel/storages/vectordb_storages/weaviate.py b/camel/storages/vectordb_storages/weaviate.py
new file mode 100644
index 000000000..9a7a2dd31
--- /dev/null
+++ b/camel/storages/vectordb_storages/weaviate.py
@@ -0,0 +1,713 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import logging
+import math
+import re
+from datetime import datetime
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ List,
+ Literal,
+ Optional,
+ Union,
+ cast,
+)
+
+if TYPE_CHECKING:
+ from weaviate import WeaviateClient
+
+from camel.storages.vectordb_storages import (
+ BaseVectorStorage,
+ VectorDBQuery,
+ VectorDBQueryResult,
+ VectorDBStatus,
+ VectorRecord,
+)
+from camel.utils import dependencies_required
+
+logger = logging.getLogger(__name__)
+
+
+# Type definitions for configuration options
+ConnectionType = Literal["local", "cloud", "embedded", "custom"]
+VectorIndexType = Literal["hnsw", "flat"]
+DistanceMetric = Literal["cosine", "dot", "l2-squared", "hamming", "manhattan"]
+
+
+class WeaviateStorage(BaseVectorStorage):
+ r"""An implementation of the `BaseVectorStorage` for interacting with
+ Weaviate, a cloud-native vector search engine.
+
+ This class provides multiple ways to connect to Weaviate instances:
+ - Weaviate Cloud (WCD)
+ - Local Docker/Kubernetes instances
+ - Embedded Weaviate
+ - Custom connection parameters
+
+ Args:
+ vector_dim (int): The dimension of storing vectors.
+ collection_name (Optional[str], optional): Name for the collection in
+ Weaviate. If not provided, generates a unique name based on current
+ timestamp. (default: :obj:`None`)
+ connection_type (ConnectionType, optional): Type of connection to use.
+ Supported types: 'local', 'cloud', 'embedded', 'custom'.
+ (default: :obj:`"local"`)
+
+ # Weaviate Cloud parameters
+ wcd_cluster_url (Optional[str], optional): Weaviate Cloud cluster URL.
+ Required when connection_type='cloud'.
+ wcd_api_key (Optional[str], optional): Weaviate Cloud API key.
+ Required when connection_type='cloud'.
+
+ # Local instance parameters
+ local_host (str, optional): Local Weaviate host.
+ (default: :obj:`"localhost"`)
+ local_port (int, optional): Local Weaviate HTTP port.
+ (default: :obj:`8080`)
+ local_grpc_port (int, optional): Local Weaviate gRPC port.
+ (default: :obj:`50051`)
+ local_auth_credentials (Optional[Union[str, Any]], optional):
+ Authentication credentials for local instance. Can be an API key
+ string or Auth object. (default: :obj:`None`)
+
+ # Embedded Weaviate parameters
+ embedded_hostname (str, optional): Embedded instance hostname.
+ (default: :obj:`"127.0.0.1"`)
+ embedded_port (int, optional): Embedded instance HTTP port.
+ (default: :obj:`8079`)
+ embedded_grpc_port (int, optional): Embedded instance gRPC port.
+ (default: :obj:`50050`)
+ embedded_version (Optional[str], optional): Weaviate version for
+ embedded instance. If None, uses the default version.
+ (default: :obj:`None`)
+ embedded_persistence_data_path (Optional[str], optional): Directory
+ for embedded database files. (default: :obj:`None`)
+ embedded_binary_path (Optional[str], optional): Directory for
+ Weaviate binary. (default: :obj:`None`)
+ embedded_environment_variables (Optional[Dict[str, str]], optional):
+ Environment variables for embedded instance. (default: :obj:`None`)
+
+ # Custom connection parameters
+ custom_http_host (Optional[str], optional): Custom HTTP host.
+ custom_http_port (Optional[int], optional): Custom HTTP port.
+ custom_http_secure (Optional[bool], optional): Use HTTPS.
+ custom_grpc_host (Optional[str], optional): Custom gRPC host.
+ custom_grpc_port (Optional[int], optional): Custom gRPC port.
+ custom_grpc_secure (Optional[bool], optional): Use secure gRPC.
+ custom_auth_credentials (Optional[Any], optional): Custom auth.
+
+ # Vector index configuration parameters
+ vector_index_type (VectorIndexType, optional): Vector index type.
+ Supported types: 'hnsw', 'flat'. (default: :obj:`"hnsw"`)
+ distance_metric (DistanceMetric, optional): Distance metric for vector
+ similarity. Supported metrics: 'cosine', 'dot', 'l2-squared',
+ 'hamming', 'manhattan'. (default: :obj:`"cosine"`)
+
+ # Common parameters for all connection types
+ headers (Optional[Dict[str, str]], optional): Additional headers for
+ third-party API keys (e.g., OpenAI, Cohere). (default: :obj:`None`)
+ additional_config (Optional[Any], optional): Advanced configuration
+ options like timeouts. (default: :obj:`None`)
+ skip_init_checks (bool, optional): Skip initialization checks.
+ (default: :obj:`False`)
+
+ Raises:
+ ImportError: If `weaviate` package is not installed.
+ ValueError: If connection parameters are invalid or missing.
+ RuntimeError: If there's an error setting up the collection.
+
+
+
+ Note:
+ This implementation supports synchronous operations only.
+ The client connection is automatically handled and closed when the
+ storage instance is destroyed.
+ """
+
+ @dependencies_required('weaviate')
+ def __init__(
+ self,
+ vector_dim: int,
+ collection_name: Optional[str] = None,
+ connection_type: ConnectionType = "local",
+ # Weaviate Cloud parameters
+ wcd_cluster_url: Optional[str] = None,
+ wcd_api_key: Optional[str] = None,
+ # Local instance parameters
+ local_host: str = "localhost",
+ local_port: int = 8080,
+ local_grpc_port: int = 50051,
+ local_auth_credentials: Optional[Union[str, Any]] = None,
+ # Embedded Weaviate parameters
+ embedded_hostname: str = "127.0.0.1",
+ embedded_port: int = 8079,
+ embedded_grpc_port: int = 50050,
+ embedded_version: Optional[str] = None,
+ embedded_persistence_data_path: Optional[str] = None,
+ embedded_binary_path: Optional[str] = None,
+ embedded_environment_variables: Optional[Dict[str, str]] = None,
+ # Custom connection parameters
+ custom_http_host: Optional[str] = None,
+ custom_http_port: Optional[int] = None,
+ custom_http_secure: Optional[bool] = None,
+ custom_grpc_host: Optional[str] = None,
+ custom_grpc_port: Optional[int] = None,
+ custom_grpc_secure: Optional[bool] = None,
+ custom_auth_credentials: Optional[Any] = None,
+ # Vector index configuration parameters
+ vector_index_type: VectorIndexType = "hnsw",
+ distance_metric: DistanceMetric = "cosine",
+ # Common parameters
+ headers: Optional[Dict[str, str]] = None,
+ additional_config: Optional[Any] = None,
+ skip_init_checks: bool = False,
+ **kwargs: Any,
+ ) -> None:
+ self.vector_dim = vector_dim
+ self.collection_name = (
+ collection_name or self._generate_collection_name()
+ )
+
+ # Store vector index configuration
+ self.vector_index_type: VectorIndexType = vector_index_type
+ self.distance_metric: DistanceMetric = distance_metric
+
+ # Store connection type configuration
+ self.connection_type: ConnectionType = connection_type
+
+ # Store connection parameters for later use
+ self._connection_params = {
+ 'wcd_cluster_url': wcd_cluster_url,
+ 'wcd_api_key': wcd_api_key,
+ 'local_host': local_host,
+ 'local_port': local_port,
+ 'local_grpc_port': local_grpc_port,
+ 'local_auth_credentials': local_auth_credentials,
+ 'embedded_hostname': embedded_hostname,
+ 'embedded_port': embedded_port,
+ 'embedded_grpc_port': embedded_grpc_port,
+ 'embedded_version': embedded_version,
+ 'embedded_persistence_data_path': embedded_persistence_data_path,
+ 'embedded_binary_path': embedded_binary_path,
+ 'embedded_environment_variables': embedded_environment_variables,
+ 'custom_http_host': custom_http_host,
+ 'custom_http_port': custom_http_port,
+ 'custom_http_secure': custom_http_secure,
+ 'custom_grpc_host': custom_grpc_host,
+ 'custom_grpc_port': custom_grpc_port,
+ 'custom_grpc_secure': custom_grpc_secure,
+ 'custom_auth_credentials': custom_auth_credentials,
+ 'headers': headers,
+ 'additional_config': additional_config,
+ 'skip_init_checks': skip_init_checks,
+ **kwargs,
+ }
+
+ # Create client using the new unified approach
+ self._client = self._get_connection_client()
+ logger.info(
+ f"Created Weaviate client with connection type: "
+ f"{self.connection_type}"
+ )
+
+ self._check_and_create_collection(**kwargs)
+
+ def _get_connection_client(self) -> "WeaviateClient":
+ r"""Get Weaviate client based on connection type and user settings."""
+ import weaviate
+
+ # Map connection types to handler methods
+ connection_handlers: Dict[ConnectionType, Any] = {
+ 'cloud': self._create_cloud_client,
+ 'local': self._create_local_client,
+ 'embedded': self._create_embedded_client,
+ 'custom': self._create_custom_client,
+ }
+
+ # Get connection handler
+ handler = connection_handlers[self.connection_type]
+
+ try:
+ return handler(weaviate)
+ except Exception as e:
+ logger.error(
+ f"Failed to create {self.connection_type} client: {e}"
+ )
+ raise
+
+ def _create_cloud_client(self, weaviate_module: Any) -> "WeaviateClient":
+ r"""Create a Weaviate Cloud client."""
+ cluster_url = self._connection_params.get('wcd_cluster_url')
+ api_key = self._connection_params.get('wcd_api_key')
+
+ if not cluster_url:
+ raise ValueError(
+ "wcd_cluster_url is required for cloud connection"
+ )
+ if not api_key:
+ raise ValueError("wcd_api_key is required for cloud connection")
+
+ return weaviate_module.connect_to_weaviate_cloud(
+ cluster_url=cluster_url,
+ auth_credentials=api_key,
+ headers=self._connection_params.get('headers'),
+ additional_config=self._connection_params.get('additional_config'),
+ skip_init_checks=self._connection_params.get(
+ 'skip_init_checks', False
+ ),
+ )
+
+ def _create_local_client(self, weaviate_module: Any) -> "WeaviateClient":
+ r"""Create a local Weaviate client."""
+ return weaviate_module.connect_to_local(
+ host=self._connection_params.get('local_host', 'localhost'),
+ port=self._connection_params.get('local_port', 8080),
+ grpc_port=self._connection_params.get('local_grpc_port', 50051),
+ headers=self._connection_params.get('headers'),
+ additional_config=self._connection_params.get('additional_config'),
+ skip_init_checks=self._connection_params.get(
+ 'skip_init_checks', False
+ ),
+ auth_credentials=self._connection_params.get(
+ 'local_auth_credentials'
+ ),
+ )
+
+ def _create_embedded_client(
+ self, weaviate_module: Any
+ ) -> "WeaviateClient":
+ r"""Create an embedded Weaviate client."""
+ embedded_kwargs = {
+ 'hostname': self._connection_params.get(
+ 'embedded_hostname', '127.0.0.1'
+ ),
+ 'port': self._connection_params.get('embedded_port', 8079),
+ 'grpc_port': self._connection_params.get(
+ 'embedded_grpc_port', 50050
+ ),
+ 'headers': self._connection_params.get('headers'),
+ 'additional_config': self._connection_params.get(
+ 'additional_config'
+ ),
+ }
+
+ # Add optional embedded parameters
+ if self._connection_params.get('embedded_version') is not None:
+ embedded_kwargs['version'] = self._connection_params[
+ 'embedded_version'
+ ]
+ if (
+ self._connection_params.get('embedded_persistence_data_path')
+ is not None
+ ):
+ embedded_kwargs['persistence_data_path'] = self._connection_params[
+ 'embedded_persistence_data_path'
+ ]
+ if self._connection_params.get('embedded_binary_path') is not None:
+ embedded_kwargs['binary_path'] = self._connection_params[
+ 'embedded_binary_path'
+ ]
+ if (
+ self._connection_params.get('embedded_environment_variables')
+ is not None
+ ):
+ embedded_kwargs['environment_variables'] = self._connection_params[
+ 'embedded_environment_variables'
+ ]
+
+ return weaviate_module.connect_to_embedded(**embedded_kwargs)
+
+ def _create_custom_client(self, weaviate_module: Any) -> "WeaviateClient":
+ r"""Create a custom Weaviate client."""
+ # Validate required custom parameters
+ required_params = [
+ 'custom_http_host',
+ 'custom_http_port',
+ 'custom_http_secure',
+ 'custom_grpc_host',
+ 'custom_grpc_port',
+ 'custom_grpc_secure',
+ ]
+
+ for param in required_params:
+ if self._connection_params.get(param) is None:
+ raise ValueError(f"{param} is required for custom connection")
+
+ return weaviate_module.connect_to_custom(
+ http_host=self._connection_params['custom_http_host'],
+ http_port=self._connection_params['custom_http_port'],
+ http_secure=self._connection_params['custom_http_secure'],
+ grpc_host=self._connection_params['custom_grpc_host'],
+ grpc_port=self._connection_params['custom_grpc_port'],
+ grpc_secure=self._connection_params['custom_grpc_secure'],
+ headers=self._connection_params.get('headers'),
+ additional_config=self._connection_params.get('additional_config'),
+ auth_credentials=self._connection_params.get(
+ 'custom_auth_credentials'
+ ),
+ skip_init_checks=self._connection_params.get(
+ 'skip_init_checks', False
+ ),
+ )
+
+ def __del__(self):
+ r"""Clean up client connection."""
+ try:
+ if hasattr(self, '_client') and self._client is not None:
+ self._client.close()
+ logger.debug("Closed Weaviate client connection")
+ except Exception as e:
+ logger.warning(f"Error closing Weaviate client: {e}")
+
+ def close(self) -> None:
+ r"""Explicitly close the client connection."""
+ if self._client is not None:
+ self._client.close()
+ logger.info("Explicitly closed Weaviate client connection")
+
+ def _generate_collection_name(self) -> str:
+ r"""Generate a collection name if user doesn't provide one."""
+ timestamp = datetime.now().isoformat()
+ # Weaviate collection names must start with uppercase and be valid
+ # GraphQL names
+ valid_name = "Collection_" + re.sub(r'[^a-zA-Z0-9_]', '_', timestamp)
+ return valid_name
+
+ def _check_and_create_collection(self, **kwargs: Any) -> None:
+ r"""Check if collection exists and create if it doesn't."""
+ if not self._collection_exists(self.collection_name):
+ self._create_collection(**kwargs)
+
+ def _collection_exists(self, collection_name: str) -> bool:
+ r"""Check if the collection exists."""
+ try:
+ collection = self._client.collections.get(collection_name)
+ collection.config.get()
+ return True
+ except Exception:
+ return False
+
+ def _get_vector_index_config(self, **kwargs: Any) -> Any:
+ r"""Get vector index configuration based on user settings."""
+ import weaviate.classes.config as wvc
+
+ # Map distance metrics - type safety guaranteed by Literal
+ distance_metric_mapping: Dict[DistanceMetric, Any] = {
+ 'cosine': wvc.VectorDistances.COSINE,
+ 'dot': wvc.VectorDistances.DOT,
+ 'l2-squared': wvc.VectorDistances.L2_SQUARED,
+ 'hamming': wvc.VectorDistances.HAMMING,
+ 'manhattan': wvc.VectorDistances.MANHATTAN,
+ }
+
+ # Get distance metric - no need for None check due to Literal typing
+ distance_metric = distance_metric_mapping[self.distance_metric]
+
+ # Configure vector index based on type
+ if self.vector_index_type == 'hnsw':
+ return wvc.Configure.VectorIndex.hnsw(
+ distance_metric=distance_metric,
+ **kwargs,
+ )
+ else: # must be 'flat' due to Literal typing
+ return wvc.Configure.VectorIndex.flat(
+ distance_metric=distance_metric,
+ **kwargs,
+ )
+
+ def _create_collection(self, **kwargs: Any) -> None:
+ r"""Create a new collection in Weaviate."""
+ import weaviate.classes.config as wvc
+
+ # Separate vector index kwargs from general kwargs to avoid conflicts
+ vector_index_kwargs = {}
+ collection_kwargs = {}
+
+ # Known vector index parameters
+ vector_index_params = {
+ 'ef_construction',
+ 'max_connections',
+ 'ef',
+ 'dynamic_ef_min',
+ 'dynamic_ef_max',
+ 'dynamic_ef_factor',
+ 'vector_cache_max_objects',
+ 'flat_search_cutoff',
+ 'cleanup_interval_seconds',
+ 'pq_enabled',
+ 'pq_segments',
+ 'pq_centroids',
+ 'pq_training_limit',
+ 'pq_encoder',
+ }
+
+ for key, value in kwargs.items():
+ if key in vector_index_params:
+ vector_index_kwargs[key] = value
+ else:
+ collection_kwargs[key] = value
+
+ self._client.collections.create(
+ name=self.collection_name,
+ vectorizer_config=wvc.Configure.Vectorizer.none(),
+ properties=[
+ wvc.Property(name="payload", data_type=wvc.DataType.TEXT),
+ ],
+ vector_index_config=self._get_vector_index_config(
+ **vector_index_kwargs
+ ),
+ **collection_kwargs,
+ )
+
+ def add(
+ self,
+ records: List[VectorRecord],
+ **kwargs: Any,
+ ) -> None:
+ r"""Saves a list of vector records to the storage.
+
+ Args:
+ records (List[VectorRecord]): List of vector records to be saved.
+ **kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ RuntimeError: If there is an error during the saving process.
+ """
+ if not records:
+ return
+
+ try:
+ collection = self._client.collections.get(self.collection_name)
+
+ with collection.batch.dynamic() as batch:
+ for record in records:
+ payload_str = (
+ json.dumps(record.payload) if record.payload else ""
+ )
+ batch.add_object(
+ properties={"payload": payload_str},
+ vector=record.vector,
+ uuid=record.id,
+ **kwargs,
+ )
+ logger.debug(
+ f"Successfully added vectors to Weaviate collection: "
+ f"{self.collection_name}"
+ )
+
+ except Exception as e:
+ raise RuntimeError(f"Failed to add vectors to Weaviate: {e}")
+
+ def delete(
+ self,
+ ids: List[str],
+ **kwargs: Any,
+ ) -> None:
+ r"""Deletes a list of vectors identified by their IDs from the storage.
+
+ Args:
+ ids (List[str]): List of unique identifiers for the vectors to be
+ deleted.
+ **kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ RuntimeError: If there is an error during the deletion process.
+ """
+ if not ids:
+ return
+
+ try:
+ collection = self._client.collections.get(self.collection_name)
+ from weaviate.classes.query import Filter
+
+ collection.data.delete_many(
+ where=Filter.by_id().contains_any(ids), **kwargs
+ )
+ logger.debug(
+ f"Successfully deleted vectors in Weaviate: "
+ f"{self.collection_name}"
+ )
+
+ except Exception as e:
+ raise RuntimeError(f"Failed to delete vectors from Weaviate: {e}")
+
+ def _calculate_similarity_from_distance(
+ self, distance: Optional[float]
+ ) -> float:
+ r"""Calculate similarity score based on distance metric.
+
+ Args:
+ distance (Optional[float]): The distance value from Weaviate.
+
+ Returns:
+ float: Normalized similarity score between 0 and 1.
+ """
+ # Return 0.0 if distance is not available
+ if distance is None:
+ return 0.0
+
+ # Calculate similarity based on distance metric
+ if self.distance_metric == "cosine":
+ # Cosine: 0 (identical) to 2 (opposite)
+ # Convert to similarity: 1 - (distance / 2)
+ return max(0.0, 1.0 - (distance / 2.0))
+
+ elif self.distance_metric == "dot":
+ # Dot product: Weaviate returns -dot,
+ # smaller (more negative) is more similar
+ # Use sigmoid function to normalize to [0,1] range
+ # For dot product, we use: 1 / (1 + exp(distance))
+ return 1.0 / (1.0 + math.exp(distance))
+
+ elif self.distance_metric in ["l2-squared", "manhattan"]:
+ # L2-squared and Manhattan: 0 (identical) to ∞
+ # Convert to similarity using: 1 / (1 + distance)
+ # This provides a smoother decay than exponential
+ return 1.0 / (1.0 + distance)
+
+ elif self.distance_metric == "hamming":
+ # Hamming: 0 (identical) to vector_dim (completely different)
+ # Convert to similarity: 1 - (distance / vector_dim)
+ return max(0.0, 1.0 - (distance / self.vector_dim))
+
+ else:
+ # Unknown metric, return 0.0
+ return 0.0
+
+ def status(self) -> VectorDBStatus:
+ r"""Returns status of the vector database.
+
+ Returns:
+ VectorDBStatus: The vector database status.
+ """
+ try:
+ collection = self._client.collections.get(self.collection_name)
+ objects = collection.aggregate.over_all(total_count=True)
+
+ vector_count = (
+ objects.total_count if objects.total_count is not None else 0
+ )
+
+ return VectorDBStatus(
+ vector_dim=self.vector_dim, vector_count=vector_count
+ )
+
+ except Exception as e:
+ logger.warning(f"Failed to get status from Weaviate: {e}")
+ return VectorDBStatus(vector_dim=self.vector_dim, vector_count=0)
+
+ def query(
+ self,
+ query: VectorDBQuery,
+ **kwargs: Any,
+ ) -> List[VectorDBQueryResult]:
+ r"""Searches for similar vectors in the storage based on the provided
+ query.
+
+ Args:
+ query (VectorDBQuery): The query object containing the search
+ vector and the number of top similar vectors to retrieve.
+ **kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ List[VectorDBQueryResult]: A list of vectors retrieved from the
+ storage based on similarity to the query vector.
+ """
+ from weaviate.classes.query import MetadataQuery
+
+ try:
+ collection = self._client.collections.get(self.collection_name)
+
+ response = collection.query.near_vector(
+ near_vector=query.query_vector,
+ limit=query.top_k,
+ include_vector=True,
+ return_metadata=MetadataQuery(distance=True),
+ **kwargs,
+ )
+
+ results = []
+ for obj in response.objects:
+ # Calculate similarity score based on distance metric
+ similarity = self._calculate_similarity_from_distance(
+ obj.metadata.distance
+ )
+
+ # Handle payload
+ payload = None
+ if obj.properties.get('payload'):
+ payload_value = obj.properties['payload']
+ if isinstance(payload_value, str):
+ try:
+ payload = json.loads(payload_value)
+ except (json.JSONDecodeError, TypeError):
+ payload = {"raw": payload_value}
+ else:
+ payload = {"raw": str(payload_value)}
+
+ # Handle vector data
+ # In newer versions of Weaviate, obj.vector returns a dict
+ # with 'default' key
+
+ vector: List[float] = []
+ if hasattr(obj, 'vector') and obj.vector is not None:
+ # New format: {'default': [vector_values]}
+ vector_data = obj.vector.get('default', [])
+ if (
+ vector_data
+ and isinstance(vector_data, list)
+ and len(vector_data) > 0
+ and isinstance(vector_data[0], float)
+ ):
+ vector = cast(List[float], vector_data)
+ else:
+ # unsupported vector data format
+ vector = []
+ else:
+ vector = []
+
+ result = VectorDBQueryResult.create(
+ similarity=similarity,
+ vector=vector,
+ id=str(obj.uuid),
+ payload=payload,
+ )
+ results.append(result)
+
+ return results
+
+ except Exception as e:
+ raise RuntimeError(f"Failed to query vectors from Weaviate: {e}")
+
+ def clear(self) -> None:
+ r"""Remove all vectors from the storage."""
+ try:
+ self._client.collections.delete(self.collection_name)
+ self._create_collection()
+ except Exception as e:
+ raise RuntimeError(f"Failed to clear Weaviate collection: {e}")
+
+ def load(self) -> None:
+ r"""Load the collection hosted on cloud service."""
+ # For Weaviate, collections are automatically available when client
+ # connects
+ pass
+
+ @property
+ def client(self) -> "WeaviateClient":
+ r"""Provides access to the underlying vector database client."""
+ return self._client
diff --git a/docs/get_started/installation.md b/docs/get_started/installation.md
index e6c918a99..80c1cf594 100644
--- a/docs/get_started/installation.md
+++ b/docs/get_started/installation.md
@@ -19,7 +19,7 @@ Available extras:
- `all`: Includes all features below
- `model_platforms`: OpenAI, Google, Mistral, Anthropic Claude, Cohere etc.
- `huggingface`: Transformers, Diffusers, Accelerate, Datasets, PyTorch etc.
-- `rag`: Sentence Transformers, Qdrant, Milvus, TiDB, BM25,OceanBase etc.
+- `rag`: Sentence Transformers, Qdrant, Milvus, TiDB, BM25,OceanBase, Weaviate etc.
- `storage`: Neo4j, Redis, Azure Blob, Google Cloud Storage, AWS S3 etc.
- `web_tools`: DuckDuckGo, Wikipedia, WolframAlpha, Google Maps, Weather API etc.
- `document_tools`: PDF, Word, OpenAPI, BeautifulSoup, Unstructured etc.
diff --git a/docs/key_modules/storages.md b/docs/key_modules/storages.md
index d16ef851b..71b99ecda 100644
--- a/docs/key_modules/storages.md
+++ b/docs/key_modules/storages.md
@@ -62,6 +62,13 @@ Reference: [Qdrant](https://qdrant.tech/)
Reference: [OceanBase](https://www.oceanbase.com/)
+**`WeaviateStorage`**:
+
+- Description: A concrete implementation of `BaseVectorStorage`, tailored for interacting with Weaviate, a vector search engine.
+
+Reference: [Weaviate](https://weaviate.io/)
+
+
### 2.3 Graph Storages
**`BaseGraphStorage`**:
@@ -374,7 +381,47 @@ Vector count after clearing: 0
```
-### 3.7. Using `NebulaGraph`
+### 3.7. Using `WeaviateStorage`
+
+```python
+from camel.storages import WeaviateStorage, VectorDBQuery, VectorRecord
+
+# Create WeaviateStorage instance with dimension = 4 using Weaviate Cloud
+weaviate_storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="camel_example_vectors",
+ connection_type="cloud",
+ wcd_cluster_url="your-weaviate-cloud-url",
+ wcd_api_key="your-weaviate-api-key",
+ vector_index_type="hnsw",
+ distance_metric="cosine",
+)
+
+# Add vectors records to Weaviate
+weaviate_storage.add([
+ VectorRecord(
+ vector=[-0.1, 0.1, -0.1, 0.1],
+ payload={'key1': 'value1'},
+ ),
+ VectorRecord(
+ vector=[-0.1, 0.1, 0.1, 0.1],
+ payload={'key2': 'value2'},
+ ),
+])
+
+# Query Weaviate for similar vectors
+query_results = weaviate_storage.query(VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1))
+for result in query_results:
+ print(result.record.payload, result.similarity)
+
+# Clear all vectors
+weaviate_storage.clear()
+```
+```markdown
+>>> {'key2': 'value2'} 0.7834733128547668
+```
+
+### 3.8. Using `NebulaGraph`
```python
from camel.storages.graph_storages import NebulaGraph
@@ -386,7 +433,7 @@ query = 'SHOW TAGS;'
print(nebula_graph.query(query))
```
-### 3.8. Using `Neo4jGraph`
+### 3.9. Using `Neo4jGraph`
```python
from camel.storages import Neo4jGraph
diff --git a/examples/storages/weaviate_vector_storage.py b/examples/storages/weaviate_vector_storage.py
new file mode 100644
index 000000000..7ffdea503
--- /dev/null
+++ b/examples/storages/weaviate_vector_storage.py
@@ -0,0 +1,312 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import random
+
+from camel.storages.vectordb_storages import (
+ VectorDBQuery,
+ VectorRecord,
+ WeaviateStorage,
+)
+
+"""
+Before running this example, you need to setup a Weaviate instance(reference: https://weaviate.io/developers/weaviate):
+
+(Option 1): Local Docker Weaviate:
+
+1. Install Docker
+2. Run the following command to start Weaviate:
+ docker run -p 8080:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/
+ weaviate:1.31.0
+3. The example will automatically connect to the local instance
+
+(Option 2): Weaviate Cloud Service (WCS):
+
+1. Sign up at https://console.weaviate.cloud
+2. Create a free Weaviate cluster
+3. Get your API key and cluster URL
+4. Update the WEAVIATE_URL and WEAVIATE_API_KEY variables below
+
+(Option 3): Embedded Weaviate:
+
+Only supports Linux and macOS systems (Windows is not supported)
+
+(Option 4): Custom Connection:
+
+Configure custom HTTP/gRPC endpoints for specialized deployments
+
+Note: You may see deprecation warnings from the Weaviate client library:
+"PydanticDeprecatedSince211: Accessing the 'model_fields' attribute..."
+These warnings are from the Weaviate library itself and can be safely ignored.
+They do not affect the functionality of the code.
+
+"""
+
+
+def example_local_connection():
+ r"""Example using local Weaviate instance (default connection type)."""
+ print("=== Local Weaviate Connection Example ===")
+
+ # Create WeaviateStorage instance with local connection (default)
+ weaviate_storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="camel_local_example",
+ connection_type="local",
+ local_host="localhost",
+ local_port=8080,
+ local_grpc_port=50051,
+ # Vector index configuration
+ vector_index_type="hnsw",
+ distance_metric="cosine",
+ )
+
+ run_storage_example(weaviate_storage)
+
+
+def example_cloud_connection():
+ """Example using Weaviate Cloud Service."""
+ print("=== Weaviate Cloud Connection Example ===")
+
+ # Replace these with your Weaviate Cloud connection parameters
+ WEAVIATE_URL = "your-weaviate-cloud-url"
+ WEAVIATE_API_KEY = "your-weaviate-api-key"
+
+ # Create WeaviateStorage instance with cloud connection
+ weaviate_storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="camel_cloud_example",
+ connection_type="cloud",
+ wcd_cluster_url=WEAVIATE_URL,
+ wcd_api_key=WEAVIATE_API_KEY,
+ # Vector index configuration
+ vector_index_type="hnsw",
+ distance_metric="cosine",
+ # Additional headers for third-party API keys
+ headers={
+ "X-OpenAI-Api-Key": "your-openai-api-key",
+ "X-Cohere-Api-Key": "your-cohere-api-key",
+ },
+ )
+
+ run_storage_example(weaviate_storage)
+
+
+def example_embedded_connection():
+ """Example using embedded Weaviate."""
+ print("=== Embedded Weaviate Connection Example ===")
+
+ # Create WeaviateStorage instance with embedded connection
+ weaviate_storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="camel_embedded_example",
+ connection_type="embedded",
+ embedded_hostname="127.0.0.1",
+ embedded_port=8079,
+ embedded_grpc_port=50050,
+ # Optional: persist data
+ embedded_persistence_data_path="./weaviate_data",
+ # Optional: specify Weaviate version
+ embedded_version="1.31.0",
+ # Optional: environment variables
+ embedded_environment_variables={
+ "QUERY_DEFAULTS_LIMIT": "25",
+ "DEFAULT_VECTORIZER_MODULE": "none",
+ },
+ # Vector index configuration
+ vector_index_type="hnsw",
+ distance_metric="cosine",
+ )
+
+ run_storage_example(weaviate_storage)
+
+
+def example_custom_connection():
+ """Example using custom Weaviate connection."""
+ print("=== Custom Weaviate Connection Example ===")
+
+ # Create WeaviateStorage instance with custom connection
+ weaviate_storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="camel_custom_example",
+ connection_type="custom",
+ custom_http_host="localhost",
+ custom_http_port=8080,
+ custom_http_secure=False,
+ custom_grpc_host="localhost",
+ custom_grpc_port=50051,
+ custom_grpc_secure=False,
+ # Optional: custom authentication
+ custom_auth_credentials=None,
+ # Vector index configuration
+ vector_index_type="hnsw",
+ distance_metric="cosine",
+ )
+
+ run_storage_example(weaviate_storage)
+
+
+def run_storage_example(weaviate_storage: WeaviateStorage):
+ """Run the common storage operations example."""
+
+ # Get database status
+ status = weaviate_storage.status()
+ print(f"Vector dimension: {status.vector_dim}")
+ print(f"Initial vector count: {status.vector_count}")
+ print(f"Vector index type: {weaviate_storage.vector_index_type}")
+ print(f"Distance metric: {weaviate_storage.distance_metric}")
+
+ # Generate vector records
+ print("\nAdding vectors...")
+ random.seed(20250605)
+ vector_records = []
+ for i in range(10):
+ vector_records.append(
+ VectorRecord(
+ vector=[random.uniform(-1, 1) for _ in range(4)],
+ payload={
+ 'idx': i,
+ 'category': 'example',
+ 'description': f'Vector example {i}',
+ },
+ )
+ )
+
+ # Add vectors records to Weaviate
+ weaviate_storage.add(vector_records)
+
+ # Check updated status
+ status = weaviate_storage.status()
+ print(f"Vector count after adding: {status.vector_count}")
+
+ # Query similar vectors, use the first vector as query vector
+ print("\nQuerying similar vectors...")
+ query_vector = vector_records[0].vector
+ query_results = weaviate_storage.query(
+ VectorDBQuery(query_vector=query_vector, top_k=3)
+ )
+
+ # Display query results
+ for i, result in enumerate(query_results):
+ print(f"Result {i+1}:")
+ print(f" ID: {result.record.id}")
+ print(f" Payload: {result.record.payload}")
+ print(f" Similarity: {result.similarity:.4f}")
+ # print(f" Vector: {result.record.vector}")
+
+ # Test deletion, delete the first two vectors
+ print("\nDeleting vectors...")
+ ids_to_delete = [result.record.id for result in query_results[:2]]
+ weaviate_storage.delete(ids_to_delete)
+
+ # Verify deletion
+ status = weaviate_storage.status()
+ print(f"Vector count after deletion: {status.vector_count}")
+
+ # Clear all vectors when done
+ print("\nClearing all vectors...")
+ weaviate_storage.clear()
+ status = weaviate_storage.status()
+ print(f"Vector count after clearing: {status.vector_count}")
+
+ # Close connection explicitly
+ weaviate_storage.close()
+ print("Connection closed.\n")
+
+
+def main():
+ """Main function demonstrating different connection types and
+ configurations."""
+
+ print(
+ "This example demonstrates different Weaviate connection types and "
+ "configurations."
+ )
+ print("Make sure you have the appropriate Weaviate instance running.\n")
+
+ # Example 1: Local connection (requires local Weaviate instance)
+ try:
+ example_local_connection()
+ except Exception as e:
+ print(f"Local connection failed: {e}")
+ print(
+ "Make sure you have a local Weaviate instance running on "
+ "localhost:8080\n"
+ )
+
+ # Example 2: Cloud connection (requires valid cloud credentials)
+ # try:
+ # example_cloud_connection()
+ # except Exception as e:
+ # print(f"Cloud connection failed: {e}")
+ # print("Make sure you have valid Weaviate Cloud credentials\n")
+
+ # Example 3: Embedded connection (automatically starts embedded Weaviate)
+ # Note: Embedded Weaviate client is unsupported on Windows system
+ # try:
+ # example_embedded_connection()
+ # except Exception as e:
+ # print(f"Embedded connection failed: {e}")
+ # print(
+ # "Make sure you have weaviate-client with embedded support "
+ # "installed\n"
+ # )
+
+ # Example 4: Custom connection (requires custom Weaviate setup)
+ # try:
+ # example_custom_connection()
+ # except Exception as e:
+ # print(f"Custom connection failed: {e}")
+ # print("Make sure you have a custom Weaviate instance configured\n")
+
+
+if __name__ == "__main__":
+ main()
+
+"""
+===============================================================================
+This example demonstrates different Weaviate connection types and
+configurations.
+ Make sure you have the appropriate Weaviate instance running.
+
+=== Local Weaviate Connection Example ===
+Vector dimension: 4
+Initial vector count: 0
+Vector index type: hnsw
+Distance metric: cosine
+
+Adding vectors...
+Vector count after adding: 10
+
+Querying similar vectors...
+Result 1:
+ ID: 0b71d7a8-6913-4c21-8e8b-4ecdd164c4f9
+ Payload: {'idx': 0, 'category': 'example', 'description': 'Vector example 0'}
+ Similarity: 1.0000
+Result 2:
+ ID: 416fc9af-47ef-4648-a48a-98397b68b656
+ Payload: {'idx': 1, 'category': 'example', 'description': 'Vector example 1'}
+ Similarity: 0.8594
+Result 3:
+ ID: aac5509e-9e24-42c8-8b7e-33564c291d87
+ Payload: {'idx': 9, 'category': 'example', 'description': 'Vector example 9'}
+ Similarity: 0.8500
+
+Deleting vectors...
+Vector count after deletion: 8
+
+Clearing all vectors...
+Vector count after clearing: 0
+Connection closed.
+===============================================================================
+"""
diff --git a/pyproject.toml b/pyproject.toml
index a0ea5ed89..02b913efc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -73,6 +73,7 @@ rag = [
"pymilvus>=2.4.0,<3",
"pytidb-experimental==0.0.1.dev4",
"faiss-cpu>=1.7.2,<2",
+ "weaviate-client>=4.15.0",
"neo4j>=5.18.0,<6",
"nebula3-python==3.8.2",
"rank-bm25>=0.2.2,<0.3",
@@ -207,6 +208,7 @@ storage = [
"botocore>=1.35.3,<2",
"mem0ai>=0.1.73",
"pyobvector>=0.1.18",
+ "weaviate-client>=4.15.0",
]
owl = [
"numpy>=1.2,<=2.2",
@@ -377,6 +379,7 @@ all = [
"aci-sdk>=1.0.0b1",
"pylatex>=1.4.2",
"chunkr-ai>=0.0.50",
+ "weaviate-client>=4.15.0",
"python-pptx>=1.0.2",
]
diff --git a/test/storages/vector_storages/test_weaviate.py b/test/storages/vector_storages/test_weaviate.py
new file mode 100644
index 000000000..bd3773058
--- /dev/null
+++ b/test/storages/vector_storages/test_weaviate.py
@@ -0,0 +1,330 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from camel.storages import VectorDBQuery, VectorRecord, WeaviateStorage
+
+
+def mock_weaviate_client():
+ r"""Create a mock Weaviate client with collections interface."""
+ with patch('weaviate.connect_to_local') as mock_connect:
+ mock_client = MagicMock()
+ mock_connect.return_value = mock_client
+
+ # Mock collections interface
+ mock_collection = MagicMock()
+ mock_client.collections.get.return_value = mock_collection
+ mock_client.collections.create = MagicMock()
+ mock_client.collections.delete = MagicMock()
+
+ # Mock collection config
+ mock_collection.config.get.return_value = {}
+
+ # Mock batch interface
+ mock_batch = MagicMock()
+ mock_collection.batch.dynamic.return_value.__enter__.return_value = (
+ mock_batch
+ )
+
+ # Mock aggregate interface for status queries
+ mock_aggregate = MagicMock()
+ mock_aggregate.total_count = 2
+ mock_collection.aggregate.over_all.return_value = mock_aggregate
+
+ yield mock_client, mock_collection
+
+
+def test_weaviate_storage_initialization(mock_weaviate_client):
+ r"""Test WeaviateStorage initialization functionality."""
+ mock_client, _ = mock_weaviate_client
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ ):
+ storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="test_collection",
+ connection_type="local",
+ )
+
+ assert storage.vector_dim == 4
+ assert storage.collection_name == "test_collection"
+ assert storage.connection_type == "local"
+ assert storage.distance_metric == "cosine"
+ assert storage.vector_index_type == "hnsw"
+
+
+def test_add_vectors(mock_weaviate_client):
+ r"""Test adding vectors to storage."""
+ mock_client, mock_collection = mock_weaviate_client
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ ):
+ storage = WeaviateStorage(
+ vector_dim=4, collection_name="test_collection"
+ )
+
+ # Prepare test vectors
+ vectors = [
+ VectorRecord(
+ id="1",
+ vector=[0.1, 0.2, 0.3, 0.4],
+ payload={"label": "A", "type": "test"},
+ ),
+ VectorRecord(
+ id="2",
+ vector=[0.5, 0.6, 0.7, 0.8],
+ payload={"label": "B", "type": "test"},
+ ),
+ ]
+
+ # Execute add operation
+ storage.add(vectors)
+
+ # Verify batch interface was called
+ mock_collection.batch.dynamic.assert_called_once()
+
+
+def test_query_vectors(mock_weaviate_client):
+ r"""Test querying vectors functionality."""
+ mock_client, mock_collection = mock_weaviate_client
+
+ # Mock query response
+ mock_object = MagicMock()
+ mock_object.uuid = "1111"
+ mock_object.properties = {
+ "payload": json.dumps({"label": "A", "type": "test"})
+ }
+ mock_object.vector = {"default": [0.1, 0.2, 0.3, 0.4]}
+ mock_object.metadata.distance = 0.1
+
+ mock_response = MagicMock()
+ mock_response.objects = [mock_object]
+ mock_collection.query.near_vector.return_value = mock_response
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ ):
+ storage = WeaviateStorage(
+ vector_dim=4,
+ collection_name="test_collection",
+ distance_metric="cosine",
+ )
+
+ # Execute query
+ query = VectorDBQuery(query_vector=[0.1, 0.2, 0.3, 0.4], top_k=1)
+ results = storage.query(query)
+
+ # Verify results
+ assert len(results) == 1
+ assert results[0].record.id == "1111"
+ assert results[0].record.payload == {"label": "A", "type": "test"}
+ assert results[0].record.vector == [0.1, 0.2, 0.3, 0.4]
+ assert results[0].similarity > 0 # Similarity should be greater than 0
+
+ # Verify query method was called correctly
+ mock_collection.query.near_vector.assert_called_once()
+
+
+def test_delete_vectors(mock_weaviate_client):
+ r"""Test deleting vectors functionality.
+
+ Verifies that vectors can be deleted by ID and that the
+ delete_many interface is called correctly.
+ """
+ mock_client, mock_collection = mock_weaviate_client
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ patch('weaviate.classes.query.Filter') as mock_filter,
+ ):
+ # Mock filter chain
+ mock_filter.by_id.return_value.contains_any.return_value = MagicMock()
+
+ storage = WeaviateStorage(
+ vector_dim=4, collection_name="test_collection"
+ )
+
+ # Execute delete operation
+ ids_to_delete = ["1111", "222"]
+ storage.delete(ids_to_delete)
+
+ # Verify delete method was called
+ mock_collection.data.delete_many.assert_called_once()
+
+
+def test_status(mock_weaviate_client):
+ r"""Test getting storage status functionality.
+
+ Verifies that storage status information including vector dimension
+ and vector count can be retrieved correctly.
+ """
+ mock_client, mock_collection = mock_weaviate_client
+
+ # Mock aggregate results
+ mock_aggregate = MagicMock()
+ mock_aggregate.total_count = 5
+ mock_collection.aggregate.over_all.return_value = mock_aggregate
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ ):
+ storage = WeaviateStorage(
+ vector_dim=4, collection_name="test_collection"
+ )
+
+ # Get status
+ status = storage.status()
+
+ # Verify status information
+ assert status.vector_dim == 4
+ assert status.vector_count == 5
+
+ # Verify aggregate method was called
+ mock_collection.aggregate.over_all.assert_called_once()
+
+
+def test_clear_collection(mock_weaviate_client):
+ r"""Test clearing collection functionality.
+
+ Verifies that the collection can be cleared by deleting and
+ recreating it properly.
+ """
+ mock_client, mock_collection = mock_weaviate_client
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._create_collection'
+ ) as mock_create,
+ ):
+ storage = WeaviateStorage(
+ vector_dim=4, collection_name="test_collection"
+ )
+
+ # Execute clear operation
+ storage.clear()
+
+ # Verify collection was deleted and recreated
+ mock_client.collections.delete.assert_called_once_with(
+ "test_collection"
+ )
+ mock_create.assert_called_once()
+
+
+def test_empty_operations(mock_weaviate_client):
+ r"""Test handling of empty operations.
+
+ Verifies that empty add and delete operations are handled gracefully
+ without calling the underlying APIs.
+ """
+ mock_client, mock_collection = mock_weaviate_client
+
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',
+ return_value=mock_client,
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ ):
+ storage = WeaviateStorage(
+ vector_dim=4, collection_name="test_collection"
+ )
+
+ # Test empty add
+ storage.add([])
+ mock_collection.batch.dynamic.assert_not_called()
+
+ # Test empty delete
+ storage.delete([])
+ mock_collection.data.delete_many.assert_not_called()
+
+
+def test_similarity_calculations():
+ r"""Test similarity calculation functionality.
+
+ Verifies that different distance metrics (cosine, L2, hamming) are
+ correctly converted to similarity scores.
+ """
+ with (
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client'
+ ),
+ patch(
+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'
+ ),
+ ):
+ # Test cosine distance
+ storage_cosine = WeaviateStorage(
+ vector_dim=4, distance_metric="cosine"
+ )
+ similarity = storage_cosine._calculate_similarity_from_distance(0.2)
+ assert similarity == 0.9 # 1 - (0.2 / 2)
+
+ # Test L2 distance
+ storage_l2 = WeaviateStorage(
+ vector_dim=4, distance_metric="l2-squared"
+ )
+ similarity = storage_l2._calculate_similarity_from_distance(1.0)
+ assert similarity == 0.5 # 1 / (1 + 1)
+
+ # Test Hamming distance
+ storage_hamming = WeaviateStorage(
+ vector_dim=4, distance_metric="hamming"
+ )
+ similarity = storage_hamming._calculate_similarity_from_distance(2.0)
+ assert similarity == 0.5 # 1 - (2 / 4)
diff --git a/uv.lock b/uv.lock
index c12e8b419..8ace8eabe 100644
--- a/uv.lock
+++ b/uv.lock
@@ -404,6 +404,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815 },
]
+[[package]]
+name = "authlib"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cryptography" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/09/47/df70ecd34fbf86d69833fe4e25bb9ecbaab995c8e49df726dd416f6bb822/authlib-1.3.1.tar.gz", hash = "sha256:7ae843f03c06c5c0debd63c9db91f9fda64fa62a42a77419fa15fbb7e7a58917", size = 146074 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/87/1f/bc95e43ffb57c05b8efcc376dd55a0240bf58f47ddf5a0f92452b6457b75/Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377", size = 223827 },
+]
+
[[package]]
name = "av"
version = "14.4.0"
@@ -692,6 +704,7 @@ all = [
{ name = "types-setuptools" },
{ name = "types-tqdm" },
{ name = "unstructured" },
+ { name = "weaviate-client" },
{ name = "wikipedia" },
{ name = "wolframalpha" },
{ name = "xls2xlsx" },
@@ -870,6 +883,7 @@ rag = [
{ name = "rank-bm25" },
{ name = "sentence-transformers" },
{ name = "unstructured" },
+ { name = "weaviate-client" },
]
research-tools = [
{ name = "arxiv" },
@@ -889,6 +903,7 @@ storage = [
{ name = "pytidb-experimental" },
{ name = "qdrant-client" },
{ name = "redis" },
+ { name = "weaviate-client" },
]
test = [
{ name = "mock" },
@@ -1232,6 +1247,9 @@ requires-dist = [
{ name = "unstructured", marker = "extra == 'owl'", specifier = "==0.16.20" },
{ name = "unstructured", marker = "extra == 'rag'", specifier = "==0.16.20" },
{ name = "uv", marker = "extra == 'dev'", specifier = "==0.6.5" },
+ { name = "weaviate-client", marker = "extra == 'all'", specifier = ">=4.15.0" },
+ { name = "weaviate-client", marker = "extra == 'rag'", specifier = ">=4.15.0" },
+ { name = "weaviate-client", marker = "extra == 'storage'", specifier = ">=4.15.0" },
{ name = "wikipedia", marker = "extra == 'all'", specifier = ">=1,<2" },
{ name = "wikipedia", marker = "extra == 'owl'", specifier = ">=1,<2" },
{ name = "wikipedia", marker = "extra == 'web-tools'", specifier = ">=1,<2" },
@@ -1826,6 +1844,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998 },
]
+[[package]]
+name = "deprecation"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "packaging" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178 },
+]
+
[[package]]
name = "diffusers"
version = "0.25.1"
@@ -2748,6 +2778,59 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574 },
]
+[[package]]
+name = "grpcio-health-checking"
+version = "1.67.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "grpcio" },
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/64/dd/e3b339fa44dc75b501a1a22cb88f1af5b1f8c964488f19c4de4cfbbf05ba/grpcio_health_checking-1.67.1.tar.gz", hash = "sha256:ca90fa76a6afbb4fda71d734cb9767819bba14928b91e308cffbb0c311eb941e", size = 16775 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5c/8d/7a9878dca6616b48093d71c52d0bc79cb2dd1a2698ff6f5ce7406306de12/grpcio_health_checking-1.67.1-py3-none-any.whl", hash = "sha256:93753da5062152660aef2286c9b261e07dd87124a65e4dc9fbd47d1ce966b39d", size = 18924 },
+]
+
+[[package]]
+name = "grpcio-tools"
+version = "1.67.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "grpcio" },
+ { name = "protobuf" },
+ { name = "setuptools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/6facde12a5a8da4398a3a8947f8ba6ef33b408dfc9767c8cefc0074ddd68/grpcio_tools-1.67.1.tar.gz", hash = "sha256:d9657f5ddc62b52f58904e6054b7d8a8909ed08a1e28b734be3a707087bcf004", size = 5159073 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e5/46/668e681e2e4ca7dc80cb5ad22bc794958c8b604b5b3143f16b94be3c0118/grpcio_tools-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:c701aaa51fde1f2644bd94941aa94c337adb86f25cd03cf05e37387aaea25800", size = 2308117 },
+ { url = "https://files.pythonhosted.org/packages/d6/56/1c65fb7c836cd40470f1f1a88185973466241fdb42b42b7a83367c268622/grpcio_tools-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:6a722bba714392de2386569c40942566b83725fa5c5450b8910e3832a5379469", size = 5500152 },
+ { url = "https://files.pythonhosted.org/packages/01/ab/caf9c330241d843a83043b023e2996e959cdc2c3ab404b1a9938eb734143/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0c7415235cb154e40b5ae90e2a172a0eb8c774b6876f53947cf0af05c983d549", size = 2282055 },
+ { url = "https://files.pythonhosted.org/packages/75/e6/0cd849d140b58fedb7d3b15d907fe2eefd4dadff09b570dd687d841c5d00/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4c459098c4934f9470280baf9ff8b38c365e147f33c8abc26039a948a664a5", size = 2617360 },
+ { url = "https://files.pythonhosted.org/packages/b9/51/bd73cd6515c2e81ba0a29b3cf6f2f62ad94737326f70b32511d1972a383e/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e89bf53a268f55c16989dab1cf0b32a5bff910762f138136ffad4146129b7a10", size = 2416028 },
+ { url = "https://files.pythonhosted.org/packages/47/e5/6a16e23036f625b6d60b579996bb9bb7165485903f934d9d9d73b3f03ef5/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f09cb3e6bcb140f57b878580cf3b848976f67faaf53d850a7da9bfac12437068", size = 3224906 },
+ { url = "https://files.pythonhosted.org/packages/14/cb/230c17d4372fa46fc799a822f25fa00c8eb3f85cc86e192b9606a17f732f/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:616dd0c6686212ca90ff899bb37eb774798677e43dc6f78c6954470782d37399", size = 2870384 },
+ { url = "https://files.pythonhosted.org/packages/66/fd/6d9dd3bf5982ab7d7e773f055360185e96a96cf95f2cbc7f53ded5912ef5/grpcio_tools-1.67.1-cp310-cp310-win32.whl", hash = "sha256:58a66dbb3f0fef0396737ac09d6571a7f8d96a544ce3ed04c161f3d4fa8d51cc", size = 941138 },
+ { url = "https://files.pythonhosted.org/packages/6a/97/2fd5ebd996c12b2cb1e1202ee4a03cac0a65ba17d29dd34253bfe2079839/grpcio_tools-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:89ee7c505bdf152e67c2cced6055aed4c2d4170f53a2b46a7e543d3b90e7b977", size = 1091151 },
+ { url = "https://files.pythonhosted.org/packages/b5/9a/ec06547673c5001c2604637069ff8f287df1aef3f0f8809b09a1c936b049/grpcio_tools-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:6d80ddd87a2fb7131d242f7d720222ef4f0f86f53ec87b0a6198c343d8e4a86e", size = 2307990 },
+ { url = "https://files.pythonhosted.org/packages/ca/84/4b7c3c27a2972c00b3b6ccaadd349e0f86b7039565d3a4932e219a4d76e0/grpcio_tools-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b655425b82df51f3bd9fd3ba1a6282d5c9ce1937709f059cb3d419b224532d89", size = 5526552 },
+ { url = "https://files.pythonhosted.org/packages/a7/2d/a620e4c53a3b808ebecaa5033c2176925ee1c6cbb45c29af8bec9a249822/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:250241e6f9d20d0910a46887dfcbf2ec9108efd3b48f3fb95bb42d50d09d03f8", size = 2282137 },
+ { url = "https://files.pythonhosted.org/packages/ec/29/e188b2e438781b37532abb8f10caf5b09c611a0bf9a09940b4cf303afd5b/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6008f5a5add0b6f03082edb597acf20d5a9e4e7c55ea1edac8296c19e6a0ec8d", size = 2617333 },
+ { url = "https://files.pythonhosted.org/packages/86/aa/2bbccd3c34b1fa48b892fbad91525c33a8aa85cbedd50e8b0d17dc260dc3/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5eff9818c3831fa23735db1fa39aeff65e790044d0a312260a0c41ae29cc2d9e", size = 2415806 },
+ { url = "https://files.pythonhosted.org/packages/db/34/99853a8ced1119937d02511476018dc1d6b295a4803d4ead5dbf9c55e9bc/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:262ab7c40113f8c3c246e28e369661ddf616a351cb34169b8ba470c9a9c3b56f", size = 3224765 },
+ { url = "https://files.pythonhosted.org/packages/66/39/8537a8ace8f6242f2058677e56a429587ec731c332985af34f35d496ca58/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1eebd8c746adf5786fa4c3056258c21cc470e1eca51d3ed23a7fb6a697fe4e81", size = 2870446 },
+ { url = "https://files.pythonhosted.org/packages/28/2a/5c04375adccff58647d48675e055895c31811a0ad896e4ba310833e2154d/grpcio_tools-1.67.1-cp311-cp311-win32.whl", hash = "sha256:3eff92fb8ca1dd55e3af0ef02236c648921fb7d0e8ca206b889585804b3659ae", size = 940890 },
+ { url = "https://files.pythonhosted.org/packages/e6/ee/7861339c2cec8d55a5e859cf3682bda34eab5a040f95d0c80f775d6a3279/grpcio_tools-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:1ed18281ee17e5e0f9f6ce0c6eb3825ca9b5a0866fc1db2e17fab8aca28b8d9f", size = 1091094 },
+ { url = "https://files.pythonhosted.org/packages/d9/cf/7b1908ca72e484bac555431036292c48d2d6504a45e2789848cb5ff313a8/grpcio_tools-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:bd5caef3a484e226d05a3f72b2d69af500dca972cf434bf6b08b150880166f0b", size = 2307645 },
+ { url = "https://files.pythonhosted.org/packages/bb/15/0d1efb38af8af7e56b2342322634a3caf5f1337a6c3857a6d14aa590dfdf/grpcio_tools-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:48a2d63d1010e5b218e8e758ecb2a8d63c0c6016434e9f973df1c3558917020a", size = 5525468 },
+ { url = "https://files.pythonhosted.org/packages/52/42/a810709099f09ade7f32990c0712c555b3d7eab6a05fb62618c17f8fe9da/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:baa64a6aa009bffe86309e236c81b02cd4a88c1ebd66f2d92e84e9b97a9ae857", size = 2281768 },
+ { url = "https://files.pythonhosted.org/packages/4c/2a/64ee6cfdf1c32ef8bdd67bf04ae2f745f517f4a546281453ca1f68fa79ca/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ab318c40b5e3c097a159035fc3e4ecfbe9b3d2c9de189e55468b2c27639a6ab", size = 2617359 },
+ { url = "https://files.pythonhosted.org/packages/79/7f/1ed8cd1529253fef9cf0ef3cd8382641125a5ca2eaa08eaffbb549f84e0b/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50eba3e31f9ac1149463ad9182a37349850904f142cffbd957cd7f54ec320b8e", size = 2415323 },
+ { url = "https://files.pythonhosted.org/packages/8e/08/59f0073c58703c176c15fb1a838763b77c1c06994adba16654b92a666e1b/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:de6fbc071ecc4fe6e354a7939202191c1f1abffe37fbce9b08e7e9a5b93eba3d", size = 3225051 },
+ { url = "https://files.pythonhosted.org/packages/b7/0d/a5d703214fe49d261b4b8f0a64140a4dc1f88560724a38ad937120b899ad/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db9e87f6ea4b0ce99b2651203480585fd9e8dd0dd122a19e46836e93e3a1b749", size = 2870421 },
+ { url = "https://files.pythonhosted.org/packages/ac/af/41d79cb87eae99c0348e8f1fb3dbed9e40a6f63548b216e99f4d1165fa5c/grpcio_tools-1.67.1-cp312-cp312-win32.whl", hash = "sha256:6a595a872fb720dde924c4e8200f41d5418dd6baab8cc1a3c1e540f8f4596351", size = 940542 },
+ { url = "https://files.pythonhosted.org/packages/66/e5/096e12f5319835aa2bcb746d49ae62220bb48313ca649e89bdbef605c11d/grpcio_tools-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:92eebb9b31031604ae97ea7657ae2e43149b0394af7117ad7e15894b6cc136dc", size = 1090425 },
+]
+
[[package]]
name = "h11"
version = "0.16.0"
@@ -7863,6 +7946,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b1/4b/4cef6ce21a2aaca9d852a6e84ef4f135d99fcd74fa75105e2fc0c8308acd/uvicorn-0.34.2-py3-none-any.whl", hash = "sha256:deb49af569084536d269fe0a6d67e3754f104cf03aba7c11c40f01aadf33c403", size = 62483 },
]
+[[package]]
+name = "validators"
+version = "0.34.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/64/07/91582d69320f6f6daaf2d8072608a4ad8884683d4840e7e4f3a9dbdcc639/validators-0.34.0.tar.gz", hash = "sha256:647fe407b45af9a74d245b943b18e6a816acf4926974278f6dd617778e1e781f", size = 70955 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6e/78/36828a4d857b25896f9774c875714ba4e9b3bc8a92d2debe3f4df3a83d4f/validators-0.34.0-py3-none-any.whl", hash = "sha256:c804b476e3e6d3786fa07a30073a4ef694e617805eb1946ceee3fe5a9b8b1321", size = 43536 },
+]
+
[[package]]
name = "virtualenv"
version = "20.31.2"
@@ -7886,6 +7978,25 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166 },
]
+[[package]]
+name = "weaviate-client"
+version = "4.15.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "authlib" },
+ { name = "deprecation" },
+ { name = "grpcio" },
+ { name = "grpcio-health-checking" },
+ { name = "grpcio-tools" },
+ { name = "httpx" },
+ { name = "pydantic" },
+ { name = "validators" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/90/98/53544475466ec35260c8cdc201fd7a9e9731d3ed596121ffc912fab782cf/weaviate_client-4.15.0.tar.gz", hash = "sha256:0e1c06e0bb08c8ab6987d91cf024b625f517b349965e6dee5439f675ff90cc64", size = 663838 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b8/eb/cebfb6b401e005e2e0b5ad89e43b674cb962c7e9ec386bb6f5874c8b21a2/weaviate_client-4.15.0-py3-none-any.whl", hash = "sha256:36bb83dacfc102cab0267aa211eb21eaca31cc80921b977451d814532c9a50e6", size = 433671 },
+]
+
[[package]]
name = "webcolors"
version = "24.11.1"