🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add Weaviate vector storage #2451 (#2581)
Lucas-CX1ed730fa0460
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.pyindex 940b07184..0ff8d6767 100644--- a/camel/storages/__init__.py+++ b/camel/storages/__init__.py@@ -31,6 +31,7 @@ from .vectordb_storages.milvus import MilvusStoragefrom .vectordb_storages.oceanbase import OceanBaseStoragefrom .vectordb_storages.qdrant import QdrantStoragefrom .vectordb_storages.tidb import TiDBStorage+from .vectordb_storages.weaviate import WeaviateStorage__all__ = ['BaseKeyValueStorage',@@ -50,4 +51,5 @@ __all__ = ['NebulaGraph','Mem0Storage','OceanBaseStorage',+ 'WeaviateStorage',]diff --git a/camel/storages/vectordb_storages/__init__.py b/camel/storages/vectordb_storages/__init__.pyindex 3b8b020a8..4f04e6a7f 100644--- a/camel/storages/vectordb_storages/__init__.py+++ b/camel/storages/vectordb_storages/__init__.py@@ -24,6 +24,7 @@ from .milvus import MilvusStoragefrom .oceanbase import OceanBaseStoragefrom .qdrant import QdrantStoragefrom .tidb import TiDBStorage+from .weaviate import WeaviateStorage__all__ = ['BaseVectorStorage',@@ -34,6 +35,7 @@ __all__ = ["TiDBStorage",'FaissStorage','OceanBaseStorage',+ 'WeaviateStorage','VectorRecord','VectorDBStatus',]diff --git a/camel/storages/vectordb_storages/weaviate.py b/camel/storages/vectordb_storages/weaviate.pynew file mode 100644index 000000000..9a7a2dd31--- /dev/null+++ b/camel/storages/vectordb_storages/weaviate.py@@ -0,0 +1,713 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import json+import logging+import math+import re+from datetime import datetime+from typing import (+ TYPE_CHECKING,+ Any,+ Dict,+ List,+ Literal,+ Optional,+ Union,+ cast,+)++if TYPE_CHECKING:+ from weaviate import WeaviateClient++from camel.storages.vectordb_storages import (+ BaseVectorStorage,+ VectorDBQuery,+ VectorDBQueryResult,+ VectorDBStatus,+ VectorRecord,+)+from camel.utils import dependencies_required++logger = logging.getLogger(__name__)+++# Type definitions for configuration options+ConnectionType = Literal["local", "cloud", "embedded", "custom"]+VectorIndexType = Literal["hnsw", "flat"]+DistanceMetric = Literal["cosine", "dot", "l2-squared", "hamming", "manhattan"]+++class WeaviateStorage(BaseVectorStorage):+ r"""An implementation of the `BaseVectorStorage` for interacting with+ Weaviate, a cloud-native vector search engine.++ This class provides multiple ways to connect to Weaviate instances:+ - Weaviate Cloud (WCD)+ - Local Docker/Kubernetes instances+ - Embedded Weaviate+ - Custom connection parameters++ Args:+ vector_dim (int): The dimension of storing vectors.+ collection_name (Optional[str], optional): Name for the collection in+ Weaviate. If not provided, generates a unique name based on current+ timestamp. (default: :obj:`None`)+ connection_type (ConnectionType, optional): Type of connection to use.+ Supported types: 'local', 'cloud', 'embedded', 'custom'.+ (default: :obj:`"local"`)++ # Weaviate Cloud parameters+ wcd_cluster_url (Optional[str], optional): Weaviate Cloud cluster URL.+ Required when connection_type='cloud'.+ wcd_api_key (Optional[str], optional): Weaviate Cloud API key.+ Required when connection_type='cloud'.++ # Local instance parameters+ local_host (str, optional): Local Weaviate host.+ (default: :obj:`"localhost"`)+ local_port (int, optional): Local Weaviate HTTP port.+ (default: :obj:`8080`)+ local_grpc_port (int, optional): Local Weaviate gRPC port.+ (default: :obj:`50051`)+ local_auth_credentials (Optional[Union[str, Any]], optional):+ Authentication credentials for local instance. Can be an API key+ string or Auth object. (default: :obj:`None`)++ # Embedded Weaviate parameters+ embedded_hostname (str, optional): Embedded instance hostname.+ (default: :obj:`"127.0.0.1"`)+ embedded_port (int, optional): Embedded instance HTTP port.+ (default: :obj:`8079`)+ embedded_grpc_port (int, optional): Embedded instance gRPC port.+ (default: :obj:`50050`)+ embedded_version (Optional[str], optional): Weaviate version for+ embedded instance. If None, uses the default version.+ (default: :obj:`None`)+ embedded_persistence_data_path (Optional[str], optional): Directory+ for embedded database files. (default: :obj:`None`)+ embedded_binary_path (Optional[str], optional): Directory for+ Weaviate binary. (default: :obj:`None`)+ embedded_environment_variables (Optional[Dict[str, str]], optional):+ Environment variables for embedded instance. (default: :obj:`None`)++ # Custom connection parameters+ custom_http_host (Optional[str], optional): Custom HTTP host.+ custom_http_port (Optional[int], optional): Custom HTTP port.+ custom_http_secure (Optional[bool], optional): Use HTTPS.+ custom_grpc_host (Optional[str], optional): Custom gRPC host.+ custom_grpc_port (Optional[int], optional): Custom gRPC port.+ custom_grpc_secure (Optional[bool], optional): Use secure gRPC.+ custom_auth_credentials (Optional[Any], optional): Custom auth.++ # Vector index configuration parameters+ vector_index_type (VectorIndexType, optional): Vector index type.+ Supported types: 'hnsw', 'flat'. (default: :obj:`"hnsw"`)+ distance_metric (DistanceMetric, optional): Distance metric for vector+ similarity. Supported metrics: 'cosine', 'dot', 'l2-squared',+ 'hamming', 'manhattan'. (default: :obj:`"cosine"`)++ # Common parameters for all connection types+ headers (Optional[Dict[str, str]], optional): Additional headers for+ third-party API keys (e.g., OpenAI, Cohere). (default: :obj:`None`)+ additional_config (Optional[Any], optional): Advanced configuration+ options like timeouts. (default: :obj:`None`)+ skip_init_checks (bool, optional): Skip initialization checks.+ (default: :obj:`False`)++ Raises:+ ImportError: If `weaviate` package is not installed.+ ValueError: If connection parameters are invalid or missing.+ RuntimeError: If there's an error setting up the collection.++++ Note:+ This implementation supports synchronous operations only.+ The client connection is automatically handled and closed when the+ storage instance is destroyed.+ """++ @dependencies_required('weaviate')+ def __init__(+ self,+ vector_dim: int,+ collection_name: Optional[str] = None,+ connection_type: ConnectionType = "local",+ # Weaviate Cloud parameters+ wcd_cluster_url: Optional[str] = None,+ wcd_api_key: Optional[str] = None,+ # Local instance parameters+ local_host: str = "localhost",+ local_port: int = 8080,+ local_grpc_port: int = 50051,+ local_auth_credentials: Optional[Union[str, Any]] = None,+ # Embedded Weaviate parameters+ embedded_hostname: str = "127.0.0.1",+ embedded_port: int = 8079,+ embedded_grpc_port: int = 50050,+ embedded_version: Optional[str] = None,+ embedded_persistence_data_path: Optional[str] = None,+ embedded_binary_path: Optional[str] = None,+ embedded_environment_variables: Optional[Dict[str, str]] = None,+ # Custom connection parameters+ custom_http_host: Optional[str] = None,+ custom_http_port: Optional[int] = None,+ custom_http_secure: Optional[bool] = None,+ custom_grpc_host: Optional[str] = None,+ custom_grpc_port: Optional[int] = None,+ custom_grpc_secure: Optional[bool] = None,+ custom_auth_credentials: Optional[Any] = None,+ # Vector index configuration parameters+ vector_index_type: VectorIndexType = "hnsw",+ distance_metric: DistanceMetric = "cosine",+ # Common parameters+ headers: Optional[Dict[str, str]] = None,+ additional_config: Optional[Any] = None,+ skip_init_checks: bool = False,+ **kwargs: Any,+ ) -> None:+ self.vector_dim = vector_dim+ self.collection_name = (+ collection_name or self._generate_collection_name()+ )++ # Store vector index configuration+ self.vector_index_type: VectorIndexType = vector_index_type+ self.distance_metric: DistanceMetric = distance_metric++ # Store connection type configuration+ self.connection_type: ConnectionType = connection_type++ # Store connection parameters for later use+ self._connection_params = {+ 'wcd_cluster_url': wcd_cluster_url,+ 'wcd_api_key': wcd_api_key,+ 'local_host': local_host,+ 'local_port': local_port,+ 'local_grpc_port': local_grpc_port,+ 'local_auth_credentials': local_auth_credentials,+ 'embedded_hostname': embedded_hostname,+ 'embedded_port': embedded_port,+ 'embedded_grpc_port': embedded_grpc_port,+ 'embedded_version': embedded_version,+ 'embedded_persistence_data_path': embedded_persistence_data_path,+ 'embedded_binary_path': embedded_binary_path,+ 'embedded_environment_variables': embedded_environment_variables,+ 'custom_http_host': custom_http_host,+ 'custom_http_port': custom_http_port,+ 'custom_http_secure': custom_http_secure,+ 'custom_grpc_host': custom_grpc_host,+ 'custom_grpc_port': custom_grpc_port,+ 'custom_grpc_secure': custom_grpc_secure,+ 'custom_auth_credentials': custom_auth_credentials,+ 'headers': headers,+ 'additional_config': additional_config,+ 'skip_init_checks': skip_init_checks,+ **kwargs,+ }++ # Create client using the new unified approach+ self._client = self._get_connection_client()+ logger.info(+ f"Created Weaviate client with connection type: "+ f"{self.connection_type}"+ )++ self._check_and_create_collection(**kwargs)++ def _get_connection_client(self) -> "WeaviateClient":+ r"""Get Weaviate client based on connection type and user settings."""+ import weaviate++ # Map connection types to handler methods+ connection_handlers: Dict[ConnectionType, Any] = {+ 'cloud': self._create_cloud_client,+ 'local': self._create_local_client,+ 'embedded': self._create_embedded_client,+ 'custom': self._create_custom_client,+ }++ # Get connection handler+ handler = connection_handlers[self.connection_type]++ try:+ return handler(weaviate)+ except Exception as e:+ logger.error(+ f"Failed to create {self.connection_type} client: {e}"+ )+ raise++ def _create_cloud_client(self, weaviate_module: Any) -> "WeaviateClient":+ r"""Create a Weaviate Cloud client."""+ cluster_url = self._connection_params.get('wcd_cluster_url')+ api_key = self._connection_params.get('wcd_api_key')++ if not cluster_url:+ raise ValueError(+ "wcd_cluster_url is required for cloud connection"+ )+ if not api_key:+ raise ValueError("wcd_api_key is required for cloud connection")++ return weaviate_module.connect_to_weaviate_cloud(+ cluster_url=cluster_url,+ auth_credentials=api_key,+ headers=self._connection_params.get('headers'),+ additional_config=self._connection_params.get('additional_config'),+ skip_init_checks=self._connection_params.get(+ 'skip_init_checks', False+ ),+ )++ def _create_local_client(self, weaviate_module: Any) -> "WeaviateClient":+ r"""Create a local Weaviate client."""+ return weaviate_module.connect_to_local(+ host=self._connection_params.get('local_host', 'localhost'),+ port=self._connection_params.get('local_port', 8080),+ grpc_port=self._connection_params.get('local_grpc_port', 50051),+ headers=self._connection_params.get('headers'),+ additional_config=self._connection_params.get('additional_config'),+ skip_init_checks=self._connection_params.get(+ 'skip_init_checks', False+ ),+ auth_credentials=self._connection_params.get(+ 'local_auth_credentials'+ ),+ )++ def _create_embedded_client(+ self, weaviate_module: Any+ ) -> "WeaviateClient":+ r"""Create an embedded Weaviate client."""+ embedded_kwargs = {+ 'hostname': self._connection_params.get(+ 'embedded_hostname', '127.0.0.1'+ ),+ 'port': self._connection_params.get('embedded_port', 8079),+ 'grpc_port': self._connection_params.get(+ 'embedded_grpc_port', 50050+ ),+ 'headers': self._connection_params.get('headers'),+ 'additional_config': self._connection_params.get(+ 'additional_config'+ ),+ }++ # Add optional embedded parameters+ if self._connection_params.get('embedded_version') is not None:+ embedded_kwargs['version'] = self._connection_params[+ 'embedded_version'+ ]+ if (+ self._connection_params.get('embedded_persistence_data_path')+ is not None+ ):+ embedded_kwargs['persistence_data_path'] = self._connection_params[+ 'embedded_persistence_data_path'+ ]+ if self._connection_params.get('embedded_binary_path') is not None:+ embedded_kwargs['binary_path'] = self._connection_params[+ 'embedded_binary_path'+ ]+ if (+ self._connection_params.get('embedded_environment_variables')+ is not None+ ):+ embedded_kwargs['environment_variables'] = self._connection_params[+ 'embedded_environment_variables'+ ]++ return weaviate_module.connect_to_embedded(**embedded_kwargs)++ def _create_custom_client(self, weaviate_module: Any) -> "WeaviateClient":+ r"""Create a custom Weaviate client."""+ # Validate required custom parameters+ required_params = [+ 'custom_http_host',+ 'custom_http_port',+ 'custom_http_secure',+ 'custom_grpc_host',+ 'custom_grpc_port',+ 'custom_grpc_secure',+ ]++ for param in required_params:+ if self._connection_params.get(param) is None:+ raise ValueError(f"{param} is required for custom connection")++ return weaviate_module.connect_to_custom(+ http_host=self._connection_params['custom_http_host'],+ http_port=self._connection_params['custom_http_port'],+ http_secure=self._connection_params['custom_http_secure'],+ grpc_host=self._connection_params['custom_grpc_host'],+ grpc_port=self._connection_params['custom_grpc_port'],+ grpc_secure=self._connection_params['custom_grpc_secure'],+ headers=self._connection_params.get('headers'),+ additional_config=self._connection_params.get('additional_config'),+ auth_credentials=self._connection_params.get(+ 'custom_auth_credentials'+ ),+ skip_init_checks=self._connection_params.get(+ 'skip_init_checks', False+ ),+ )++ def __del__(self):+ r"""Clean up client connection."""+ try:+ if hasattr(self, '_client') and self._client is not None:+ self._client.close()+ logger.debug("Closed Weaviate client connection")+ except Exception as e:+ logger.warning(f"Error closing Weaviate client: {e}")++ def close(self) -> None:+ r"""Explicitly close the client connection."""+ if self._client is not None:+ self._client.close()+ logger.info("Explicitly closed Weaviate client connection")++ def _generate_collection_name(self) -> str:+ r"""Generate a collection name if user doesn't provide one."""+ timestamp = datetime.now().isoformat()+ # Weaviate collection names must start with uppercase and be valid+ # GraphQL names+ valid_name = "Collection_" + re.sub(r'[^a-zA-Z0-9_]', '_', timestamp)+ return valid_name++ def _check_and_create_collection(self, **kwargs: Any) -> None:+ r"""Check if collection exists and create if it doesn't."""+ if not self._collection_exists(self.collection_name):+ self._create_collection(**kwargs)++ def _collection_exists(self, collection_name: str) -> bool:+ r"""Check if the collection exists."""+ try:+ collection = self._client.collections.get(collection_name)+ collection.config.get()+ return True+ except Exception:+ return False++ def _get_vector_index_config(self, **kwargs: Any) -> Any:+ r"""Get vector index configuration based on user settings."""+ import weaviate.classes.config as wvc++ # Map distance metrics - type safety guaranteed by Literal+ distance_metric_mapping: Dict[DistanceMetric, Any] = {+ 'cosine': wvc.VectorDistances.COSINE,+ 'dot': wvc.VectorDistances.DOT,+ 'l2-squared': wvc.VectorDistances.L2_SQUARED,+ 'hamming': wvc.VectorDistances.HAMMING,+ 'manhattan': wvc.VectorDistances.MANHATTAN,+ }++ # Get distance metric - no need for None check due to Literal typing+ distance_metric = distance_metric_mapping[self.distance_metric]++ # Configure vector index based on type+ if self.vector_index_type == 'hnsw':+ return wvc.Configure.VectorIndex.hnsw(+ distance_metric=distance_metric,+ **kwargs,+ )+ else: # must be 'flat' due to Literal typing+ return wvc.Configure.VectorIndex.flat(+ distance_metric=distance_metric,+ **kwargs,+ )++ def _create_collection(self, **kwargs: Any) -> None:+ r"""Create a new collection in Weaviate."""+ import weaviate.classes.config as wvc++ # Separate vector index kwargs from general kwargs to avoid conflicts+ vector_index_kwargs = {}+ collection_kwargs = {}++ # Known vector index parameters+ vector_index_params = {+ 'ef_construction',+ 'max_connections',+ 'ef',+ 'dynamic_ef_min',+ 'dynamic_ef_max',+ 'dynamic_ef_factor',+ 'vector_cache_max_objects',+ 'flat_search_cutoff',+ 'cleanup_interval_seconds',+ 'pq_enabled',+ 'pq_segments',+ 'pq_centroids',+ 'pq_training_limit',+ 'pq_encoder',+ }++ for key, value in kwargs.items():+ if key in vector_index_params:+ vector_index_kwargs[key] = value+ else:+ collection_kwargs[key] = value++ self._client.collections.create(+ name=self.collection_name,+ vectorizer_config=wvc.Configure.Vectorizer.none(),+ properties=[+ wvc.Property(name="payload", data_type=wvc.DataType.TEXT),+ ],+ vector_index_config=self._get_vector_index_config(+ **vector_index_kwargs+ ),+ **collection_kwargs,+ )++ def add(+ self,+ records: List[VectorRecord],+ **kwargs: Any,+ ) -> None:+ r"""Saves a list of vector records to the storage.++ Args:+ records (List[VectorRecord]): List of vector records to be saved.+ **kwargs (Any): Additional keyword arguments.++ Raises:+ RuntimeError: If there is an error during the saving process.+ """+ if not records:+ return++ try:+ collection = self._client.collections.get(self.collection_name)++ with collection.batch.dynamic() as batch:+ for record in records:+ payload_str = (+ json.dumps(record.payload) if record.payload else ""+ )+ batch.add_object(+ properties={"payload": payload_str},+ vector=record.vector,+ uuid=record.id,+ **kwargs,+ )+ logger.debug(+ f"Successfully added vectors to Weaviate collection: "+ f"{self.collection_name}"+ )++ except Exception as e:+ raise RuntimeError(f"Failed to add vectors to Weaviate: {e}")++ def delete(+ self,+ ids: List[str],+ **kwargs: Any,+ ) -> None:+ r"""Deletes a list of vectors identified by their IDs from the storage.++ Args:+ ids (List[str]): List of unique identifiers for the vectors to be+ deleted.+ **kwargs (Any): Additional keyword arguments.++ Raises:+ RuntimeError: If there is an error during the deletion process.+ """+ if not ids:+ return++ try:+ collection = self._client.collections.get(self.collection_name)+ from weaviate.classes.query import Filter++ collection.data.delete_many(+ where=Filter.by_id().contains_any(ids), **kwargs+ )+ logger.debug(+ f"Successfully deleted vectors in Weaviate: "+ f"{self.collection_name}"+ )++ except Exception as e:+ raise RuntimeError(f"Failed to delete vectors from Weaviate: {e}")++ def _calculate_similarity_from_distance(+ self, distance: Optional[float]+ ) -> float:+ r"""Calculate similarity score based on distance metric.++ Args:+ distance (Optional[float]): The distance value from Weaviate.++ Returns:+ float: Normalized similarity score between 0 and 1.+ """+ # Return 0.0 if distance is not available+ if distance is None:+ return 0.0++ # Calculate similarity based on distance metric+ if self.distance_metric == "cosine":+ # Cosine: 0 (identical) to 2 (opposite)+ # Convert to similarity: 1 - (distance / 2)+ return max(0.0, 1.0 - (distance / 2.0))++ elif self.distance_metric == "dot":+ # Dot product: Weaviate returns -dot,+ # smaller (more negative) is more similar+ # Use sigmoid function to normalize to [0,1] range+ # For dot product, we use: 1 / (1 + exp(distance))+ return 1.0 / (1.0 + math.exp(distance))++ elif self.distance_metric in ["l2-squared", "manhattan"]:+ # L2-squared and Manhattan: 0 (identical) to ∞+ # Convert to similarity using: 1 / (1 + distance)+ # This provides a smoother decay than exponential+ return 1.0 / (1.0 + distance)++ elif self.distance_metric == "hamming":+ # Hamming: 0 (identical) to vector_dim (completely different)+ # Convert to similarity: 1 - (distance / vector_dim)+ return max(0.0, 1.0 - (distance / self.vector_dim))++ else:+ # Unknown metric, return 0.0+ return 0.0++ def status(self) -> VectorDBStatus:+ r"""Returns status of the vector database.++ Returns:+ VectorDBStatus: The vector database status.+ """+ try:+ collection = self._client.collections.get(self.collection_name)+ objects = collection.aggregate.over_all(total_count=True)++ vector_count = (+ objects.total_count if objects.total_count is not None else 0+ )++ return VectorDBStatus(+ vector_dim=self.vector_dim, vector_count=vector_count+ )++ except Exception as e:+ logger.warning(f"Failed to get status from Weaviate: {e}")+ return VectorDBStatus(vector_dim=self.vector_dim, vector_count=0)++ def query(+ self,+ query: VectorDBQuery,+ **kwargs: Any,+ ) -> List[VectorDBQueryResult]:+ r"""Searches for similar vectors in the storage based on the provided+ query.++ Args:+ query (VectorDBQuery): The query object containing the search+ vector and the number of top similar vectors to retrieve.+ **kwargs (Any): Additional keyword arguments.++ Returns:+ List[VectorDBQueryResult]: A list of vectors retrieved from the+ storage based on similarity to the query vector.+ """+ from weaviate.classes.query import MetadataQuery++ try:+ collection = self._client.collections.get(self.collection_name)++ response = collection.query.near_vector(+ near_vector=query.query_vector,+ limit=query.top_k,+ include_vector=True,+ return_metadata=MetadataQuery(distance=True),+ **kwargs,+ )++ results = []+ for obj in response.objects:+ # Calculate similarity score based on distance metric+ similarity = self._calculate_similarity_from_distance(+ obj.metadata.distance+ )++ # Handle payload+ payload = None+ if obj.properties.get('payload'):+ payload_value = obj.properties['payload']+ if isinstance(payload_value, str):+ try:+ payload = json.loads(payload_value)+ except (json.JSONDecodeError, TypeError):+ payload = {"raw": payload_value}+ else:+ payload = {"raw": str(payload_value)}++ # Handle vector data+ # In newer versions of Weaviate, obj.vector returns a dict+ # with 'default' key++ vector: List[float] = []+ if hasattr(obj, 'vector') and obj.vector is not None:+ # New format: {'default': [vector_values]}+ vector_data = obj.vector.get('default', [])+ if (+ vector_data+ and isinstance(vector_data, list)+ and len(vector_data) > 0+ and isinstance(vector_data[0], float)+ ):+ vector = cast(List[float], vector_data)+ else:+ # unsupported vector data format+ vector = []+ else:+ vector = []++ result = VectorDBQueryResult.create(+ similarity=similarity,+ vector=vector,+ id=str(obj.uuid),+ payload=payload,+ )+ results.append(result)++ return results++ except Exception as e:+ raise RuntimeError(f"Failed to query vectors from Weaviate: {e}")++ def clear(self) -> None:+ r"""Remove all vectors from the storage."""+ try:+ self._client.collections.delete(self.collection_name)+ self._create_collection()+ except Exception as e:+ raise RuntimeError(f"Failed to clear Weaviate collection: {e}")++ def load(self) -> None:+ r"""Load the collection hosted on cloud service."""+ # For Weaviate, collections are automatically available when client+ # connects+ pass++ @property+ def client(self) -> "WeaviateClient":+ r"""Provides access to the underlying vector database client."""+ return self._clientdiff --git a/docs/get_started/installation.md b/docs/get_started/installation.mdindex e6c918a99..80c1cf594 100644--- a/docs/get_started/installation.md+++ b/docs/get_started/installation.md@@ -19,7 +19,7 @@ Available extras:- `all`: Includes all features below- `model_platforms`: OpenAI, Google, Mistral, Anthropic Claude, Cohere etc.- `huggingface`: Transformers, Diffusers, Accelerate, Datasets, PyTorch etc.-- `rag`: Sentence Transformers, Qdrant, Milvus, TiDB, BM25,OceanBase etc.+- `rag`: Sentence Transformers, Qdrant, Milvus, TiDB, BM25,OceanBase, Weaviate etc.- `storage`: Neo4j, Redis, Azure Blob, Google Cloud Storage, AWS S3 etc.- `web_tools`: DuckDuckGo, Wikipedia, WolframAlpha, Google Maps, Weather API etc.- `document_tools`: PDF, Word, OpenAPI, BeautifulSoup, Unstructured etc.diff --git a/docs/key_modules/storages.md b/docs/key_modules/storages.mdindex d16ef851b..71b99ecda 100644--- a/docs/key_modules/storages.md+++ b/docs/key_modules/storages.md@@ -62,6 +62,13 @@ Reference: [Qdrant](https://qdrant.tech/)Reference: [OceanBase](https://www.oceanbase.com/)+**`WeaviateStorage`**:++- Description: A concrete implementation of `BaseVectorStorage`, tailored for interacting with Weaviate, a vector search engine.++Reference: [Weaviate](https://weaviate.io/)++### 2.3 Graph Storages**`BaseGraphStorage`**:@@ -374,7 +381,47 @@ Vector count after clearing: 0```-### 3.7. Using `NebulaGraph`+### 3.7. Using `WeaviateStorage`++```python+from camel.storages import WeaviateStorage, VectorDBQuery, VectorRecord++# Create WeaviateStorage instance with dimension = 4 using Weaviate Cloud+weaviate_storage = WeaviateStorage(+ vector_dim=4,+ collection_name="camel_example_vectors",+ connection_type="cloud",+ wcd_cluster_url="your-weaviate-cloud-url",+ wcd_api_key="your-weaviate-api-key",+ vector_index_type="hnsw",+ distance_metric="cosine",+)++# Add vectors records to Weaviate+weaviate_storage.add([+ VectorRecord(+ vector=[-0.1, 0.1, -0.1, 0.1],+ payload={'key1': 'value1'},+ ),+ VectorRecord(+ vector=[-0.1, 0.1, 0.1, 0.1],+ payload={'key2': 'value2'},+ ),+])++# Query Weaviate for similar vectors+query_results = weaviate_storage.query(VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1))+for result in query_results:+ print(result.record.payload, result.similarity)++# Clear all vectors+weaviate_storage.clear()+```+```markdown+>>> {'key2': 'value2'} 0.7834733128547668+```++### 3.8. Using `NebulaGraph````pythonfrom camel.storages.graph_storages import NebulaGraph@@ -386,7 +433,7 @@ query = 'SHOW TAGS;'print(nebula_graph.query(query))```-### 3.8. Using `Neo4jGraph`+### 3.9. Using `Neo4jGraph````pythonfrom camel.storages import Neo4jGraphdiff --git a/examples/storages/weaviate_vector_storage.py b/examples/storages/weaviate_vector_storage.pynew file mode 100644index 000000000..7ffdea503--- /dev/null+++ b/examples/storages/weaviate_vector_storage.py@@ -0,0 +1,312 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import random++from camel.storages.vectordb_storages import (+ VectorDBQuery,+ VectorRecord,+ WeaviateStorage,+)++"""+Before running this example, you need to setup a Weaviate instance(reference: https://weaviate.io/developers/weaviate):++(Option 1): Local Docker Weaviate:++1. Install Docker+2. Run the following command to start Weaviate:+ docker run -p 8080:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/+ weaviate:1.31.0+3. The example will automatically connect to the local instance++(Option 2): Weaviate Cloud Service (WCS):++1. Sign up at https://console.weaviate.cloud+2. Create a free Weaviate cluster+3. Get your API key and cluster URL+4. Update the WEAVIATE_URL and WEAVIATE_API_KEY variables below++(Option 3): Embedded Weaviate:++Only supports Linux and macOS systems (Windows is not supported)++(Option 4): Custom Connection:++Configure custom HTTP/gRPC endpoints for specialized deployments++Note: You may see deprecation warnings from the Weaviate client library:+"PydanticDeprecatedSince211: Accessing the 'model_fields' attribute..."+These warnings are from the Weaviate library itself and can be safely ignored.+They do not affect the functionality of the code.++"""+++def example_local_connection():+ r"""Example using local Weaviate instance (default connection type)."""+ print("=== Local Weaviate Connection Example ===")++ # Create WeaviateStorage instance with local connection (default)+ weaviate_storage = WeaviateStorage(+ vector_dim=4,+ collection_name="camel_local_example",+ connection_type="local",+ local_host="localhost",+ local_port=8080,+ local_grpc_port=50051,+ # Vector index configuration+ vector_index_type="hnsw",+ distance_metric="cosine",+ )++ run_storage_example(weaviate_storage)+++def example_cloud_connection():+ """Example using Weaviate Cloud Service."""+ print("=== Weaviate Cloud Connection Example ===")++ # Replace these with your Weaviate Cloud connection parameters+ WEAVIATE_URL = "your-weaviate-cloud-url"+ WEAVIATE_API_KEY = "your-weaviate-api-key"++ # Create WeaviateStorage instance with cloud connection+ weaviate_storage = WeaviateStorage(+ vector_dim=4,+ collection_name="camel_cloud_example",+ connection_type="cloud",+ wcd_cluster_url=WEAVIATE_URL,+ wcd_api_key=WEAVIATE_API_KEY,+ # Vector index configuration+ vector_index_type="hnsw",+ distance_metric="cosine",+ # Additional headers for third-party API keys+ headers={+ "X-OpenAI-Api-Key": "your-openai-api-key",+ "X-Cohere-Api-Key": "your-cohere-api-key",+ },+ )++ run_storage_example(weaviate_storage)+++def example_embedded_connection():+ """Example using embedded Weaviate."""+ print("=== Embedded Weaviate Connection Example ===")++ # Create WeaviateStorage instance with embedded connection+ weaviate_storage = WeaviateStorage(+ vector_dim=4,+ collection_name="camel_embedded_example",+ connection_type="embedded",+ embedded_hostname="127.0.0.1",+ embedded_port=8079,+ embedded_grpc_port=50050,+ # Optional: persist data+ embedded_persistence_data_path="./weaviate_data",+ # Optional: specify Weaviate version+ embedded_version="1.31.0",+ # Optional: environment variables+ embedded_environment_variables={+ "QUERY_DEFAULTS_LIMIT": "25",+ "DEFAULT_VECTORIZER_MODULE": "none",+ },+ # Vector index configuration+ vector_index_type="hnsw",+ distance_metric="cosine",+ )++ run_storage_example(weaviate_storage)+++def example_custom_connection():+ """Example using custom Weaviate connection."""+ print("=== Custom Weaviate Connection Example ===")++ # Create WeaviateStorage instance with custom connection+ weaviate_storage = WeaviateStorage(+ vector_dim=4,+ collection_name="camel_custom_example",+ connection_type="custom",+ custom_http_host="localhost",+ custom_http_port=8080,+ custom_http_secure=False,+ custom_grpc_host="localhost",+ custom_grpc_port=50051,+ custom_grpc_secure=False,+ # Optional: custom authentication+ custom_auth_credentials=None,+ # Vector index configuration+ vector_index_type="hnsw",+ distance_metric="cosine",+ )++ run_storage_example(weaviate_storage)+++def run_storage_example(weaviate_storage: WeaviateStorage):+ """Run the common storage operations example."""++ # Get database status+ status = weaviate_storage.status()+ print(f"Vector dimension: {status.vector_dim}")+ print(f"Initial vector count: {status.vector_count}")+ print(f"Vector index type: {weaviate_storage.vector_index_type}")+ print(f"Distance metric: {weaviate_storage.distance_metric}")++ # Generate vector records+ print("\nAdding vectors...")+ random.seed(20250605)+ vector_records = []+ for i in range(10):+ vector_records.append(+ VectorRecord(+ vector=[random.uniform(-1, 1) for _ in range(4)],+ payload={+ 'idx': i,+ 'category': 'example',+ 'description': f'Vector example {i}',+ },+ )+ )++ # Add vectors records to Weaviate+ weaviate_storage.add(vector_records)++ # Check updated status+ status = weaviate_storage.status()+ print(f"Vector count after adding: {status.vector_count}")++ # Query similar vectors, use the first vector as query vector+ print("\nQuerying similar vectors...")+ query_vector = vector_records[0].vector+ query_results = weaviate_storage.query(+ VectorDBQuery(query_vector=query_vector, top_k=3)+ )++ # Display query results+ for i, result in enumerate(query_results):+ print(f"Result {i+1}:")+ print(f" ID: {result.record.id}")+ print(f" Payload: {result.record.payload}")+ print(f" Similarity: {result.similarity:.4f}")+ # print(f" Vector: {result.record.vector}")++ # Test deletion, delete the first two vectors+ print("\nDeleting vectors...")+ ids_to_delete = [result.record.id for result in query_results[:2]]+ weaviate_storage.delete(ids_to_delete)++ # Verify deletion+ status = weaviate_storage.status()+ print(f"Vector count after deletion: {status.vector_count}")++ # Clear all vectors when done+ print("\nClearing all vectors...")+ weaviate_storage.clear()+ status = weaviate_storage.status()+ print(f"Vector count after clearing: {status.vector_count}")++ # Close connection explicitly+ weaviate_storage.close()+ print("Connection closed.\n")+++def main():+ """Main function demonstrating different connection types and+ configurations."""++ print(+ "This example demonstrates different Weaviate connection types and "+ "configurations."+ )+ print("Make sure you have the appropriate Weaviate instance running.\n")++ # Example 1: Local connection (requires local Weaviate instance)+ try:+ example_local_connection()+ except Exception as e:+ print(f"Local connection failed: {e}")+ print(+ "Make sure you have a local Weaviate instance running on "+ "localhost:8080\n"+ )++ # Example 2: Cloud connection (requires valid cloud credentials)+ # try:+ # example_cloud_connection()+ # except Exception as e:+ # print(f"Cloud connection failed: {e}")+ # print("Make sure you have valid Weaviate Cloud credentials\n")++ # Example 3: Embedded connection (automatically starts embedded Weaviate)+ # Note: Embedded Weaviate client is unsupported on Windows system+ # try:+ # example_embedded_connection()+ # except Exception as e:+ # print(f"Embedded connection failed: {e}")+ # print(+ # "Make sure you have weaviate-client with embedded support "+ # "installed\n"+ # )++ # Example 4: Custom connection (requires custom Weaviate setup)+ # try:+ # example_custom_connection()+ # except Exception as e:+ # print(f"Custom connection failed: {e}")+ # print("Make sure you have a custom Weaviate instance configured\n")+++if __name__ == "__main__":+ main()++"""+===============================================================================+This example demonstrates different Weaviate connection types and+configurations.+ Make sure you have the appropriate Weaviate instance running.++=== Local Weaviate Connection Example ===+Vector dimension: 4+Initial vector count: 0+Vector index type: hnsw+Distance metric: cosine++Adding vectors...+Vector count after adding: 10++Querying similar vectors...+Result 1:+ ID: 0b71d7a8-6913-4c21-8e8b-4ecdd164c4f9+ Payload: {'idx': 0, 'category': 'example', 'description': 'Vector example 0'}+ Similarity: 1.0000+Result 2:+ ID: 416fc9af-47ef-4648-a48a-98397b68b656+ Payload: {'idx': 1, 'category': 'example', 'description': 'Vector example 1'}+ Similarity: 0.8594+Result 3:+ ID: aac5509e-9e24-42c8-8b7e-33564c291d87+ Payload: {'idx': 9, 'category': 'example', 'description': 'Vector example 9'}+ Similarity: 0.8500++Deleting vectors...+Vector count after deletion: 8++Clearing all vectors...+Vector count after clearing: 0+Connection closed.+===============================================================================+"""diff --git a/pyproject.toml b/pyproject.tomlindex a0ea5ed89..02b913efc 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -73,6 +73,7 @@ rag = ["pymilvus>=2.4.0,<3","pytidb-experimental==0.0.1.dev4","faiss-cpu>=1.7.2,<2",+ "weaviate-client>=4.15.0","neo4j>=5.18.0,<6","nebula3-python==3.8.2","rank-bm25>=0.2.2,<0.3",@@ -207,6 +208,7 @@ storage = ["botocore>=1.35.3,<2","mem0ai>=0.1.73","pyobvector>=0.1.18",+ "weaviate-client>=4.15.0",]owl = ["numpy>=1.2,<=2.2",@@ -377,6 +379,7 @@ all = ["aci-sdk>=1.0.0b1","pylatex>=1.4.2","chunkr-ai>=0.0.50",+ "weaviate-client>=4.15.0","python-pptx>=1.0.2",]diff --git a/test/storages/vector_storages/test_weaviate.py b/test/storages/vector_storages/test_weaviate.pynew file mode 100644index 000000000..bd3773058--- /dev/null+++ b/test/storages/vector_storages/test_weaviate.py@@ -0,0 +1,330 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import json+from unittest.mock import MagicMock, patch++import pytest++from camel.storages import VectorDBQuery, VectorRecord, WeaviateStorage+++def mock_weaviate_client():+ r"""Create a mock Weaviate client with collections interface."""+ with patch('weaviate.connect_to_local') as mock_connect:+ mock_client = MagicMock()+ mock_connect.return_value = mock_client++ # Mock collections interface+ mock_collection = MagicMock()+ mock_client.collections.get.return_value = mock_collection+ mock_client.collections.create = MagicMock()+ mock_client.collections.delete = MagicMock()++ # Mock collection config+ mock_collection.config.get.return_value = {}++ # Mock batch interface+ mock_batch = MagicMock()+ mock_collection.batch.dynamic.return_value.__enter__.return_value = (+ mock_batch+ )++ # Mock aggregate interface for status queries+ mock_aggregate = MagicMock()+ mock_aggregate.total_count = 2+ mock_collection.aggregate.over_all.return_value = mock_aggregate++ yield mock_client, mock_collection+++def test_weaviate_storage_initialization(mock_weaviate_client):+ r"""Test WeaviateStorage initialization functionality."""+ mock_client, _ = mock_weaviate_client++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ ):+ storage = WeaviateStorage(+ vector_dim=4,+ collection_name="test_collection",+ connection_type="local",+ )++ assert storage.vector_dim == 4+ assert storage.collection_name == "test_collection"+ assert storage.connection_type == "local"+ assert storage.distance_metric == "cosine"+ assert storage.vector_index_type == "hnsw"+++def test_add_vectors(mock_weaviate_client):+ r"""Test adding vectors to storage."""+ mock_client, mock_collection = mock_weaviate_client++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ ):+ storage = WeaviateStorage(+ vector_dim=4, collection_name="test_collection"+ )++ # Prepare test vectors+ vectors = [+ VectorRecord(+ id="1",+ vector=[0.1, 0.2, 0.3, 0.4],+ payload={"label": "A", "type": "test"},+ ),+ VectorRecord(+ id="2",+ vector=[0.5, 0.6, 0.7, 0.8],+ payload={"label": "B", "type": "test"},+ ),+ ]++ # Execute add operation+ storage.add(vectors)++ # Verify batch interface was called+ mock_collection.batch.dynamic.assert_called_once()+++def test_query_vectors(mock_weaviate_client):+ r"""Test querying vectors functionality."""+ mock_client, mock_collection = mock_weaviate_client++ # Mock query response+ mock_object = MagicMock()+ mock_object.uuid = "1111"+ mock_object.properties = {+ "payload": json.dumps({"label": "A", "type": "test"})+ }+ mock_object.vector = {"default": [0.1, 0.2, 0.3, 0.4]}+ mock_object.metadata.distance = 0.1++ mock_response = MagicMock()+ mock_response.objects = [mock_object]+ mock_collection.query.near_vector.return_value = mock_response++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ ):+ storage = WeaviateStorage(+ vector_dim=4,+ collection_name="test_collection",+ distance_metric="cosine",+ )++ # Execute query+ query = VectorDBQuery(query_vector=[0.1, 0.2, 0.3, 0.4], top_k=1)+ results = storage.query(query)++ # Verify results+ assert len(results) == 1+ assert results[0].record.id == "1111"+ assert results[0].record.payload == {"label": "A", "type": "test"}+ assert results[0].record.vector == [0.1, 0.2, 0.3, 0.4]+ assert results[0].similarity > 0 # Similarity should be greater than 0++ # Verify query method was called correctly+ mock_collection.query.near_vector.assert_called_once()+++def test_delete_vectors(mock_weaviate_client):+ r"""Test deleting vectors functionality.++ Verifies that vectors can be deleted by ID and that the+ delete_many interface is called correctly.+ """+ mock_client, mock_collection = mock_weaviate_client++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ patch('weaviate.classes.query.Filter') as mock_filter,+ ):+ # Mock filter chain+ mock_filter.by_id.return_value.contains_any.return_value = MagicMock()++ storage = WeaviateStorage(+ vector_dim=4, collection_name="test_collection"+ )++ # Execute delete operation+ ids_to_delete = ["1111", "222"]+ storage.delete(ids_to_delete)++ # Verify delete method was called+ mock_collection.data.delete_many.assert_called_once()+++def test_status(mock_weaviate_client):+ r"""Test getting storage status functionality.++ Verifies that storage status information including vector dimension+ and vector count can be retrieved correctly.+ """+ mock_client, mock_collection = mock_weaviate_client++ # Mock aggregate results+ mock_aggregate = MagicMock()+ mock_aggregate.total_count = 5+ mock_collection.aggregate.over_all.return_value = mock_aggregate++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ ):+ storage = WeaviateStorage(+ vector_dim=4, collection_name="test_collection"+ )++ # Get status+ status = storage.status()++ # Verify status information+ assert status.vector_dim == 4+ assert status.vector_count == 5++ # Verify aggregate method was called+ mock_collection.aggregate.over_all.assert_called_once()+++def test_clear_collection(mock_weaviate_client):+ r"""Test clearing collection functionality.++ Verifies that the collection can be cleared by deleting and+ recreating it properly.+ """+ mock_client, mock_collection = mock_weaviate_client++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._create_collection'+ ) as mock_create,+ ):+ storage = WeaviateStorage(+ vector_dim=4, collection_name="test_collection"+ )++ # Execute clear operation+ storage.clear()++ # Verify collection was deleted and recreated+ mock_client.collections.delete.assert_called_once_with(+ "test_collection"+ )+ mock_create.assert_called_once()+++def test_empty_operations(mock_weaviate_client):+ r"""Test handling of empty operations.++ Verifies that empty add and delete operations are handled gracefully+ without calling the underlying APIs.+ """+ mock_client, mock_collection = mock_weaviate_client++ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client',+ return_value=mock_client,+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ ):+ storage = WeaviateStorage(+ vector_dim=4, collection_name="test_collection"+ )++ # Test empty add+ storage.add([])+ mock_collection.batch.dynamic.assert_not_called()++ # Test empty delete+ storage.delete([])+ mock_collection.data.delete_many.assert_not_called()+++def test_similarity_calculations():+ r"""Test similarity calculation functionality.++ Verifies that different distance metrics (cosine, L2, hamming) are+ correctly converted to similarity scores.+ """+ with (+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._get_connection_client'+ ),+ patch(+ 'camel.storages.vectordb_storages.weaviate.WeaviateStorage._check_and_create_collection'+ ),+ ):+ # Test cosine distance+ storage_cosine = WeaviateStorage(+ vector_dim=4, distance_metric="cosine"+ )+ similarity = storage_cosine._calculate_similarity_from_distance(0.2)+ assert similarity == 0.9 # 1 - (0.2 / 2)++ # Test L2 distance+ storage_l2 = WeaviateStorage(+ vector_dim=4, distance_metric="l2-squared"+ )+ similarity = storage_l2._calculate_similarity_from_distance(1.0)+ assert similarity == 0.5 # 1 / (1 + 1)++ # Test Hamming distance+ storage_hamming = WeaviateStorage(+ vector_dim=4, distance_metric="hamming"+ )+ similarity = storage_hamming._calculate_similarity_from_distance(2.0)+ assert similarity == 0.5 # 1 - (2 / 4)diff --git a/uv.lock b/uv.lockindex c12e8b419..8ace8eabe 100644--- a/uv.lock+++ b/uv.lock@@ -404,6 +404,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815 },]+[[package]]+name = "authlib"+version = "1.3.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "cryptography" },+]+sdist = { url = "https://files.pythonhosted.org/packages/09/47/df70ecd34fbf86d69833fe4e25bb9ecbaab995c8e49df726dd416f6bb822/authlib-1.3.1.tar.gz", hash = "sha256:7ae843f03c06c5c0debd63c9db91f9fda64fa62a42a77419fa15fbb7e7a58917", size = 146074 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/87/1f/bc95e43ffb57c05b8efcc376dd55a0240bf58f47ddf5a0f92452b6457b75/Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377", size = 223827 },+]+[[package]]name = "av"version = "14.4.0"@@ -692,6 +704,7 @@ all = [{ name = "types-setuptools" },{ name = "types-tqdm" },{ name = "unstructured" },+ { name = "weaviate-client" },{ name = "wikipedia" },{ name = "wolframalpha" },{ name = "xls2xlsx" },@@ -870,6 +883,7 @@ rag = [{ name = "rank-bm25" },{ name = "sentence-transformers" },{ name = "unstructured" },+ { name = "weaviate-client" },]research-tools = [{ name = "arxiv" },@@ -889,6 +903,7 @@ storage = [{ name = "pytidb-experimental" },{ name = "qdrant-client" },{ name = "redis" },+ { name = "weaviate-client" },]test = [{ name = "mock" },@@ -1232,6 +1247,9 @@ requires-dist = [{ name = "unstructured", marker = "extra == 'owl'", specifier = "==0.16.20" },{ name = "unstructured", marker = "extra == 'rag'", specifier = "==0.16.20" },{ name = "uv", marker = "extra == 'dev'", specifier = "==0.6.5" },+ { name = "weaviate-client", marker = "extra == 'all'", specifier = ">=4.15.0" },+ { name = "weaviate-client", marker = "extra == 'rag'", specifier = ">=4.15.0" },+ { name = "weaviate-client", marker = "extra == 'storage'", specifier = ">=4.15.0" },{ name = "wikipedia", marker = "extra == 'all'", specifier = ">=1,<2" },{ name = "wikipedia", marker = "extra == 'owl'", specifier = ">=1,<2" },{ name = "wikipedia", marker = "extra == 'web-tools'", specifier = ">=1,<2" },@@ -1826,6 +1844,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998 },]+[[package]]+name = "deprecation"+version = "2.1.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "packaging" },+]+sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178 },+]+[[package]]name = "diffusers"version = "0.25.1"@@ -2748,6 +2778,59 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574 },]+[[package]]+name = "grpcio-health-checking"+version = "1.67.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "grpcio" },+ { name = "protobuf" },+]+sdist = { url = "https://files.pythonhosted.org/packages/64/dd/e3b339fa44dc75b501a1a22cb88f1af5b1f8c964488f19c4de4cfbbf05ba/grpcio_health_checking-1.67.1.tar.gz", hash = "sha256:ca90fa76a6afbb4fda71d734cb9767819bba14928b91e308cffbb0c311eb941e", size = 16775 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/5c/8d/7a9878dca6616b48093d71c52d0bc79cb2dd1a2698ff6f5ce7406306de12/grpcio_health_checking-1.67.1-py3-none-any.whl", hash = "sha256:93753da5062152660aef2286c9b261e07dd87124a65e4dc9fbd47d1ce966b39d", size = 18924 },+]++[[package]]+name = "grpcio-tools"+version = "1.67.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "grpcio" },+ { name = "protobuf" },+ { name = "setuptools" },+]+sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/6facde12a5a8da4398a3a8947f8ba6ef33b408dfc9767c8cefc0074ddd68/grpcio_tools-1.67.1.tar.gz", hash = "sha256:d9657f5ddc62b52f58904e6054b7d8a8909ed08a1e28b734be3a707087bcf004", size = 5159073 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e5/46/668e681e2e4ca7dc80cb5ad22bc794958c8b604b5b3143f16b94be3c0118/grpcio_tools-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:c701aaa51fde1f2644bd94941aa94c337adb86f25cd03cf05e37387aaea25800", size = 2308117 },+ { url = "https://files.pythonhosted.org/packages/d6/56/1c65fb7c836cd40470f1f1a88185973466241fdb42b42b7a83367c268622/grpcio_tools-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:6a722bba714392de2386569c40942566b83725fa5c5450b8910e3832a5379469", size = 5500152 },+ { url = "https://files.pythonhosted.org/packages/01/ab/caf9c330241d843a83043b023e2996e959cdc2c3ab404b1a9938eb734143/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0c7415235cb154e40b5ae90e2a172a0eb8c774b6876f53947cf0af05c983d549", size = 2282055 },+ { url = "https://files.pythonhosted.org/packages/75/e6/0cd849d140b58fedb7d3b15d907fe2eefd4dadff09b570dd687d841c5d00/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4c459098c4934f9470280baf9ff8b38c365e147f33c8abc26039a948a664a5", size = 2617360 },+ { url = "https://files.pythonhosted.org/packages/b9/51/bd73cd6515c2e81ba0a29b3cf6f2f62ad94737326f70b32511d1972a383e/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e89bf53a268f55c16989dab1cf0b32a5bff910762f138136ffad4146129b7a10", size = 2416028 },+ { url = "https://files.pythonhosted.org/packages/47/e5/6a16e23036f625b6d60b579996bb9bb7165485903f934d9d9d73b3f03ef5/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f09cb3e6bcb140f57b878580cf3b848976f67faaf53d850a7da9bfac12437068", size = 3224906 },+ { url = "https://files.pythonhosted.org/packages/14/cb/230c17d4372fa46fc799a822f25fa00c8eb3f85cc86e192b9606a17f732f/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:616dd0c6686212ca90ff899bb37eb774798677e43dc6f78c6954470782d37399", size = 2870384 },+ { url = "https://files.pythonhosted.org/packages/66/fd/6d9dd3bf5982ab7d7e773f055360185e96a96cf95f2cbc7f53ded5912ef5/grpcio_tools-1.67.1-cp310-cp310-win32.whl", hash = "sha256:58a66dbb3f0fef0396737ac09d6571a7f8d96a544ce3ed04c161f3d4fa8d51cc", size = 941138 },+ { url = "https://files.pythonhosted.org/packages/6a/97/2fd5ebd996c12b2cb1e1202ee4a03cac0a65ba17d29dd34253bfe2079839/grpcio_tools-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:89ee7c505bdf152e67c2cced6055aed4c2d4170f53a2b46a7e543d3b90e7b977", size = 1091151 },+ { url = "https://files.pythonhosted.org/packages/b5/9a/ec06547673c5001c2604637069ff8f287df1aef3f0f8809b09a1c936b049/grpcio_tools-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:6d80ddd87a2fb7131d242f7d720222ef4f0f86f53ec87b0a6198c343d8e4a86e", size = 2307990 },+ { url = "https://files.pythonhosted.org/packages/ca/84/4b7c3c27a2972c00b3b6ccaadd349e0f86b7039565d3a4932e219a4d76e0/grpcio_tools-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b655425b82df51f3bd9fd3ba1a6282d5c9ce1937709f059cb3d419b224532d89", size = 5526552 },+ { url = "https://files.pythonhosted.org/packages/a7/2d/a620e4c53a3b808ebecaa5033c2176925ee1c6cbb45c29af8bec9a249822/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:250241e6f9d20d0910a46887dfcbf2ec9108efd3b48f3fb95bb42d50d09d03f8", size = 2282137 },+ { url = "https://files.pythonhosted.org/packages/ec/29/e188b2e438781b37532abb8f10caf5b09c611a0bf9a09940b4cf303afd5b/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6008f5a5add0b6f03082edb597acf20d5a9e4e7c55ea1edac8296c19e6a0ec8d", size = 2617333 },+ { url = "https://files.pythonhosted.org/packages/86/aa/2bbccd3c34b1fa48b892fbad91525c33a8aa85cbedd50e8b0d17dc260dc3/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5eff9818c3831fa23735db1fa39aeff65e790044d0a312260a0c41ae29cc2d9e", size = 2415806 },+ { url = "https://files.pythonhosted.org/packages/db/34/99853a8ced1119937d02511476018dc1d6b295a4803d4ead5dbf9c55e9bc/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:262ab7c40113f8c3c246e28e369661ddf616a351cb34169b8ba470c9a9c3b56f", size = 3224765 },+ { url = "https://files.pythonhosted.org/packages/66/39/8537a8ace8f6242f2058677e56a429587ec731c332985af34f35d496ca58/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1eebd8c746adf5786fa4c3056258c21cc470e1eca51d3ed23a7fb6a697fe4e81", size = 2870446 },+ { url = "https://files.pythonhosted.org/packages/28/2a/5c04375adccff58647d48675e055895c31811a0ad896e4ba310833e2154d/grpcio_tools-1.67.1-cp311-cp311-win32.whl", hash = "sha256:3eff92fb8ca1dd55e3af0ef02236c648921fb7d0e8ca206b889585804b3659ae", size = 940890 },+ { url = "https://files.pythonhosted.org/packages/e6/ee/7861339c2cec8d55a5e859cf3682bda34eab5a040f95d0c80f775d6a3279/grpcio_tools-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:1ed18281ee17e5e0f9f6ce0c6eb3825ca9b5a0866fc1db2e17fab8aca28b8d9f", size = 1091094 },+ { url = "https://files.pythonhosted.org/packages/d9/cf/7b1908ca72e484bac555431036292c48d2d6504a45e2789848cb5ff313a8/grpcio_tools-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:bd5caef3a484e226d05a3f72b2d69af500dca972cf434bf6b08b150880166f0b", size = 2307645 },+ { url = "https://files.pythonhosted.org/packages/bb/15/0d1efb38af8af7e56b2342322634a3caf5f1337a6c3857a6d14aa590dfdf/grpcio_tools-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:48a2d63d1010e5b218e8e758ecb2a8d63c0c6016434e9f973df1c3558917020a", size = 5525468 },+ { url = "https://files.pythonhosted.org/packages/52/42/a810709099f09ade7f32990c0712c555b3d7eab6a05fb62618c17f8fe9da/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:baa64a6aa009bffe86309e236c81b02cd4a88c1ebd66f2d92e84e9b97a9ae857", size = 2281768 },+ { url = "https://files.pythonhosted.org/packages/4c/2a/64ee6cfdf1c32ef8bdd67bf04ae2f745f517f4a546281453ca1f68fa79ca/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ab318c40b5e3c097a159035fc3e4ecfbe9b3d2c9de189e55468b2c27639a6ab", size = 2617359 },+ { url = "https://files.pythonhosted.org/packages/79/7f/1ed8cd1529253fef9cf0ef3cd8382641125a5ca2eaa08eaffbb549f84e0b/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50eba3e31f9ac1149463ad9182a37349850904f142cffbd957cd7f54ec320b8e", size = 2415323 },+ { url = "https://files.pythonhosted.org/packages/8e/08/59f0073c58703c176c15fb1a838763b77c1c06994adba16654b92a666e1b/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:de6fbc071ecc4fe6e354a7939202191c1f1abffe37fbce9b08e7e9a5b93eba3d", size = 3225051 },+ { url = "https://files.pythonhosted.org/packages/b7/0d/a5d703214fe49d261b4b8f0a64140a4dc1f88560724a38ad937120b899ad/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db9e87f6ea4b0ce99b2651203480585fd9e8dd0dd122a19e46836e93e3a1b749", size = 2870421 },+ { url = "https://files.pythonhosted.org/packages/ac/af/41d79cb87eae99c0348e8f1fb3dbed9e40a6f63548b216e99f4d1165fa5c/grpcio_tools-1.67.1-cp312-cp312-win32.whl", hash = "sha256:6a595a872fb720dde924c4e8200f41d5418dd6baab8cc1a3c1e540f8f4596351", size = 940542 },+ { url = "https://files.pythonhosted.org/packages/66/e5/096e12f5319835aa2bcb746d49ae62220bb48313ca649e89bdbef605c11d/grpcio_tools-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:92eebb9b31031604ae97ea7657ae2e43149b0394af7117ad7e15894b6cc136dc", size = 1090425 },+]+[[package]]name = "h11"version = "0.16.0"@@ -7863,6 +7946,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/b1/4b/4cef6ce21a2aaca9d852a6e84ef4f135d99fcd74fa75105e2fc0c8308acd/uvicorn-0.34.2-py3-none-any.whl", hash = "sha256:deb49af569084536d269fe0a6d67e3754f104cf03aba7c11c40f01aadf33c403", size = 62483 },]+[[package]]+name = "validators"+version = "0.34.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/64/07/91582d69320f6f6daaf2d8072608a4ad8884683d4840e7e4f3a9dbdcc639/validators-0.34.0.tar.gz", hash = "sha256:647fe407b45af9a74d245b943b18e6a816acf4926974278f6dd617778e1e781f", size = 70955 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/6e/78/36828a4d857b25896f9774c875714ba4e9b3bc8a92d2debe3f4df3a83d4f/validators-0.34.0-py3-none-any.whl", hash = "sha256:c804b476e3e6d3786fa07a30073a4ef694e617805eb1946ceee3fe5a9b8b1321", size = 43536 },+]+[[package]]name = "virtualenv"version = "20.31.2"@@ -7886,6 +7978,25 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166 },]+[[package]]+name = "weaviate-client"+version = "4.15.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "authlib" },+ { name = "deprecation" },+ { name = "grpcio" },+ { name = "grpcio-health-checking" },+ { name = "grpcio-tools" },+ { name = "httpx" },+ { name = "pydantic" },+ { name = "validators" },+]+sdist = { url = "https://files.pythonhosted.org/packages/90/98/53544475466ec35260c8cdc201fd7a9e9731d3ed596121ffc912fab782cf/weaviate_client-4.15.0.tar.gz", hash = "sha256:0e1c06e0bb08c8ab6987d91cf024b625f517b349965e6dee5439f675ff90cc64", size = 663838 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/b8/eb/cebfb6b401e005e2e0b5ad89e43b674cb962c7e9ec386bb6f5874c8b21a2/weaviate_client-4.15.0-py3-none-any.whl", hash = "sha256:36bb83dacfc102cab0267aa211eb21eaca31cc80921b977451d814532c9a50e6", size = 433671 },+]+[[package]]name = "webcolors"version = "24.11.1"