🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
`feat: Add new vector storage db FAISS` (#2034)
Ziqi Chen5dd55f705006
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.pyindex 5ab6ae7b1..5a9c5dec4 100644--- a/camel/storages/__init__.py+++ b/camel/storages/__init__.py@@ -30,6 +30,7 @@ from .vectordb_storages.milvus import MilvusStoragefrom .vectordb_storages.oceanbase import OceanBaseStoragefrom .vectordb_storages.qdrant import QdrantStoragefrom .vectordb_storages.tidb import TiDBStorage+from .vectordb_storages.faiss import FaissStorage__all__ = ['BaseKeyValueStorage',@@ -43,6 +44,7 @@ __all__ = ['QdrantStorage','MilvusStorage',"TiDBStorage",+ "FaissStorage",'BaseGraphStorage','Neo4jGraph','NebulaGraph',diff --git a/camel/storages/vectordb_storages/__init__.py b/camel/storages/vectordb_storages/__init__.pyindex bf31e9ee2..49c0a200c 100644--- a/camel/storages/vectordb_storages/__init__.py+++ b/camel/storages/vectordb_storages/__init__.py@@ -23,6 +23,7 @@ from .milvus import MilvusStoragefrom .oceanbase import OceanBaseStoragefrom .qdrant import QdrantStoragefrom .tidb import TiDBStorage+from .faiss import FaissStorage__all__ = ['BaseVectorStorage',@@ -31,6 +32,7 @@ __all__ = ['QdrantStorage','MilvusStorage',"TiDBStorage",+ 'FaissStorage','OceanBaseStorage','VectorRecord','VectorDBStatus',diff --git a/camel/storages/vectordb_storages/faiss.py b/camel/storages/vectordb_storages/faiss.pynew file mode 100644index 000000000..be7e7d3cc--- /dev/null+++ b/camel/storages/vectordb_storages/faiss.py@@ -0,0 +1,641 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import os+import logging+import pickle+from datetime import datetime+from typing import Any, Dict, List, Optional, Literal++import numpy as np++from camel.storages.vector_db.base import (+ BaseVectorStorage,+ VectorDBQuery,+ VectorDBQueryResult,+ VectorDBStatus,+ VectorRecord,+)+from camel.types import VectorDistance+from camel.utils import dependencies_required++logger = logging.get_logger(__name__)+++class FaissStorage(BaseVectorStorage):+ """An implementation of the `BaseVectorStorage` using FAISS,+ Facebook AI's Similarity Search library for efficient vector search.++ The detailed information about FAISS is available at:+ `FAISS <https://github.com/facebookresearch/faiss>`_++ Args:+ vector_dim (int): The dimension of storing vectors.+ index_type (str, optional): Type of FAISS index to create.+ Options include 'Flat', 'IVF', 'HNSW', etc. (default: :obj:`'Flat'`)+ collection_name (Optional[str], optional): Name for the collection.+ If not provided, set it to the current time with iso format.+ (default: :obj:`None`)+ storage_path (Optional[str], optional): Path to directory where+ the index will be stored. If None, index will only exist in memory.+ (default: :obj:`None`)+ distance (VectorDistance, optional): The distance metric for vector+ comparison (default: :obj:`VectorDistance.COSINE`)+ nlist (int, optional): Number of cluster centroids for IVF indexes.+ Only used if index_type includes 'IVF'. (default: :obj:`100`)+ m (int, optional): HNSW parameter. Number of connections per node.+ Only used if index_type includes 'HNSW'. (default: :obj:`16`)+ **kwargs (Any): Additional keyword arguments.++ Notes:+ - FAISS offers various index types optimized for different use cases:+ - 'Flat': Exact search, but slowest for large datasets+ - 'IVF': Inverted file index, good balance of speed and recall+ - 'HNSW': Hierarchical Navigable Small World, fast with high recall+ - 'PQ': Product Quantization for memory-efficient storage+ - The choice of index should be based on your specific requirements+ for search speed, memory usage, and accuracy.+ """++ @dependencies_required('faiss')+ def __init__(+ self,+ vector_dim: int,+ index_type: str = 'Flat',+ collection_name: Optional[str] = None,+ storage_path: Optional[str] = None,+ distance: VectorDistance = VectorDistance.COSINE,+ nlist: int = 100,+ m: int = 16,+ **kwargs: Any,+ ) -> None:+ """Initialize the FAISS vector storage.++ Args:+ vector_dim: Dimension of vectors to be stored+ index_type: FAISS index type ('Flat', 'IVF', 'HNSW', etc.)+ collection_name: Name of the collection (defaults to timestamp)+ storage_path: Directory to save the index (None for in-memory only)+ distance: Vector distance metric+ nlist: Number of clusters for IVF indexes+ m: HNSW parameter for connections per node+ **kwargs: Additional parameters+ """+ import faiss++ self.vector_dim = vector_dim+ self.index_type = index_type+ self.collection_name = collection_name or self._generate_collection_name()+ self.storage_path = storage_path+ self.distance = distance+ self.nlist = nlist+ self.m = m+ self._faiss_client = faiss # Store the faiss module as the client++ # Create directory for storage if it doesn't exist+ if self.storage_path is not None:+ os.makedirs(self.storage_path, exist_ok=True)++ # Initialize the FAISS index+ self._index = self._create_index()++ # Storage for IDs and payloads (FAISS only stores vectors)+ self._id_to_index: Dict[str, int] = {}+ self._index_to_id: Dict[int, str] = {}+ self._payloads: Dict[str, Dict[str, Any]] = {}+ self._vectors: Dict[str, np.ndarray] = {}++ # Load existing index if it exists+ if self.storage_path:+ self._load_from_disk()++ def _generate_collection_name(self) -> str:+ """Generates a collection name if user doesn't provide"""+ return f"faiss_index_{datetime.now().isoformat()}"++ def _get_index_path(self) -> str:+ """Returns the path to the index file"""+ if self.storage_path is None:+ raise ValueError("Storage path is not set.")+ return os.path.join(self.storage_path, f"{self.collection_name}.index")++ def _get_metadata_path(self) -> str:+ """Returns the path to the metadata file"""+ if self.storage_path is None:+ raise ValueError("Storage path is not set.")+ return os.path.join(self.storage_path, f"{self.collection_name}.metadata")++ def _create_index(self):+ """Creates a new FAISS index based on specified parameters.++ Returns:+ A FAISS index object configured according to the parameters.+ """+ import faiss++ # Determine the metric to use based on distance type+ if self.distance == VectorDistance.COSINE:+ # For cosine similarity, we need to normalize vectors+ metric = faiss.METRIC_INNER_PRODUCT+ elif self.distance == VectorDistance.EUCLIDEAN:+ metric = faiss.METRIC_L2+ elif self.distance == VectorDistance.DOT:+ metric = faiss.METRIC_INNER_PRODUCT+ else:+ raise ValueError(f"Unsupported distance metric: {self.distance}")++ # Create the appropriate index based on index_type+ if self.index_type == 'Flat':+ if metric == faiss.METRIC_INNER_PRODUCT:+ index = faiss.IndexFlatIP(self.vector_dim)+ else:+ index = faiss.IndexFlatL2(self.vector_dim)+ elif self.index_type.startswith('IVF'):+ # IVF requires a quantizer (often a flat index)+ quantizer = faiss.IndexFlatL2(self.vector_dim)+ if 'Flat' in self.index_type:+ index = faiss.IndexIVFFlat(quantizer, self.vector_dim, self.nlist, metric)+ elif 'PQ' in self.index_type:+ # M value for PQ, typically a divisor of vector_dim+ m = self.vector_dim // 4 # default setting+ nbits = 8 # typically 8 bits per sub-vector+ index = faiss.IndexIVFPQ(quantizer, self.vector_dim, self.nlist, m, nbits)+ else:+ raise ValueError(f"Unsupported IVF index type: {self.index_type}")++ # IVF indexes need to be trained before use+ # This is a placeholder since actual training requires data+ index.nprobe = 10 # Number of clusters to search (trade-off between speed and accuracy)+ elif self.index_type == 'HNSW':+ index = faiss.IndexHNSWFlat(self.vector_dim, self.m, metric)+ else:+ raise ValueError(f"Unsupported index type: {self.index_type}")++ return index++ def _save_to_disk(self) -> None:+ """Saves the index and metadata to disk if storage_path is provided."""+ if self.storage_path is None:+ return++ import faiss++ # Save the FAISS index+ faiss.write_index(self._index, self._get_index_path())++ # Save the metadata (IDs, payloads, vectors)+ metadata = {+ "id_to_index": self._id_to_index,+ "index_to_id": self._index_to_id,+ "payloads": self._payloads,+ "vectors": self._vectors,+ "vector_dim": self.vector_dim,+ "distance": self.distance,+ }++ with open(self._get_metadata_path(), 'wb') as f:+ pickle.dump(metadata, f)++ logger.info(f"Saved FAISS index and metadata to {self.storage_path}")++ def _load_from_disk(self) -> None:+ """Loads the index and metadata from disk if they exist."""+ if self.storage_path is None:+ return++ import faiss++ index_path = self._get_index_path()+ metadata_path = self._get_metadata_path()++ if os.path.exists(index_path) and os.path.exists(metadata_path):+ try:+ # Load the FAISS index+ self._index = faiss.read_index(index_path)++ # Load the metadata+ with open(metadata_path, 'rb') as f:+ metadata = pickle.load(f)++ self._id_to_index = metadata["id_to_index"]+ self._index_to_id = metadata["index_to_id"]+ self._payloads = metadata["payloads"]+ self._vectors = metadata["vectors"]++ # Check that the loaded index is compatible+ if metadata["vector_dim"] != self.vector_dim:+ logger.warning(+ f"Loaded index has different vector dimension "+ f"({metadata['vector_dim']}) than specified ({self.vector_dim})."+ )++ logger.info(f"Loaded FAISS index and metadata from {self.storage_path}")+ except Exception as e:+ logger.error(f"Failed to load index from disk: {e}")+ # Initialize a new index+ self._index = self._create_index()+ else:+ logger.info("No existing index found. Creating new one.")++ def add(+ self,+ records: List[VectorRecord],+ **kwargs,+ ) -> None:+ """Adds a list of vectors to the index.++ Args:+ records (List[VectorRecord]): List of vector records to be added.+ **kwargs (Any): Additional keyword arguments.++ Raises:+ RuntimeError: If there was an error in the addition process.+ """+ if not records:+ return++ # Check if the index needs training (for IVF indexes)+ if self.index_type.startswith('IVF') and not self._index.is_trained:+ # For IVF indexes, we need to train with vectors before adding+ vectors = np.array([record.vector for record in records], dtype=np.float32)+ try:+ self._index.train(vectors)+ except Exception as e:+ raise RuntimeError(f"Failed to train FAISS index: {e}")++ # Add each record to the index+ for record in records:+ # Normalize vector if using cosine similarity+ vector = np.array(record.vector, dtype=np.float32).reshape(1, -1)+ if self.distance == VectorDistance.COSINE:+ vector = self._normalize_vector(vector)++ # Get the next index+ idx = len(self._id_to_index)++ # Add to FAISS index and update mappings atomically+ try:+ self._index.add(vector)+ # Store mapping from ID to index+ self._id_to_index[record.id] = idx+ self._index_to_id[idx] = record.id+ # Store payload+ if record.payload is not None:+ self._payloads[record.id] = record.payload+ else:+ self._payloads[record.id] = {}+ # Store the original vector for later retrieval+ self._vectors[record.id] = vector.flatten()+ except Exception as e:+ # If adding to the index fails, roll back any partial changes+ if record.id in self._id_to_index:+ del self._id_to_index[record.id]+ if idx in self._index_to_id:+ del self._index_to_id[idx]+ if record.id in self._payloads:+ del self._payloads[record.id]+ if record.id in self._vectors:+ del self._vectors[record.id]+ raise RuntimeError(f"Failed to add vector to FAISS index: {e}")++ # Save to disk if storage path is provided+ self._save_to_disk()++ def update_payload(+ self, ids: List[str], payload: Dict[str, Any], **kwargs: Any+ ) -> None:+ """Updates the payload of the vectors identified by their IDs.++ Args:+ ids (List[str]): List of unique identifiers for the vectors to be+ updated.+ payload (Dict[str, Any]): Payload to be updated for all specified IDs.+ **kwargs (Any): Additional keyword arguments.++ Raises:+ KeyError: If any of the provided IDs does not exist in the index.+ """+ for id in ids:+ if id not in self._payloads:+ raise KeyError(f"Vector with ID {id} not found in the index.")++ # Update payload (merge with existing payload)+ if id in self._payloads:+ self._payloads[id].update(payload)+ else:+ self._payloads[id] = payload.copy()++ # Save to disk if storage path is provided+ self._save_to_disk()++ def delete_collection(self) -> None:+ """Deletes the entire collection (index and metadata)."""+ # Reset the index+ self._index = self._create_index()++ # Clear metadata+ self._id_to_index = {}+ self._index_to_id = {}+ self._payloads = {}+ self._vectors = {}++ # Remove files from disk if storage path is provided+ if self.storage_path:+ index_path = self._get_index_path()+ metadata_path = self._get_metadata_path()++ if os.path.exists(index_path):+ os.remove(index_path)++ if os.path.exists(metadata_path):+ os.remove(metadata_path)++ logger.info(f"Deleted FAISS index and metadata from {self.storage_path}")++ def delete(+ self,+ ids: Optional[List[str]] = None,+ payload_filter: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> None:+ """Deletes vectors from the index based on either IDs or payload+ filters.++ Args:+ ids (Optional[List[str]], optional): List of unique identifiers+ for the vectors to be deleted.+ payload_filter (Optional[Dict[str, Any]], optional): A filter for+ the payload to delete points matching specific conditions.+ **kwargs (Any): Additional keyword arguments.++ Raises:+ ValueError: If neither `ids` nor `payload_filter` is provided.+ RuntimeError: If the FAISS index does not support removal.++ Notes:+ - FAISS does not support efficient single vector removal for most+ index types. This implementation recreates the index without the+ deleted vectors, which can be inefficient for large datasets.+ - If both `ids` and `payload_filter` are provided, both filters+ will be applied (vectors matching either will be deleted).+ """+ import faiss++ if not ids and not payload_filter:+ raise ValueError(+ "You must provide either `ids` or `payload_filter` to delete "+ "vectors."+ )++ # Get IDs to delete from payload filter+ if payload_filter:+ filtered_ids = [+ id for id, payload in self._payloads.items()+ if all(payload.get(key) == value for key, value in payload_filter.items())+ ]+ else:+ filtered_ids = []++ # Combine with explicit IDs+ ids_to_delete = set(ids or []) | set(filtered_ids)+ if not ids_to_delete:+ return++ # Check if the index supports removal+ if hasattr(self._index, 'remove_ids'):+ # Convert IDs to indices+ indices_to_remove = np.array(+ [self._id_to_index[id] for id in ids_to_delete if id in self._id_to_index],+ dtype=np.int64+ )++ if len(indices_to_remove) > 0:+ # Create a selector where 1 means "remove"+ selector = np.zeros(self._index.ntotal, dtype=bool)+ selector[indices_to_remove] = True++ try:+ # Remove from FAISS index+ id_selector = faiss.IDSelectorArray(len(indices_to_remove), indices_to_remove)+ self._index.remove_ids(id_selector)++ # Update mappings and storage+ for id in ids_to_delete:+ if id in self._id_to_index:+ idx = self._id_to_index[id]+ del self._index_to_id[idx]+ del self._id_to_index[id]+ if id in self._payloads:+ del self._payloads[id]+ if id in self._vectors:+ del self._vectors[id]++ # Save to disk if storage path is provided+ self._save_to_disk()+ except Exception as e:+ raise RuntimeError(f"Failed to remove vectors from FAISS index: {e}")+ else:+ # Index doesn't support removal, need to rebuild+ logger.warning(+ "This FAISS index type doesn't support direct removal. "+ "Rebuilding entire index (may be slow for large datasets)."+ )++ # Get all vectors and IDs that should be kept+ keep_ids = [id for id in self._id_to_index.keys() if id not in ids_to_delete]++ # Create a new index+ new_index = self._create_index()++ # If it's an IVF index and there are vectors to train on, train it+ if self.index_type.startswith('IVF') and keep_ids:+ train_vectors = np.vstack([self._vectors[id].reshape(1, -1) for id in keep_ids])+ new_index.train(train_vectors)++ # Add the vectors to keep to the new index+ new_id_to_index = {}+ new_index_to_id = {}++ for new_idx, id in enumerate(keep_ids):+ vector = self._vectors[id].reshape(1, -1)+ new_index.add(vector)+ new_id_to_index[id] = new_idx+ new_index_to_id[new_idx] = id++ # Replace the old index and mappings+ self._index = new_index+ self._id_to_index = new_id_to_index+ self._index_to_id = new_index_to_id++ # Remove deleted IDs from payloads and vectors+ for id in ids_to_delete:+ if id in self._payloads:+ del self._payloads[id]+ if id in self._vectors:+ del self._vectors[id]++ # Save to disk if storage path is provided+ self._save_to_disk()++ def status(self) -> VectorDBStatus:+ """Returns the status of the vector database.++ Returns:+ VectorDBStatus: Current status of the vector database.+ """+ return VectorDBStatus(+ vector_dim=self.vector_dim,+ vector_count=self._index.ntotal,+ )++ def query(+ self,+ query: VectorDBQuery,+ filter_conditions: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> List[VectorDBQueryResult]:+ """Searches for similar vectors in the storage based on the provided+ query.++ Args:+ query (VectorDBQuery): The query object containing the search+ vector and the number of top similar vectors to retrieve.+ filter_conditions (Optional[Dict[str, Any]], optional): A+ dictionary specifying conditions to filter the query results.+ **kwargs (Any): Additional keyword arguments.++ Returns:+ List[VectorDBQueryResult]: A list of query results ordered by+ similarity.+ """+ if self._index.ntotal == 0:+ return []++ # Prepare the query vector+ query_vector = np.array(query.query_vector, dtype=np.float32)+ if query_vector.ndim == 1:+ query_vector = query_vector.reshape(1, -1)++ # Normalize if using cosine similarity+ if self.distance == VectorDistance.COSINE:+ query_vector = self._normalize_vector(query_vector)++ # For IVF indexes, set the number of clusters to probe (higher = more accurate but slower)+ if hasattr(self._index, 'nprobe'):+ nprobe = kwargs.get('nprobe', 10) # Default to 10+ self._index.nprobe = nprobe++ # Determine how many results to fetch initially when using filters+ k = query.top_k+ if filter_conditions:+ # Fetch more results if filtering is applied to ensure we have enough after filtering+ # This is a simple heuristic - multiply by 2 or by a user-provided factor+ fetch_factor = kwargs.get('fetch_factor', 2)+ fetch_k = min(int(k * fetch_factor), self._index.ntotal)+ else:+ fetch_k = min(k, self._index.ntotal)++ # Perform the search+ distances, indices = self._index.search(query_vector, fetch_k)++ # Convert results to VectorDBQueryResult objects+ results = []+ for i, idx in enumerate(indices[0]):+ if idx == -1: # FAISS returns -1 for empty slots+ continue++ vector_id = self._index_to_id.get(idx)+ if not vector_id:+ continue++ # If there are filter conditions, check if this result passes+ if filter_conditions and not self._matches_filter(vector_id, filter_conditions):+ continue++ # Adjust similarity score based on distance metric+ if self.distance == VectorDistance.EUCLIDEAN:+ # For Euclidean distance, smaller is better, so we invert+ similarity = 1.0 / (1.0 + distances[0][i])+ else:+ # For inner product/cosine, higher is better+ similarity = float(distances[0][i])++ vector = self._vectors.get(vector_id)+ if vector is not None:+ vector = vector.tolist()+ else:+ vector = []++ results.append(+ VectorDBQueryResult(+ similarity=similarity,+ id=vector_id,+ payload=self._payloads.get(vector_id, {}),+ vector=vector,+ )+ )++ # Stop once we have enough results+ if len(results) >= k:+ break++ return results++ def clear(self) -> None:+ """Remove all vectors from the storage."""+ self.delete_collection()++ def load(self) -> None:+ """Load the index from disk if storage_path is provided."""+ if self.storage_path:+ self._load_from_disk()+ else:+ logger.warning("No storage path provided. Cannot load index.")++ @property+ def client(self) -> Any:+ """Provides access to the underlying FAISS client."""+ return self._faiss_client++ def _matches_filter(self, vector_id: str, filter_conditions: Dict[str, Any]) -> bool:+ """Checks if a vector's payload matches the filter conditions.++ Args:+ vector_id (str): ID of the vector to check.+ filter_conditions (Dict[str, Any]): Conditions to match against.++ Returns:+ bool: True if the payload matches all conditions, False otherwise.+ """+ payload = self._payloads.get(vector_id, {})+ for key, value in filter_conditions.items():+ if key not in payload or payload[key] != value:+ return False+ return True++ def _normalize_vector(self, vector: np.ndarray) -> np.ndarray:+ """Normalizes a vector to unit length for cosine similarity.++ Args:+ vector (np.ndarray): Vector to normalize, either 1D or 2D array.++ Returns:+ np.ndarray: Normalized vector with the same shape as input.+ """+ if vector.ndim == 1:+ vector = vector.reshape(1, -1)+ norm = np.linalg.norm(vector, axis=1, keepdims=True)+ # Avoid division by zero+ norm = np.maximum(norm, 1e-10)+ return vector / normdiff --git a/docs/camel.storages.vectordb_storages.rst b/docs/camel.storages.vectordb_storages.rstindex b1c2af596..eeff0b9c4 100644--- a/docs/camel.storages.vectordb_storages.rst+++ b/docs/camel.storages.vectordb_storages.rst@@ -36,6 +36,14 @@ camel.storages.vectordb\_storages.tidb module:undoc-members::show-inheritance:+camel.storages.vectordb\_storages.faiss module+-----------------------------------------------++.. automodule:: camel.storages.vectordb_storages.faiss+ :members:+ :undoc-members:+ :show-inheritance:+Module contents---------------diff --git a/pyproject.toml b/pyproject.tomlindex d9cac3001..4c368ed95 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -72,6 +72,8 @@ rag = ["qdrant-client>=1.9.0,<2","pymilvus>=2.4.0,<3","pytidb-experimental==0.0.1.dev4",+ "faiss-cpu>=1.7.2,<2",+ "faiss-gpu>=1.7.2,<2","neo4j>=5.18.0,<6","nebula3-python==3.8.2","rank-bm25>=0.2.2,<0.3",@@ -194,6 +196,8 @@ storage = ["qdrant-client>=1.9.0,<2","pymilvus>=2.4.0,<3","pytidb-experimental==0.0.1.dev4",+ "faiss-cpu>=1.7.2,<2",+ "faiss-gpu>=1.7.2,<2","neo4j>=5.18.0,<6","nebula3-python==3.8.2","redis>=5.0.6,<6",@@ -316,6 +320,8 @@ all = ["qdrant-client>=1.9.0,<2","pymilvus>=2.4.0,<3","pytidb-experimental==0.0.1.dev4",+ "faiss-cpu>=1.7.2,<2",+ "faiss-gpu>=1.7.2,<2","cohere>=5.11.0,<6","sentence-transformers>=3.0.1,<4","neo4j>=5.18.0,<6",@@ -493,8 +499,9 @@ module = ["fish_audio_sdk","sentence_transformers.*","pymilvus",- "pytidb","pytidb.schema",+ "faiss-cpu",+ "faiss-gpu","pillow","slack-sdk","slack-bolt",diff --git a/test/storages/vector_storages/test_faiss.py b/test/storages/vector_storages/test_faiss.pynew file mode 100644index 000000000..21a30ff07--- /dev/null+++ b/test/storages/vector_storages/test_faiss.py@@ -0,0 +1,175 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from unittest.mock import MagicMock, create_autospec, patch++import pytest+import numpy as np++from camel.storages import FaissStorage, VectorDBQuery, VectorRecord, VectorDistance+++def mock_faiss_storage():+ with patch('camel.storages.FaissStorage') as MockFaissStorage:+ mock_storage1 = create_autospec(FaissStorage)+ mock_storage2 = create_autospec(FaissStorage)+ MockFaissStorage.side_effect = [mock_storage1, mock_storage2]+ yield mock_storage1, mock_storage2+++def setup_mock_storage(mock_storage, vectors, query_result_id, payload):+ mock_query_result = MagicMock()+ mock_query_result.record.id = query_result_id+ mock_query_result.record.payload = payload+ mock_storage.query.return_value = [mock_query_result]+ mock_storage.add(vectors)+ mock_storage.status.return_value = MagicMock(vector_count=len(vectors), vector_dim=4)+++def test_multiple_faiss_instances(mock_faiss_storage):+ mock_storage1, mock_storage2 = mock_faiss_storage++ # Example vectors for testing+ vectors1 = [+ VectorRecord(+ vector=[0.1, 0.1, 0.1, 0.1], payload={"message": "text1"}+ ),+ VectorRecord(+ vector=[0.1, -0.1, -0.1, 0.1], payload={"message": "text2"}+ ),+ ]+ vectors2 = [+ VectorRecord(+ vector=[-0.1, 0.1, -0.1, 0.1], payload={"message": "text3"}+ ),+ VectorRecord(+ vector=[-0.1, 0.1, 0.1, 0.1],+ payload={"message": "text4", "number": 1},+ ),+ ]+ setup_mock_storage(+ mock_storage1, vectors1, vectors1[0].id, {"message": "text1"}+ )+ setup_mock_storage(+ mock_storage2, vectors2, vectors2[0].id, {"message": "text3"}+ )++ # Assert add method was called correctly+ mock_storage1.add.assert_called_once_with(vectors1)+ mock_storage2.add.assert_called_once_with(vectors2)++ # Perform and verify queries+ query1 = VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1)+ result1 = mock_storage1.query(query1)+ assert result1[0].record.id == vectors1[0].id+ assert result1[0].record.payload == {"message": "text1"}++ query2 = VectorDBQuery(query_vector=[-0.1, 0.2, -0.1, 0.1], top_k=1)+ result2 = mock_storage2.query(query2)+ assert result2[0].record.id == vectors2[0].id+ assert result2[0].record.payload == {"message": "text3"}++ # Clear and check status for each storage+ mock_storage1.clear()+ status1 = mock_storage1.status()+ assert status1.vector_count == 2+ assert status1.vector_dim == 4++ mock_storage2.clear()+ status2 = mock_storage2.status()+ assert status2.vector_count == 2+ assert status2.vector_dim == 4+++def mock_faiss_module():+ with patch('faiss.IndexFlatIP') as mock_index_ip, \+ patch('faiss.IndexFlatL2') as mock_index_l2, \+ patch('faiss.write_index') as mock_write, \+ patch('faiss.read_index') as mock_read:+ mock_index = MagicMock()+ mock_index_ip.return_value = mock_index+ mock_index_l2.return_value = mock_index+ yield {+ 'index': mock_index,+ 'write_index': mock_write,+ 'read_index': mock_read+ }+++def test_faiss_storage_initialization(mock_faiss_module):+ # Test different initialization parameters+ storage1 = FaissStorage(vector_dim=128)+ storage2 = FaissStorage(+ vector_dim=512,+ index_type='Flat',+ distance=VectorDistance.EUCLIDEAN+ )+ storage3 = FaissStorage(+ vector_dim=1536,+ index_type='IVFFlat',+ nlist=100,+ distance=VectorDistance.COSINE+ )++ # Check that indices were created correctly+ assert storage1.vector_dim == 128+ assert storage2.vector_dim == 512+ assert storage3.vector_dim == 1536+ assert storage1.distance == VectorDistance.COSINE # Default+ assert storage2.distance == VectorDistance.EUCLIDEAN+++def test_faiss_storage_operations():+ with patch('faiss.IndexFlatIP') as mock_index_class:+ mock_index = MagicMock()+ mock_index_class.return_value = mock_index++ # Initialize storage+ storage = FaissStorage(vector_dim=4)++ # Create test vectors+ vectors = [+ VectorRecord(+ vector=[0.1, 0.2, 0.3, 0.4],+ payload={"text": "Sample text", "source": "doc1"}+ ),+ VectorRecord(+ vector=[0.5, 0.6, 0.7, 0.8],+ payload={"text": "Another example", "source": "doc2"}+ )+ ]++ # Test add+ storage.add(vectors)+ assert mock_index.add.call_count == 2++ # Set up query results+ mock_index.search.return_value = (+ np.array([[0.95, 0.85]]), # Similarities+ np.array([[0, 1]]) # Indices+ )++ # Test query+ query = VectorDBQuery(query_vector=[0.2, 0.3, 0.4, 0.5], top_k=2)+ results = storage.query(query)++ # Check query was called correctly+ mock_index.search.assert_called_once()++ # Test delete+ storage.delete(ids=[vectors[0].id])++ # Test clear+ storage.clear()diff --git a/uv.lock b/uv.lockindex 9bbbb1770..63e65265f 100644--- a/uv.lock+++ b/uv.lock@@ -604,6 +604,8 @@ all = [{ name = "docx2txt" },{ name = "duckduckgo-search" },{ name = "e2b-code-interpreter" },+ { name = "faiss-cpu" },+ { name = "faiss-gpu" },{ name = "exa-py" },{ name = "fastapi" },{ name = "ffmpeg-python" },@@ -848,7 +850,9 @@ owl = [rag = [{ name = "cohere" },{ name = "crawl4ai" },- { name = "google-genai" },+ { name = "faiss-cpu" },+ { name = "faiss-gpu" },+ { name = "google-genai" }{ name = "nebula3-python" },{ name = "neo4j" },{ name = "numpy" },@@ -870,6 +874,8 @@ research-tools = [storage = [{ name = "azure-storage-blob" },{ name = "botocore" },+ { name = "faiss-cpu" },+ { name = "faiss-gpu" },{ name = "google-cloud-storage" },{ name = "mem0ai" },{ name = "nebula3-python" },@@ -982,6 +988,12 @@ requires-dist = [{ name = "e2b-code-interpreter", marker = "extra == 'owl'", specifier = ">=1.0.3,<2" },{ name = "exa-py", marker = "extra == 'all'", specifier = ">=1.10.0,<2" },{ name = "exa-py", marker = "extra == 'web-tools'", specifier = ">=1.10.0,<2" },+ { name = "faiss-cpu", marker = "extra == 'all'", specifier = ">=1.7.2,<2" },+ { name = "faiss-cpu", marker = "extra == 'rag'", specifier = ">=1.7.2,<2" },+ { name = "faiss-cpu", marker = "extra == 'storage'", specifier = ">=1.7.2,<2" },+ { name = "faiss-gpu", marker = "extra == 'all'", specifier = ">=1.7.2,<2" },+ { name = "faiss-gpu", marker = "extra == 'rag'", specifier = ">=1.7.2,<2" },+ { name = "faiss-gpu", marker = "extra == 'storage'", specifier = ">=1.7.2,<2" },{ name = "fastapi", marker = "extra == 'all'", specifier = ">=0.115.11" },{ name = "fastapi", marker = "extra == 'web-tools'", specifier = ">=0.115.11" },{ name = "ffmpeg-python", marker = "extra == 'all'", specifier = ">=0.2.0,<0.3" },@@ -2102,6 +2114,36 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/18/d6/314868f573b09d9f0590a2c2f7dd7463153d3dab1049f0ba5e7008776d91/Faker-19.13.0-py3-none-any.whl", hash = "sha256:da880a76322db7a879c848a0771e129338e0a680a9f695fd9a3e7a6ac82b45e1", size = 1737393 },]+[[package]]+name = "faiss-cpu"+version = "1.9.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "numpy", version = ">=1.17.0" },+]+sdist = { url = "https://files.pythonhosted.org/packages/b0/e4/71e3ce81a1902aaca10beee28830b7d0273d2231462e26b1be236a32afeb/faiss_cpu-1.9.0.tar.gz", hash = "sha256:2ab59a5678c39b1624c7e1bcd77c83a9672aa500494bf7a62e88c7ffcfd9f41e", size = 3346556 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/a3/e0/c8384aebcb3d7139a17bf5dd06bccbdd2071c1dbad7a15d07bca1e764fc5/faiss_cpu-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0e337ffeaa56eefc8c43ea989cb80d4263bbec4e9d0c8d593ce61b7aa5826ea", size = 10282377 },+ { url = "https://files.pythonhosted.org/packages/86/af/38f945bde305c4a83f11cec1bbff43d929a3bb6d2a370a827cf1481605b7/faiss_cpu-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10b9e096aa9a803a19e5152f5291ec3b3e87a6c40f854b5fcc97009f85455292", size = 10282379 },+ { url = "https://files.pythonhosted.org/packages/e8/a9/0f7eb0f8eb5def2018482e85b3d201e7a31420a9bae24486c499f1c9c988/faiss_cpu-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39c89dbf3c9d37789c522e049105d547ee5e2a83eeb06e5a1a82d8685a424d59", size = 10282379 },+ { url = "https://files.pythonhosted.org/packages/f9/55/a3c5c9d4f4bbb685d9a1a9bbf87f89302eadbd3a77e2ffe0de3b75253950/faiss_cpu-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08166ebbe5ed3d5e29f3b9c0839313196994174a7cb5bb460499a7e8659a8022", size = 10282379 },+]++[[package]]+name = "faiss-gpu"+version = "1.9.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "numpy", version = ">=1.17.0" },+]+sdist = { url = "https://files.pythonhosted.org/packages/c4/a0/799c9b3842d95fb23d6b6a0ee9c2324508dc15919efcbd3ae7ee89fd241e/faiss_gpu-1.9.0.tar.gz", hash = "sha256:9f27c17ebad005326db4303e9d82791ac25e58c5ece14d18c1d929e2c93ec67b", size = 3477882 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/8c/60/842222c53fa7d2399204e1c5c020fe9b08dc7699e76797280bda67308921/faiss_gpu-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dc14011b2c8a8fe380b1e4c22b973e3d8090caf49c1ec39ecbfff9c88aacbb9", size = 41365306 },+ { url = "https://files.pythonhosted.org/packages/f8/71/f494e6c2a6c8f7406d8f8efc120716a4ffbbb13e7370be022e805781bc19/faiss_gpu-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9354f365989de4432c96616b7448d6c6855fb424ff4e7f5618bfe7d6a215be42", size = 41365307 },+ { url = "https://files.pythonhosted.org/packages/e4/8d/ff5be0776623bbacfe74ebc70c6916911ca317f404592e136676e37a9b58/faiss_gpu-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1681a34a89a082e81f9dcac43eb9e67f90c859c584701ba789f3b48d9ca8304b", size = 41365307 },+ { url = "https://files.pythonhosted.org/packages/45/cf/9199e49b31507c395f7d0d970acfa93641d45e4ae82be14760dbfd409380/faiss_gpu-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4279f5c4574bdcc002c5d3ea9f4b5cc71f8513d3cbf1e28e3ea9f632c8de7ce5", size = 41365307 },+]+[[package]]name = "fastapi"version = "0.115.12"