Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

`feat: Add new vector storage db FAISS` (#2034)

Ziqi Chen5dd55f705006
diff --git a/camel/storages/__init__.py b/camel/storages/__init__.py
index 5ab6ae7b1..5a9c5dec4 100644
--- a/camel/storages/__init__.py
+++ b/camel/storages/__init__.py
@@ -30,6 +30,7 @@ from .vectordb_storages.milvus import MilvusStorage
from .vectordb_storages.oceanbase import OceanBaseStorage
from .vectordb_storages.qdrant import QdrantStorage
from .vectordb_storages.tidb import TiDBStorage
+from .vectordb_storages.faiss import FaissStorage
__all__ = [
'BaseKeyValueStorage',
@@ -43,6 +44,7 @@ __all__ = [
'QdrantStorage',
'MilvusStorage',
"TiDBStorage",
+ "FaissStorage",
'BaseGraphStorage',
'Neo4jGraph',
'NebulaGraph',
diff --git a/camel/storages/vectordb_storages/__init__.py b/camel/storages/vectordb_storages/__init__.py
index bf31e9ee2..49c0a200c 100644
--- a/camel/storages/vectordb_storages/__init__.py
+++ b/camel/storages/vectordb_storages/__init__.py
@@ -23,6 +23,7 @@ from .milvus import MilvusStorage
from .oceanbase import OceanBaseStorage
from .qdrant import QdrantStorage
from .tidb import TiDBStorage
+from .faiss import FaissStorage
__all__ = [
'BaseVectorStorage',
@@ -31,6 +32,7 @@ __all__ = [
'QdrantStorage',
'MilvusStorage',
"TiDBStorage",
+ 'FaissStorage',
'OceanBaseStorage',
'VectorRecord',
'VectorDBStatus',
diff --git a/camel/storages/vectordb_storages/faiss.py b/camel/storages/vectordb_storages/faiss.py
new file mode 100644
index 000000000..be7e7d3cc
--- /dev/null
+++ b/camel/storages/vectordb_storages/faiss.py
@@ -0,0 +1,641 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import logging
+import pickle
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Literal
+
+import numpy as np
+
+from camel.storages.vector_db.base import (
+ BaseVectorStorage,
+ VectorDBQuery,
+ VectorDBQueryResult,
+ VectorDBStatus,
+ VectorRecord,
+)
+from camel.types import VectorDistance
+from camel.utils import dependencies_required
+
+logger = logging.get_logger(__name__)
+
+
+class FaissStorage(BaseVectorStorage):
+ """An implementation of the `BaseVectorStorage` using FAISS,
+ Facebook AI's Similarity Search library for efficient vector search.
+
+ The detailed information about FAISS is available at:
+ `FAISS <https://github.com/facebookresearch/faiss>`_
+
+ Args:
+ vector_dim (int): The dimension of storing vectors.
+ index_type (str, optional): Type of FAISS index to create.
+ Options include 'Flat', 'IVF', 'HNSW', etc. (default: :obj:`'Flat'`)
+ collection_name (Optional[str], optional): Name for the collection.
+ If not provided, set it to the current time with iso format.
+ (default: :obj:`None`)
+ storage_path (Optional[str], optional): Path to directory where
+ the index will be stored. If None, index will only exist in memory.
+ (default: :obj:`None`)
+ distance (VectorDistance, optional): The distance metric for vector
+ comparison (default: :obj:`VectorDistance.COSINE`)
+ nlist (int, optional): Number of cluster centroids for IVF indexes.
+ Only used if index_type includes 'IVF'. (default: :obj:`100`)
+ m (int, optional): HNSW parameter. Number of connections per node.
+ Only used if index_type includes 'HNSW'. (default: :obj:`16`)
+ **kwargs (Any): Additional keyword arguments.
+
+ Notes:
+ - FAISS offers various index types optimized for different use cases:
+ - 'Flat': Exact search, but slowest for large datasets
+ - 'IVF': Inverted file index, good balance of speed and recall
+ - 'HNSW': Hierarchical Navigable Small World, fast with high recall
+ - 'PQ': Product Quantization for memory-efficient storage
+ - The choice of index should be based on your specific requirements
+ for search speed, memory usage, and accuracy.
+ """
+
+ @dependencies_required('faiss')
+ def __init__(
+ self,
+ vector_dim: int,
+ index_type: str = 'Flat',
+ collection_name: Optional[str] = None,
+ storage_path: Optional[str] = None,
+ distance: VectorDistance = VectorDistance.COSINE,
+ nlist: int = 100,
+ m: int = 16,
+ **kwargs: Any,
+ ) -> None:
+ """Initialize the FAISS vector storage.
+
+ Args:
+ vector_dim: Dimension of vectors to be stored
+ index_type: FAISS index type ('Flat', 'IVF', 'HNSW', etc.)
+ collection_name: Name of the collection (defaults to timestamp)
+ storage_path: Directory to save the index (None for in-memory only)
+ distance: Vector distance metric
+ nlist: Number of clusters for IVF indexes
+ m: HNSW parameter for connections per node
+ **kwargs: Additional parameters
+ """
+ import faiss
+
+ self.vector_dim = vector_dim
+ self.index_type = index_type
+ self.collection_name = collection_name or self._generate_collection_name()
+ self.storage_path = storage_path
+ self.distance = distance
+ self.nlist = nlist
+ self.m = m
+ self._faiss_client = faiss # Store the faiss module as the client
+
+ # Create directory for storage if it doesn't exist
+ if self.storage_path is not None:
+ os.makedirs(self.storage_path, exist_ok=True)
+
+ # Initialize the FAISS index
+ self._index = self._create_index()
+
+ # Storage for IDs and payloads (FAISS only stores vectors)
+ self._id_to_index: Dict[str, int] = {}
+ self._index_to_id: Dict[int, str] = {}
+ self._payloads: Dict[str, Dict[str, Any]] = {}
+ self._vectors: Dict[str, np.ndarray] = {}
+
+ # Load existing index if it exists
+ if self.storage_path:
+ self._load_from_disk()
+
+ def _generate_collection_name(self) -> str:
+ """Generates a collection name if user doesn't provide"""
+ return f"faiss_index_{datetime.now().isoformat()}"
+
+ def _get_index_path(self) -> str:
+ """Returns the path to the index file"""
+ if self.storage_path is None:
+ raise ValueError("Storage path is not set.")
+ return os.path.join(self.storage_path, f"{self.collection_name}.index")
+
+ def _get_metadata_path(self) -> str:
+ """Returns the path to the metadata file"""
+ if self.storage_path is None:
+ raise ValueError("Storage path is not set.")
+ return os.path.join(self.storage_path, f"{self.collection_name}.metadata")
+
+ def _create_index(self):
+ """Creates a new FAISS index based on specified parameters.
+
+ Returns:
+ A FAISS index object configured according to the parameters.
+ """
+ import faiss
+
+ # Determine the metric to use based on distance type
+ if self.distance == VectorDistance.COSINE:
+ # For cosine similarity, we need to normalize vectors
+ metric = faiss.METRIC_INNER_PRODUCT
+ elif self.distance == VectorDistance.EUCLIDEAN:
+ metric = faiss.METRIC_L2
+ elif self.distance == VectorDistance.DOT:
+ metric = faiss.METRIC_INNER_PRODUCT
+ else:
+ raise ValueError(f"Unsupported distance metric: {self.distance}")
+
+ # Create the appropriate index based on index_type
+ if self.index_type == 'Flat':
+ if metric == faiss.METRIC_INNER_PRODUCT:
+ index = faiss.IndexFlatIP(self.vector_dim)
+ else:
+ index = faiss.IndexFlatL2(self.vector_dim)
+ elif self.index_type.startswith('IVF'):
+ # IVF requires a quantizer (often a flat index)
+ quantizer = faiss.IndexFlatL2(self.vector_dim)
+ if 'Flat' in self.index_type:
+ index = faiss.IndexIVFFlat(quantizer, self.vector_dim, self.nlist, metric)
+ elif 'PQ' in self.index_type:
+ # M value for PQ, typically a divisor of vector_dim
+ m = self.vector_dim // 4 # default setting
+ nbits = 8 # typically 8 bits per sub-vector
+ index = faiss.IndexIVFPQ(quantizer, self.vector_dim, self.nlist, m, nbits)
+ else:
+ raise ValueError(f"Unsupported IVF index type: {self.index_type}")
+
+ # IVF indexes need to be trained before use
+ # This is a placeholder since actual training requires data
+ index.nprobe = 10 # Number of clusters to search (trade-off between speed and accuracy)
+ elif self.index_type == 'HNSW':
+ index = faiss.IndexHNSWFlat(self.vector_dim, self.m, metric)
+ else:
+ raise ValueError(f"Unsupported index type: {self.index_type}")
+
+ return index
+
+ def _save_to_disk(self) -> None:
+ """Saves the index and metadata to disk if storage_path is provided."""
+ if self.storage_path is None:
+ return
+
+ import faiss
+
+ # Save the FAISS index
+ faiss.write_index(self._index, self._get_index_path())
+
+ # Save the metadata (IDs, payloads, vectors)
+ metadata = {
+ "id_to_index": self._id_to_index,
+ "index_to_id": self._index_to_id,
+ "payloads": self._payloads,
+ "vectors": self._vectors,
+ "vector_dim": self.vector_dim,
+ "distance": self.distance,
+ }
+
+ with open(self._get_metadata_path(), 'wb') as f:
+ pickle.dump(metadata, f)
+
+ logger.info(f"Saved FAISS index and metadata to {self.storage_path}")
+
+ def _load_from_disk(self) -> None:
+ """Loads the index and metadata from disk if they exist."""
+ if self.storage_path is None:
+ return
+
+ import faiss
+
+ index_path = self._get_index_path()
+ metadata_path = self._get_metadata_path()
+
+ if os.path.exists(index_path) and os.path.exists(metadata_path):
+ try:
+ # Load the FAISS index
+ self._index = faiss.read_index(index_path)
+
+ # Load the metadata
+ with open(metadata_path, 'rb') as f:
+ metadata = pickle.load(f)
+
+ self._id_to_index = metadata["id_to_index"]
+ self._index_to_id = metadata["index_to_id"]
+ self._payloads = metadata["payloads"]
+ self._vectors = metadata["vectors"]
+
+ # Check that the loaded index is compatible
+ if metadata["vector_dim"] != self.vector_dim:
+ logger.warning(
+ f"Loaded index has different vector dimension "
+ f"({metadata['vector_dim']}) than specified ({self.vector_dim})."
+ )
+
+ logger.info(f"Loaded FAISS index and metadata from {self.storage_path}")
+ except Exception as e:
+ logger.error(f"Failed to load index from disk: {e}")
+ # Initialize a new index
+ self._index = self._create_index()
+ else:
+ logger.info("No existing index found. Creating new one.")
+
+ def add(
+ self,
+ records: List[VectorRecord],
+ **kwargs,
+ ) -> None:
+ """Adds a list of vectors to the index.
+
+ Args:
+ records (List[VectorRecord]): List of vector records to be added.
+ **kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ RuntimeError: If there was an error in the addition process.
+ """
+ if not records:
+ return
+
+ # Check if the index needs training (for IVF indexes)
+ if self.index_type.startswith('IVF') and not self._index.is_trained:
+ # For IVF indexes, we need to train with vectors before adding
+ vectors = np.array([record.vector for record in records], dtype=np.float32)
+ try:
+ self._index.train(vectors)
+ except Exception as e:
+ raise RuntimeError(f"Failed to train FAISS index: {e}")
+
+ # Add each record to the index
+ for record in records:
+ # Normalize vector if using cosine similarity
+ vector = np.array(record.vector, dtype=np.float32).reshape(1, -1)
+ if self.distance == VectorDistance.COSINE:
+ vector = self._normalize_vector(vector)
+
+ # Get the next index
+ idx = len(self._id_to_index)
+
+ # Add to FAISS index and update mappings atomically
+ try:
+ self._index.add(vector)
+ # Store mapping from ID to index
+ self._id_to_index[record.id] = idx
+ self._index_to_id[idx] = record.id
+ # Store payload
+ if record.payload is not None:
+ self._payloads[record.id] = record.payload
+ else:
+ self._payloads[record.id] = {}
+ # Store the original vector for later retrieval
+ self._vectors[record.id] = vector.flatten()
+ except Exception as e:
+ # If adding to the index fails, roll back any partial changes
+ if record.id in self._id_to_index:
+ del self._id_to_index[record.id]
+ if idx in self._index_to_id:
+ del self._index_to_id[idx]
+ if record.id in self._payloads:
+ del self._payloads[record.id]
+ if record.id in self._vectors:
+ del self._vectors[record.id]
+ raise RuntimeError(f"Failed to add vector to FAISS index: {e}")
+
+ # Save to disk if storage path is provided
+ self._save_to_disk()
+
+ def update_payload(
+ self, ids: List[str], payload: Dict[str, Any], **kwargs: Any
+ ) -> None:
+ """Updates the payload of the vectors identified by their IDs.
+
+ Args:
+ ids (List[str]): List of unique identifiers for the vectors to be
+ updated.
+ payload (Dict[str, Any]): Payload to be updated for all specified IDs.
+ **kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ KeyError: If any of the provided IDs does not exist in the index.
+ """
+ for id in ids:
+ if id not in self._payloads:
+ raise KeyError(f"Vector with ID {id} not found in the index.")
+
+ # Update payload (merge with existing payload)
+ if id in self._payloads:
+ self._payloads[id].update(payload)
+ else:
+ self._payloads[id] = payload.copy()
+
+ # Save to disk if storage path is provided
+ self._save_to_disk()
+
+ def delete_collection(self) -> None:
+ """Deletes the entire collection (index and metadata)."""
+ # Reset the index
+ self._index = self._create_index()
+
+ # Clear metadata
+ self._id_to_index = {}
+ self._index_to_id = {}
+ self._payloads = {}
+ self._vectors = {}
+
+ # Remove files from disk if storage path is provided
+ if self.storage_path:
+ index_path = self._get_index_path()
+ metadata_path = self._get_metadata_path()
+
+ if os.path.exists(index_path):
+ os.remove(index_path)
+
+ if os.path.exists(metadata_path):
+ os.remove(metadata_path)
+
+ logger.info(f"Deleted FAISS index and metadata from {self.storage_path}")
+
+ def delete(
+ self,
+ ids: Optional[List[str]] = None,
+ payload_filter: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> None:
+ """Deletes vectors from the index based on either IDs or payload
+ filters.
+
+ Args:
+ ids (Optional[List[str]], optional): List of unique identifiers
+ for the vectors to be deleted.
+ payload_filter (Optional[Dict[str, Any]], optional): A filter for
+ the payload to delete points matching specific conditions.
+ **kwargs (Any): Additional keyword arguments.
+
+ Raises:
+ ValueError: If neither `ids` nor `payload_filter` is provided.
+ RuntimeError: If the FAISS index does not support removal.
+
+ Notes:
+ - FAISS does not support efficient single vector removal for most
+ index types. This implementation recreates the index without the
+ deleted vectors, which can be inefficient for large datasets.
+ - If both `ids` and `payload_filter` are provided, both filters
+ will be applied (vectors matching either will be deleted).
+ """
+ import faiss
+
+ if not ids and not payload_filter:
+ raise ValueError(
+ "You must provide either `ids` or `payload_filter` to delete "
+ "vectors."
+ )
+
+ # Get IDs to delete from payload filter
+ if payload_filter:
+ filtered_ids = [
+ id for id, payload in self._payloads.items()
+ if all(payload.get(key) == value for key, value in payload_filter.items())
+ ]
+ else:
+ filtered_ids = []
+
+ # Combine with explicit IDs
+ ids_to_delete = set(ids or []) | set(filtered_ids)
+ if not ids_to_delete:
+ return
+
+ # Check if the index supports removal
+ if hasattr(self._index, 'remove_ids'):
+ # Convert IDs to indices
+ indices_to_remove = np.array(
+ [self._id_to_index[id] for id in ids_to_delete if id in self._id_to_index],
+ dtype=np.int64
+ )
+
+ if len(indices_to_remove) > 0:
+ # Create a selector where 1 means "remove"
+ selector = np.zeros(self._index.ntotal, dtype=bool)
+ selector[indices_to_remove] = True
+
+ try:
+ # Remove from FAISS index
+ id_selector = faiss.IDSelectorArray(len(indices_to_remove), indices_to_remove)
+ self._index.remove_ids(id_selector)
+
+ # Update mappings and storage
+ for id in ids_to_delete:
+ if id in self._id_to_index:
+ idx = self._id_to_index[id]
+ del self._index_to_id[idx]
+ del self._id_to_index[id]
+ if id in self._payloads:
+ del self._payloads[id]
+ if id in self._vectors:
+ del self._vectors[id]
+
+ # Save to disk if storage path is provided
+ self._save_to_disk()
+ except Exception as e:
+ raise RuntimeError(f"Failed to remove vectors from FAISS index: {e}")
+ else:
+ # Index doesn't support removal, need to rebuild
+ logger.warning(
+ "This FAISS index type doesn't support direct removal. "
+ "Rebuilding entire index (may be slow for large datasets)."
+ )
+
+ # Get all vectors and IDs that should be kept
+ keep_ids = [id for id in self._id_to_index.keys() if id not in ids_to_delete]
+
+ # Create a new index
+ new_index = self._create_index()
+
+ # If it's an IVF index and there are vectors to train on, train it
+ if self.index_type.startswith('IVF') and keep_ids:
+ train_vectors = np.vstack([self._vectors[id].reshape(1, -1) for id in keep_ids])
+ new_index.train(train_vectors)
+
+ # Add the vectors to keep to the new index
+ new_id_to_index = {}
+ new_index_to_id = {}
+
+ for new_idx, id in enumerate(keep_ids):
+ vector = self._vectors[id].reshape(1, -1)
+ new_index.add(vector)
+ new_id_to_index[id] = new_idx
+ new_index_to_id[new_idx] = id
+
+ # Replace the old index and mappings
+ self._index = new_index
+ self._id_to_index = new_id_to_index
+ self._index_to_id = new_index_to_id
+
+ # Remove deleted IDs from payloads and vectors
+ for id in ids_to_delete:
+ if id in self._payloads:
+ del self._payloads[id]
+ if id in self._vectors:
+ del self._vectors[id]
+
+ # Save to disk if storage path is provided
+ self._save_to_disk()
+
+ def status(self) -> VectorDBStatus:
+ """Returns the status of the vector database.
+
+ Returns:
+ VectorDBStatus: Current status of the vector database.
+ """
+ return VectorDBStatus(
+ vector_dim=self.vector_dim,
+ vector_count=self._index.ntotal,
+ )
+
+ def query(
+ self,
+ query: VectorDBQuery,
+ filter_conditions: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> List[VectorDBQueryResult]:
+ """Searches for similar vectors in the storage based on the provided
+ query.
+
+ Args:
+ query (VectorDBQuery): The query object containing the search
+ vector and the number of top similar vectors to retrieve.
+ filter_conditions (Optional[Dict[str, Any]], optional): A
+ dictionary specifying conditions to filter the query results.
+ **kwargs (Any): Additional keyword arguments.
+
+ Returns:
+ List[VectorDBQueryResult]: A list of query results ordered by
+ similarity.
+ """
+ if self._index.ntotal == 0:
+ return []
+
+ # Prepare the query vector
+ query_vector = np.array(query.query_vector, dtype=np.float32)
+ if query_vector.ndim == 1:
+ query_vector = query_vector.reshape(1, -1)
+
+ # Normalize if using cosine similarity
+ if self.distance == VectorDistance.COSINE:
+ query_vector = self._normalize_vector(query_vector)
+
+ # For IVF indexes, set the number of clusters to probe (higher = more accurate but slower)
+ if hasattr(self._index, 'nprobe'):
+ nprobe = kwargs.get('nprobe', 10) # Default to 10
+ self._index.nprobe = nprobe
+
+ # Determine how many results to fetch initially when using filters
+ k = query.top_k
+ if filter_conditions:
+ # Fetch more results if filtering is applied to ensure we have enough after filtering
+ # This is a simple heuristic - multiply by 2 or by a user-provided factor
+ fetch_factor = kwargs.get('fetch_factor', 2)
+ fetch_k = min(int(k * fetch_factor), self._index.ntotal)
+ else:
+ fetch_k = min(k, self._index.ntotal)
+
+ # Perform the search
+ distances, indices = self._index.search(query_vector, fetch_k)
+
+ # Convert results to VectorDBQueryResult objects
+ results = []
+ for i, idx in enumerate(indices[0]):
+ if idx == -1: # FAISS returns -1 for empty slots
+ continue
+
+ vector_id = self._index_to_id.get(idx)
+ if not vector_id:
+ continue
+
+ # If there are filter conditions, check if this result passes
+ if filter_conditions and not self._matches_filter(vector_id, filter_conditions):
+ continue
+
+ # Adjust similarity score based on distance metric
+ if self.distance == VectorDistance.EUCLIDEAN:
+ # For Euclidean distance, smaller is better, so we invert
+ similarity = 1.0 / (1.0 + distances[0][i])
+ else:
+ # For inner product/cosine, higher is better
+ similarity = float(distances[0][i])
+
+ vector = self._vectors.get(vector_id)
+ if vector is not None:
+ vector = vector.tolist()
+ else:
+ vector = []
+
+ results.append(
+ VectorDBQueryResult(
+ similarity=similarity,
+ id=vector_id,
+ payload=self._payloads.get(vector_id, {}),
+ vector=vector,
+ )
+ )
+
+ # Stop once we have enough results
+ if len(results) >= k:
+ break
+
+ return results
+
+ def clear(self) -> None:
+ """Remove all vectors from the storage."""
+ self.delete_collection()
+
+ def load(self) -> None:
+ """Load the index from disk if storage_path is provided."""
+ if self.storage_path:
+ self._load_from_disk()
+ else:
+ logger.warning("No storage path provided. Cannot load index.")
+
+ @property
+ def client(self) -> Any:
+ """Provides access to the underlying FAISS client."""
+ return self._faiss_client
+
+ def _matches_filter(self, vector_id: str, filter_conditions: Dict[str, Any]) -> bool:
+ """Checks if a vector's payload matches the filter conditions.
+
+ Args:
+ vector_id (str): ID of the vector to check.
+ filter_conditions (Dict[str, Any]): Conditions to match against.
+
+ Returns:
+ bool: True if the payload matches all conditions, False otherwise.
+ """
+ payload = self._payloads.get(vector_id, {})
+ for key, value in filter_conditions.items():
+ if key not in payload or payload[key] != value:
+ return False
+ return True
+
+ def _normalize_vector(self, vector: np.ndarray) -> np.ndarray:
+ """Normalizes a vector to unit length for cosine similarity.
+
+ Args:
+ vector (np.ndarray): Vector to normalize, either 1D or 2D array.
+
+ Returns:
+ np.ndarray: Normalized vector with the same shape as input.
+ """
+ if vector.ndim == 1:
+ vector = vector.reshape(1, -1)
+ norm = np.linalg.norm(vector, axis=1, keepdims=True)
+ # Avoid division by zero
+ norm = np.maximum(norm, 1e-10)
+ return vector / norm
diff --git a/docs/camel.storages.vectordb_storages.rst b/docs/camel.storages.vectordb_storages.rst
index b1c2af596..eeff0b9c4 100644
--- a/docs/camel.storages.vectordb_storages.rst
+++ b/docs/camel.storages.vectordb_storages.rst
@@ -36,6 +36,14 @@ camel.storages.vectordb\_storages.tidb module
:undoc-members:
:show-inheritance:
+camel.storages.vectordb\_storages.faiss module
+-----------------------------------------------
+
+.. automodule:: camel.storages.vectordb_storages.faiss
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
Module contents
---------------
diff --git a/pyproject.toml b/pyproject.toml
index d9cac3001..4c368ed95 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -72,6 +72,8 @@ rag = [
"qdrant-client>=1.9.0,<2",
"pymilvus>=2.4.0,<3",
"pytidb-experimental==0.0.1.dev4",
+ "faiss-cpu>=1.7.2,<2",
+ "faiss-gpu>=1.7.2,<2",
"neo4j>=5.18.0,<6",
"nebula3-python==3.8.2",
"rank-bm25>=0.2.2,<0.3",
@@ -194,6 +196,8 @@ storage = [
"qdrant-client>=1.9.0,<2",
"pymilvus>=2.4.0,<3",
"pytidb-experimental==0.0.1.dev4",
+ "faiss-cpu>=1.7.2,<2",
+ "faiss-gpu>=1.7.2,<2",
"neo4j>=5.18.0,<6",
"nebula3-python==3.8.2",
"redis>=5.0.6,<6",
@@ -316,6 +320,8 @@ all = [
"qdrant-client>=1.9.0,<2",
"pymilvus>=2.4.0,<3",
"pytidb-experimental==0.0.1.dev4",
+ "faiss-cpu>=1.7.2,<2",
+ "faiss-gpu>=1.7.2,<2",
"cohere>=5.11.0,<6",
"sentence-transformers>=3.0.1,<4",
"neo4j>=5.18.0,<6",
@@ -493,8 +499,9 @@ module = [
"fish_audio_sdk",
"sentence_transformers.*",
"pymilvus",
- "pytidb",
"pytidb.schema",
+ "faiss-cpu",
+ "faiss-gpu",
"pillow",
"slack-sdk",
"slack-bolt",
diff --git a/test/storages/vector_storages/test_faiss.py b/test/storages/vector_storages/test_faiss.py
new file mode 100644
index 000000000..21a30ff07
--- /dev/null
+++ b/test/storages/vector_storages/test_faiss.py
@@ -0,0 +1,175 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from unittest.mock import MagicMock, create_autospec, patch
+
+import pytest
+import numpy as np
+
+from camel.storages import FaissStorage, VectorDBQuery, VectorRecord, VectorDistance
+
+
+def mock_faiss_storage():
+ with patch('camel.storages.FaissStorage') as MockFaissStorage:
+ mock_storage1 = create_autospec(FaissStorage)
+ mock_storage2 = create_autospec(FaissStorage)
+ MockFaissStorage.side_effect = [mock_storage1, mock_storage2]
+ yield mock_storage1, mock_storage2
+
+
+def setup_mock_storage(mock_storage, vectors, query_result_id, payload):
+ mock_query_result = MagicMock()
+ mock_query_result.record.id = query_result_id
+ mock_query_result.record.payload = payload
+ mock_storage.query.return_value = [mock_query_result]
+ mock_storage.add(vectors)
+ mock_storage.status.return_value = MagicMock(vector_count=len(vectors), vector_dim=4)
+
+
+def test_multiple_faiss_instances(mock_faiss_storage):
+ mock_storage1, mock_storage2 = mock_faiss_storage
+
+ # Example vectors for testing
+ vectors1 = [
+ VectorRecord(
+ vector=[0.1, 0.1, 0.1, 0.1], payload={"message": "text1"}
+ ),
+ VectorRecord(
+ vector=[0.1, -0.1, -0.1, 0.1], payload={"message": "text2"}
+ ),
+ ]
+ vectors2 = [
+ VectorRecord(
+ vector=[-0.1, 0.1, -0.1, 0.1], payload={"message": "text3"}
+ ),
+ VectorRecord(
+ vector=[-0.1, 0.1, 0.1, 0.1],
+ payload={"message": "text4", "number": 1},
+ ),
+ ]
+ setup_mock_storage(
+ mock_storage1, vectors1, vectors1[0].id, {"message": "text1"}
+ )
+ setup_mock_storage(
+ mock_storage2, vectors2, vectors2[0].id, {"message": "text3"}
+ )
+
+ # Assert add method was called correctly
+ mock_storage1.add.assert_called_once_with(vectors1)
+ mock_storage2.add.assert_called_once_with(vectors2)
+
+ # Perform and verify queries
+ query1 = VectorDBQuery(query_vector=[0.1, 0.2, 0.1, 0.1], top_k=1)
+ result1 = mock_storage1.query(query1)
+ assert result1[0].record.id == vectors1[0].id
+ assert result1[0].record.payload == {"message": "text1"}
+
+ query2 = VectorDBQuery(query_vector=[-0.1, 0.2, -0.1, 0.1], top_k=1)
+ result2 = mock_storage2.query(query2)
+ assert result2[0].record.id == vectors2[0].id
+ assert result2[0].record.payload == {"message": "text3"}
+
+ # Clear and check status for each storage
+ mock_storage1.clear()
+ status1 = mock_storage1.status()
+ assert status1.vector_count == 2
+ assert status1.vector_dim == 4
+
+ mock_storage2.clear()
+ status2 = mock_storage2.status()
+ assert status2.vector_count == 2
+ assert status2.vector_dim == 4
+
+
+def mock_faiss_module():
+ with patch('faiss.IndexFlatIP') as mock_index_ip, \
+ patch('faiss.IndexFlatL2') as mock_index_l2, \
+ patch('faiss.write_index') as mock_write, \
+ patch('faiss.read_index') as mock_read:
+ mock_index = MagicMock()
+ mock_index_ip.return_value = mock_index
+ mock_index_l2.return_value = mock_index
+ yield {
+ 'index': mock_index,
+ 'write_index': mock_write,
+ 'read_index': mock_read
+ }
+
+
+def test_faiss_storage_initialization(mock_faiss_module):
+ # Test different initialization parameters
+ storage1 = FaissStorage(vector_dim=128)
+ storage2 = FaissStorage(
+ vector_dim=512,
+ index_type='Flat',
+ distance=VectorDistance.EUCLIDEAN
+ )
+ storage3 = FaissStorage(
+ vector_dim=1536,
+ index_type='IVFFlat',
+ nlist=100,
+ distance=VectorDistance.COSINE
+ )
+
+ # Check that indices were created correctly
+ assert storage1.vector_dim == 128
+ assert storage2.vector_dim == 512
+ assert storage3.vector_dim == 1536
+ assert storage1.distance == VectorDistance.COSINE # Default
+ assert storage2.distance == VectorDistance.EUCLIDEAN
+
+
+def test_faiss_storage_operations():
+ with patch('faiss.IndexFlatIP') as mock_index_class:
+ mock_index = MagicMock()
+ mock_index_class.return_value = mock_index
+
+ # Initialize storage
+ storage = FaissStorage(vector_dim=4)
+
+ # Create test vectors
+ vectors = [
+ VectorRecord(
+ vector=[0.1, 0.2, 0.3, 0.4],
+ payload={"text": "Sample text", "source": "doc1"}
+ ),
+ VectorRecord(
+ vector=[0.5, 0.6, 0.7, 0.8],
+ payload={"text": "Another example", "source": "doc2"}
+ )
+ ]
+
+ # Test add
+ storage.add(vectors)
+ assert mock_index.add.call_count == 2
+
+ # Set up query results
+ mock_index.search.return_value = (
+ np.array([[0.95, 0.85]]), # Similarities
+ np.array([[0, 1]]) # Indices
+ )
+
+ # Test query
+ query = VectorDBQuery(query_vector=[0.2, 0.3, 0.4, 0.5], top_k=2)
+ results = storage.query(query)
+
+ # Check query was called correctly
+ mock_index.search.assert_called_once()
+
+ # Test delete
+ storage.delete(ids=[vectors[0].id])
+
+ # Test clear
+ storage.clear()
diff --git a/uv.lock b/uv.lock
index 9bbbb1770..63e65265f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -604,6 +604,8 @@ all = [
{ name = "docx2txt" },
{ name = "duckduckgo-search" },
{ name = "e2b-code-interpreter" },
+ { name = "faiss-cpu" },
+ { name = "faiss-gpu" },
{ name = "exa-py" },
{ name = "fastapi" },
{ name = "ffmpeg-python" },
@@ -848,7 +850,9 @@ owl = [
rag = [
{ name = "cohere" },
{ name = "crawl4ai" },
- { name = "google-genai" },
+ { name = "faiss-cpu" },
+ { name = "faiss-gpu" },
+ { name = "google-genai" }
{ name = "nebula3-python" },
{ name = "neo4j" },
{ name = "numpy" },
@@ -870,6 +874,8 @@ research-tools = [
storage = [
{ name = "azure-storage-blob" },
{ name = "botocore" },
+ { name = "faiss-cpu" },
+ { name = "faiss-gpu" },
{ name = "google-cloud-storage" },
{ name = "mem0ai" },
{ name = "nebula3-python" },
@@ -982,6 +988,12 @@ requires-dist = [
{ name = "e2b-code-interpreter", marker = "extra == 'owl'", specifier = ">=1.0.3,<2" },
{ name = "exa-py", marker = "extra == 'all'", specifier = ">=1.10.0,<2" },
{ name = "exa-py", marker = "extra == 'web-tools'", specifier = ">=1.10.0,<2" },
+ { name = "faiss-cpu", marker = "extra == 'all'", specifier = ">=1.7.2,<2" },
+ { name = "faiss-cpu", marker = "extra == 'rag'", specifier = ">=1.7.2,<2" },
+ { name = "faiss-cpu", marker = "extra == 'storage'", specifier = ">=1.7.2,<2" },
+ { name = "faiss-gpu", marker = "extra == 'all'", specifier = ">=1.7.2,<2" },
+ { name = "faiss-gpu", marker = "extra == 'rag'", specifier = ">=1.7.2,<2" },
+ { name = "faiss-gpu", marker = "extra == 'storage'", specifier = ">=1.7.2,<2" },
{ name = "fastapi", marker = "extra == 'all'", specifier = ">=0.115.11" },
{ name = "fastapi", marker = "extra == 'web-tools'", specifier = ">=0.115.11" },
{ name = "ffmpeg-python", marker = "extra == 'all'", specifier = ">=0.2.0,<0.3" },
@@ -2102,6 +2114,36 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/18/d6/314868f573b09d9f0590a2c2f7dd7463153d3dab1049f0ba5e7008776d91/Faker-19.13.0-py3-none-any.whl", hash = "sha256:da880a76322db7a879c848a0771e129338e0a680a9f695fd9a3e7a6ac82b45e1", size = 1737393 },
]
+[[package]]
+name = "faiss-cpu"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "numpy", version = ">=1.17.0" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b0/e4/71e3ce81a1902aaca10beee28830b7d0273d2231462e26b1be236a32afeb/faiss_cpu-1.9.0.tar.gz", hash = "sha256:2ab59a5678c39b1624c7e1bcd77c83a9672aa500494bf7a62e88c7ffcfd9f41e", size = 3346556 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a3/e0/c8384aebcb3d7139a17bf5dd06bccbdd2071c1dbad7a15d07bca1e764fc5/faiss_cpu-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0e337ffeaa56eefc8c43ea989cb80d4263bbec4e9d0c8d593ce61b7aa5826ea", size = 10282377 },
+ { url = "https://files.pythonhosted.org/packages/86/af/38f945bde305c4a83f11cec1bbff43d929a3bb6d2a370a827cf1481605b7/faiss_cpu-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10b9e096aa9a803a19e5152f5291ec3b3e87a6c40f854b5fcc97009f85455292", size = 10282379 },
+ { url = "https://files.pythonhosted.org/packages/e8/a9/0f7eb0f8eb5def2018482e85b3d201e7a31420a9bae24486c499f1c9c988/faiss_cpu-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39c89dbf3c9d37789c522e049105d547ee5e2a83eeb06e5a1a82d8685a424d59", size = 10282379 },
+ { url = "https://files.pythonhosted.org/packages/f9/55/a3c5c9d4f4bbb685d9a1a9bbf87f89302eadbd3a77e2ffe0de3b75253950/faiss_cpu-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08166ebbe5ed3d5e29f3b9c0839313196994174a7cb5bb460499a7e8659a8022", size = 10282379 },
+]
+
+[[package]]
+name = "faiss-gpu"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "numpy", version = ">=1.17.0" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c4/a0/799c9b3842d95fb23d6b6a0ee9c2324508dc15919efcbd3ae7ee89fd241e/faiss_gpu-1.9.0.tar.gz", hash = "sha256:9f27c17ebad005326db4303e9d82791ac25e58c5ece14d18c1d929e2c93ec67b", size = 3477882 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8c/60/842222c53fa7d2399204e1c5c020fe9b08dc7699e76797280bda67308921/faiss_gpu-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dc14011b2c8a8fe380b1e4c22b973e3d8090caf49c1ec39ecbfff9c88aacbb9", size = 41365306 },
+ { url = "https://files.pythonhosted.org/packages/f8/71/f494e6c2a6c8f7406d8f8efc120716a4ffbbb13e7370be022e805781bc19/faiss_gpu-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9354f365989de4432c96616b7448d6c6855fb424ff4e7f5618bfe7d6a215be42", size = 41365307 },
+ { url = "https://files.pythonhosted.org/packages/e4/8d/ff5be0776623bbacfe74ebc70c6916911ca317f404592e136676e37a9b58/faiss_gpu-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1681a34a89a082e81f9dcac43eb9e67f90c859c584701ba789f3b48d9ca8304b", size = 41365307 },
+ { url = "https://files.pythonhosted.org/packages/45/cf/9199e49b31507c395f7d0d970acfa93641d45e4ae82be14760dbfd409380/faiss_gpu-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4279f5c4574bdcc002c5d3ea9f4b5cc71f8513d3cbf1e28e3ea9f632c8de7ce5", size = 41365307 },
+]
+
[[package]]
name = "fastapi"
version = "0.115.12"