🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: auto retriever (#469)
Wendong-Fanc276ab3e1fdb
diff --git a/camel/retrievers/__init__.py b/camel/retrievers/__init__.pyindex e35c092e7..03dea03ce 100644--- a/camel/retrievers/__init__.py+++ b/camel/retrievers/__init__.py@@ -13,8 +13,10 @@# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========from .base import BaseRetrieverfrom .vector_retriever import VectorRetriever+from .auto_retriever import AutoRetriever__all__ = ['BaseRetriever','VectorRetriever',+ 'AutoRetriever',]diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.pynew file mode 100644index 000000000..cd383837b--- /dev/null+++ b/camel/retrievers/auto_retriever.py@@ -0,0 +1,277 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import datetime+import os+import re+from pathlib import Path+from typing import List, Optional, Tuple, Union+from urllib.parse import urlparse++from camel.embeddings import BaseEmbedding, OpenAIEmbedding+from camel.retrievers import VectorRetriever+from camel.storages import (+ BaseVectorStorage,+ MilvusStorage,+ QdrantStorage,+ VectorDBQuery,+)+from camel.types import StorageType++DEFAULT_TOP_K_RESULTS = 1+DEFAULT_SIMILARITY_THRESHOLD = 0.75+++class AutoRetriever():+ r""" Facilitates the automatic retrieval of information using a+ query-based approach with pre-defined elements.++ Attributes:+ url_and_api_key (Optional[Tuple[str, str]]): URL and API key for+ accessing the vector storage remotely.+ vector_storage_local_path (Optional[str]): Local path for vector+ storage, if applicable.+ storage_type (Optional[StorageType]): The type of vector storage to+ use. Defaults to `StorageType.MILVUS`.+ embedding_model (Optional[BaseEmbedding]): Model used for embedding+ queries and documents. Defaults to `OpenAIEmbedding()`.+ """++ def __init__(+ self,+ url_and_api_key: Optional[Tuple[str, str]] = None,+ vector_storage_local_path: Optional[str] = None,+ storage_type: Optional[StorageType] = None,+ embedding_model: Optional[BaseEmbedding] = None,+ ):++ self.storage_type = storage_type or StorageType.MILVUS+ self.embedding_model = embedding_model or OpenAIEmbedding()+ self.vector_storage_local_path = vector_storage_local_path+ self.url_and_api_key = url_and_api_key++ def _initialize_vector_storage(+ self,+ collection_name: Optional[str] = None,+ ) -> BaseVectorStorage:+ r"""Sets up and returns a vector storage instance with specified parameters.++ Args:+ collection_name (Optional[str]): Name of the collection in the+ vector storage.++ Returns:+ BaseVectorStorage: Configured vector storage instance.+ """+ if self.storage_type == StorageType.MILVUS:+ if self.url_and_api_key is None:+ raise ValueError(+ "URL and API key required for Milvus storage are not"+ "provided.")+ return MilvusStorage(+ vector_dim=self.embedding_model.get_output_dim(),+ collection_name=collection_name,+ url_and_api_key=self.url_and_api_key)++ if self.storage_type == StorageType.QDRANT:+ return QdrantStorage(+ vector_dim=self.embedding_model.get_output_dim(),+ collection_name=collection_name,+ path=self.vector_storage_local_path,+ url_and_api_key=self.url_and_api_key)++ raise ValueError(+ f"Unsupported vector storage type: {self.storage_type}")++ def _collection_name_generator(self, content_input_path: str) -> str:+ r"""Generates a valid collection name from a given file path or URL.++ Args:+ content_input_path: str. The input URL or file path from which to+ generate the collection name.++ Returns:+ str: A sanitized, valid collection name suitable for use.+ """+ # Check path type+ parsed_url = urlparse(content_input_path)+ self.is_url = all([parsed_url.scheme, parsed_url.netloc])++ # Convert given path into a collection name, ensuring it only+ # contains numbers, letters, and underscores+ if self.is_url:+ # For URLs, remove https://, replace /, and any characters not+ # allowed by Milvus with _+ collection_name = re.sub(+ r'[^0-9a-zA-Z]+', '_',+ content_input_path.replace("https://", ""))+ else:+ # For file paths, get the stem and replace spaces with _, also+ # ensuring only allowed characters are present+ collection_name = re.sub(r'[^0-9a-zA-Z]+', '_',+ Path(content_input_path).stem)++ # Ensure the collection name does not start or end with underscore+ collection_name = collection_name.strip("_")+ # Limit the maximum length of the collection name to 30 characters+ collection_name = collection_name[:30]+ return collection_name++ def _get_file_modified_date_from_file(self,+ content_input_path: str) -> str:+ r"""Retrieves the last modified date and time of a given file. This+ function takes a file path as input and returns the last modified date+ and time of that file.++ Args:+ content_input_path (str): The file path of the content whose+ modified date is to be retrieved.++ Returns:+ str: The last modified time from file.+ """+ mod_time = os.path.getmtime(content_input_path)+ readable_mod_time = datetime.datetime.fromtimestamp(+ mod_time).isoformat(timespec='seconds')+ return readable_mod_time++ def _get_file_modified_date_from_storage(+ self, vector_storage_instance: BaseVectorStorage) -> str:+ r"""Retrieves the last modified date and time of a given file. This+ function takes vector storage instance as input and returns the last+ modified date from the meta data.++ Args:+ vector_storage_instance (BaseVectorStorage): The vector storage+ where modified date is to be retrieved from meta data.++ Returns:+ str: The last modified date from vector storage.+ """++ # Insert any query to get modified date from vector db+ # NOTE: Can be optimized when CAMEL vector storage support+ # direct chunk payload extraction+ query_vector_any = self.embedding_model.embed(obj="any_query")+ query_any = VectorDBQuery(query_vector_any, top_k=1)+ result_any = vector_storage_instance.query(query_any)++ # Extract the file's last modified date from the metadata+ # in the query result+ if result_any[0].record.payload is not None:+ file_modified_date_from_meta = (+ result_any[0].record.payload["metadata"]['last_modified'])+ else:+ raise ValueError(+ "The vector storage exits but the payload is None,"+ "please check the collection")++ return file_modified_date_from_meta++ def run_vector_retriever(+ self, query: str, content_input_paths: Union[str, List[str]],+ top_k: int = DEFAULT_TOP_K_RESULTS,+ similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,+ return_detailed_info: bool = False) -> str:+ r"""Executes the automatic vector retriever process using vector storage.++ Args:+ query (str): Query string for information retriever.+ content_input_paths (Union[str, List[str]]): Paths to local+ files or remote URLs.+ top_k (int, optional): The number of top results to return during+ retrieve. Must be a positive integer. Defaults to+ `DEFAULT_TOP_K_RESULTS`.+ similarity_threshold (float, optional): The similarity threshold+ for filtering results. Defaults to+ `DEFAULT_SIMILARITY_THRESHOLD`.+ return_detailed_info (bool, optional): Whether to return detailed+ information including similarity score, content path and+ metadata. Defaults to False.++ Returns:+ string: By default, returns only the text information. If+ `return_detailed_info` is `True`, return detailed information+ including similarity score, content path and metadata.++ Raises:+ ValueError: If there's an vector storage existing with content+ name in the vector path but the payload is None. If+ `content_input_paths` is empty.+ RuntimeError: If any errors occur during the retrieve process.+ """+ if not content_input_paths:+ raise ValueError("content_input_paths cannot be empty.")++ content_input_paths = [content_input_paths] if isinstance(+ content_input_paths, str) else content_input_paths++ vr = VectorRetriever()++ retrieved_infos = ""+ retrieved_infos_text = ""++ for content_input_path in content_input_paths:+ # Generate a valid collection name+ collection_name = self._collection_name_generator(+ content_input_path)+ try:+ vector_storage_instance = self._initialize_vector_storage(+ collection_name)++ # Check the modified time of the input file path, only works+ # for local path since no standard way for remote url+ file_is_modified = False # initialize with a default value+ if vector_storage_instance.status(+ ).vector_count != 0 and not self.is_url:+ # Get original modified date from file+ modified_date_from_file = \+ self._get_file_modified_date_from_file(+ content_input_path)+ # Get modified date from vector storage+ modified_date_from_storage = \+ self._get_file_modified_date_from_storage(+ vector_storage_instance)+ # Determine if the file has been modified since the last+ # check+ file_is_modified = (modified_date_from_file !=+ modified_date_from_storage)++ if vector_storage_instance.status(+ ).vector_count == 0 or file_is_modified:+ # Clear the vector storage+ vector_storage_instance.clear()+ # Process and store the content to the vector storage+ vr.process(content_input_path, vector_storage_instance)+ # Retrieve info by given query from the vector storage+ retrieved_info = vr.query(query, vector_storage_instance,+ top_k, similarity_threshold)+ # Reorganize the retrieved info with original query+ for info in retrieved_info:+ retrieved_infos += "\n" + str(info)+ retrieved_infos_text += "\n" + str(info['text'])+ output = ("Original Query:" + "\n" + "{" + query + "}" + "\n" ++ "Retrieved Context:" + retrieved_infos)+ output_text = ("Original Query:" + "\n" + "{" + query + "}" ++ "\n" + "Retrieved Context:" ++ retrieved_infos_text)++ except Exception as e:+ raise RuntimeError(+ f"Error in auto vector retriever processing: {str(e)}"+ ) from e++ if return_detailed_info:+ return output+ else:+ return output_textdiff --git a/camel/types/__init__.py b/camel/types/__init__.pyindex 251201855..45eb2c44e 100644--- a/camel/types/__init__.py+++ b/camel/types/__init__.py@@ -19,6 +19,7 @@ from .enums import (OpenAIBackendRole,EmbeddingModelType,VectorDistance,+ StorageType,)from .openai_types import (Choice,@@ -41,6 +42,7 @@ __all__ = ['OpenAIBackendRole','EmbeddingModelType','VectorDistance',+ 'StorageType','Choice','ChatCompletion','ChatCompletionChunk',diff --git a/camel/types/enums.py b/camel/types/enums.pyindex 4a3c7aa51..b905f478d 100644--- a/camel/types/enums.py+++ b/camel/types/enums.py@@ -181,3 +181,8 @@ class OpenAIBackendRole(Enum):class TerminationMode(Enum):ANY = "any"ALL = "all"+++class StorageType(Enum):+ MILVUS = "milvus"+ QDRANT = "qdrant"diff --git a/test/retrievers/test_auto_retriever.py b/test/retrievers/test_auto_retriever.pynew file mode 100644index 000000000..e4cd43b24--- /dev/null+++ b/test/retrievers/test_auto_retriever.py@@ -0,0 +1,90 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+import os+import shutil+from datetime import datetime+from unittest.mock import patch++import pytest++from camel.retrievers import AutoRetriever+from camel.storages import QdrantStorage+from camel.types import StorageType+++def temp_storage_path():+ # Define the path to the temporary storage+ path = 'test/functions/tempory_storage'++ yield path++ # Remove the files created in the temporary storage+ if os.path.exists(path):+ shutil.rmtree(path)+++def auto_retriever(temp_storage_path):+ return AutoRetriever(vector_storage_local_path=temp_storage_path,+ storage_type=StorageType.QDRANT)+++def test__initialize_vector_storage(auto_retriever):+ # with tempfile.TemporaryDirectory() as tmpdir:+ storage_custom = auto_retriever._initialize_vector_storage("collection")+ assert isinstance(storage_custom, QdrantStorage)+++def test_get_file_modified_date_from_file(auto_retriever):+ with patch('os.path.getmtime') as mocked_getmtime:+ mocked_getmtime.return_value = 1234567890+ mod_date = auto_retriever._get_file_modified_date_from_file(+ "/path/to/file")+ expected_date = datetime.fromtimestamp(1234567890).strftime(+ '%Y-%m-%dT%H:%M:%S')+ assert mod_date == expected_date++ with pytest.raises(FileNotFoundError):+ auto_retriever._get_file_modified_date_from_file(+ "/path/to/nonexistent/file")+++def test_run_vector_retriever(auto_retriever):+ # Define mock data for testing+ query_related = "what is camel"+ query_unrealted = "unrelated query"+ content_input_paths = "https://www.camel-ai.org/"+ top_k = 1+ similarity_threshold = 0.75++ # Test with query related to the content in mock data+ result_related = auto_retriever.run_vector_retriever(+ query_related, content_input_paths, top_k, similarity_threshold,+ return_detailed_info=True)++ assert "similarity score" in result_related, \+ "result_related missing 'similarity score'"+ assert "content path" in result_related, \+ "result_related missing 'content path'"+ assert "metadata" in result_related, \+ "result_related missing 'metadata'"+ assert "text" in result_related, \+ "result_related missing 'text'"++ # Test with query unrelated to the content in mock data+ result_unrelated = auto_retriever.run_vector_retriever(+ query_unrealted, content_input_paths, top_k, similarity_threshold)++ assert "No suitable information retrieved from" in result_unrelated