Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: auto retriever (#469)

Wendong-Fanc276ab3e1fdb
diff --git a/camel/retrievers/__init__.py b/camel/retrievers/__init__.py
index e35c092e7..03dea03ce 100644
--- a/camel/retrievers/__init__.py
+++ b/camel/retrievers/__init__.py
@@ -13,8 +13,10 @@
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
from .base import BaseRetriever
from .vector_retriever import VectorRetriever
+from .auto_retriever import AutoRetriever
__all__ = [
'BaseRetriever',
'VectorRetriever',
+ 'AutoRetriever',
]
diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.py
new file mode 100644
index 000000000..cd383837b
--- /dev/null
+++ b/camel/retrievers/auto_retriever.py
@@ -0,0 +1,277 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import datetime
+import os
+import re
+from pathlib import Path
+from typing import List, Optional, Tuple, Union
+from urllib.parse import urlparse
+
+from camel.embeddings import BaseEmbedding, OpenAIEmbedding
+from camel.retrievers import VectorRetriever
+from camel.storages import (
+ BaseVectorStorage,
+ MilvusStorage,
+ QdrantStorage,
+ VectorDBQuery,
+)
+from camel.types import StorageType
+
+DEFAULT_TOP_K_RESULTS = 1
+DEFAULT_SIMILARITY_THRESHOLD = 0.75
+
+
+class AutoRetriever():
+ r""" Facilitates the automatic retrieval of information using a
+ query-based approach with pre-defined elements.
+
+ Attributes:
+ url_and_api_key (Optional[Tuple[str, str]]): URL and API key for
+ accessing the vector storage remotely.
+ vector_storage_local_path (Optional[str]): Local path for vector
+ storage, if applicable.
+ storage_type (Optional[StorageType]): The type of vector storage to
+ use. Defaults to `StorageType.MILVUS`.
+ embedding_model (Optional[BaseEmbedding]): Model used for embedding
+ queries and documents. Defaults to `OpenAIEmbedding()`.
+ """
+
+ def __init__(
+ self,
+ url_and_api_key: Optional[Tuple[str, str]] = None,
+ vector_storage_local_path: Optional[str] = None,
+ storage_type: Optional[StorageType] = None,
+ embedding_model: Optional[BaseEmbedding] = None,
+ ):
+
+ self.storage_type = storage_type or StorageType.MILVUS
+ self.embedding_model = embedding_model or OpenAIEmbedding()
+ self.vector_storage_local_path = vector_storage_local_path
+ self.url_and_api_key = url_and_api_key
+
+ def _initialize_vector_storage(
+ self,
+ collection_name: Optional[str] = None,
+ ) -> BaseVectorStorage:
+ r"""Sets up and returns a vector storage instance with specified parameters.
+
+ Args:
+ collection_name (Optional[str]): Name of the collection in the
+ vector storage.
+
+ Returns:
+ BaseVectorStorage: Configured vector storage instance.
+ """
+ if self.storage_type == StorageType.MILVUS:
+ if self.url_and_api_key is None:
+ raise ValueError(
+ "URL and API key required for Milvus storage are not"
+ "provided.")
+ return MilvusStorage(
+ vector_dim=self.embedding_model.get_output_dim(),
+ collection_name=collection_name,
+ url_and_api_key=self.url_and_api_key)
+
+ if self.storage_type == StorageType.QDRANT:
+ return QdrantStorage(
+ vector_dim=self.embedding_model.get_output_dim(),
+ collection_name=collection_name,
+ path=self.vector_storage_local_path,
+ url_and_api_key=self.url_and_api_key)
+
+ raise ValueError(
+ f"Unsupported vector storage type: {self.storage_type}")
+
+ def _collection_name_generator(self, content_input_path: str) -> str:
+ r"""Generates a valid collection name from a given file path or URL.
+
+ Args:
+ content_input_path: str. The input URL or file path from which to
+ generate the collection name.
+
+ Returns:
+ str: A sanitized, valid collection name suitable for use.
+ """
+ # Check path type
+ parsed_url = urlparse(content_input_path)
+ self.is_url = all([parsed_url.scheme, parsed_url.netloc])
+
+ # Convert given path into a collection name, ensuring it only
+ # contains numbers, letters, and underscores
+ if self.is_url:
+ # For URLs, remove https://, replace /, and any characters not
+ # allowed by Milvus with _
+ collection_name = re.sub(
+ r'[^0-9a-zA-Z]+', '_',
+ content_input_path.replace("https://", ""))
+ else:
+ # For file paths, get the stem and replace spaces with _, also
+ # ensuring only allowed characters are present
+ collection_name = re.sub(r'[^0-9a-zA-Z]+', '_',
+ Path(content_input_path).stem)
+
+ # Ensure the collection name does not start or end with underscore
+ collection_name = collection_name.strip("_")
+ # Limit the maximum length of the collection name to 30 characters
+ collection_name = collection_name[:30]
+ return collection_name
+
+ def _get_file_modified_date_from_file(self,
+ content_input_path: str) -> str:
+ r"""Retrieves the last modified date and time of a given file. This
+ function takes a file path as input and returns the last modified date
+ and time of that file.
+
+ Args:
+ content_input_path (str): The file path of the content whose
+ modified date is to be retrieved.
+
+ Returns:
+ str: The last modified time from file.
+ """
+ mod_time = os.path.getmtime(content_input_path)
+ readable_mod_time = datetime.datetime.fromtimestamp(
+ mod_time).isoformat(timespec='seconds')
+ return readable_mod_time
+
+ def _get_file_modified_date_from_storage(
+ self, vector_storage_instance: BaseVectorStorage) -> str:
+ r"""Retrieves the last modified date and time of a given file. This
+ function takes vector storage instance as input and returns the last
+ modified date from the meta data.
+
+ Args:
+ vector_storage_instance (BaseVectorStorage): The vector storage
+ where modified date is to be retrieved from meta data.
+
+ Returns:
+ str: The last modified date from vector storage.
+ """
+
+ # Insert any query to get modified date from vector db
+ # NOTE: Can be optimized when CAMEL vector storage support
+ # direct chunk payload extraction
+ query_vector_any = self.embedding_model.embed(obj="any_query")
+ query_any = VectorDBQuery(query_vector_any, top_k=1)
+ result_any = vector_storage_instance.query(query_any)
+
+ # Extract the file's last modified date from the metadata
+ # in the query result
+ if result_any[0].record.payload is not None:
+ file_modified_date_from_meta = (
+ result_any[0].record.payload["metadata"]['last_modified'])
+ else:
+ raise ValueError(
+ "The vector storage exits but the payload is None,"
+ "please check the collection")
+
+ return file_modified_date_from_meta
+
+ def run_vector_retriever(
+ self, query: str, content_input_paths: Union[str, List[str]],
+ top_k: int = DEFAULT_TOP_K_RESULTS,
+ similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+ return_detailed_info: bool = False) -> str:
+ r"""Executes the automatic vector retriever process using vector storage.
+
+ Args:
+ query (str): Query string for information retriever.
+ content_input_paths (Union[str, List[str]]): Paths to local
+ files or remote URLs.
+ top_k (int, optional): The number of top results to return during
+ retrieve. Must be a positive integer. Defaults to
+ `DEFAULT_TOP_K_RESULTS`.
+ similarity_threshold (float, optional): The similarity threshold
+ for filtering results. Defaults to
+ `DEFAULT_SIMILARITY_THRESHOLD`.
+ return_detailed_info (bool, optional): Whether to return detailed
+ information including similarity score, content path and
+ metadata. Defaults to False.
+
+ Returns:
+ string: By default, returns only the text information. If
+ `return_detailed_info` is `True`, return detailed information
+ including similarity score, content path and metadata.
+
+ Raises:
+ ValueError: If there's an vector storage existing with content
+ name in the vector path but the payload is None. If
+ `content_input_paths` is empty.
+ RuntimeError: If any errors occur during the retrieve process.
+ """
+ if not content_input_paths:
+ raise ValueError("content_input_paths cannot be empty.")
+
+ content_input_paths = [content_input_paths] if isinstance(
+ content_input_paths, str) else content_input_paths
+
+ vr = VectorRetriever()
+
+ retrieved_infos = ""
+ retrieved_infos_text = ""
+
+ for content_input_path in content_input_paths:
+ # Generate a valid collection name
+ collection_name = self._collection_name_generator(
+ content_input_path)
+ try:
+ vector_storage_instance = self._initialize_vector_storage(
+ collection_name)
+
+ # Check the modified time of the input file path, only works
+ # for local path since no standard way for remote url
+ file_is_modified = False # initialize with a default value
+ if vector_storage_instance.status(
+ ).vector_count != 0 and not self.is_url:
+ # Get original modified date from file
+ modified_date_from_file = \
+ self._get_file_modified_date_from_file(
+ content_input_path)
+ # Get modified date from vector storage
+ modified_date_from_storage = \
+ self._get_file_modified_date_from_storage(
+ vector_storage_instance)
+ # Determine if the file has been modified since the last
+ # check
+ file_is_modified = (modified_date_from_file !=
+ modified_date_from_storage)
+
+ if vector_storage_instance.status(
+ ).vector_count == 0 or file_is_modified:
+ # Clear the vector storage
+ vector_storage_instance.clear()
+ # Process and store the content to the vector storage
+ vr.process(content_input_path, vector_storage_instance)
+ # Retrieve info by given query from the vector storage
+ retrieved_info = vr.query(query, vector_storage_instance,
+ top_k, similarity_threshold)
+ # Reorganize the retrieved info with original query
+ for info in retrieved_info:
+ retrieved_infos += "\n" + str(info)
+ retrieved_infos_text += "\n" + str(info['text'])
+ output = ("Original Query:" + "\n" + "{" + query + "}" + "\n" +
+ "Retrieved Context:" + retrieved_infos)
+ output_text = ("Original Query:" + "\n" + "{" + query + "}" +
+ "\n" + "Retrieved Context:" +
+ retrieved_infos_text)
+
+ except Exception as e:
+ raise RuntimeError(
+ f"Error in auto vector retriever processing: {str(e)}"
+ ) from e
+
+ if return_detailed_info:
+ return output
+ else:
+ return output_text
diff --git a/camel/types/__init__.py b/camel/types/__init__.py
index 251201855..45eb2c44e 100644
--- a/camel/types/__init__.py
+++ b/camel/types/__init__.py
@@ -19,6 +19,7 @@ from .enums import (
OpenAIBackendRole,
EmbeddingModelType,
VectorDistance,
+ StorageType,
)
from .openai_types import (
Choice,
@@ -41,6 +42,7 @@ __all__ = [
'OpenAIBackendRole',
'EmbeddingModelType',
'VectorDistance',
+ 'StorageType',
'Choice',
'ChatCompletion',
'ChatCompletionChunk',
diff --git a/camel/types/enums.py b/camel/types/enums.py
index 4a3c7aa51..b905f478d 100644
--- a/camel/types/enums.py
+++ b/camel/types/enums.py
@@ -181,3 +181,8 @@ class OpenAIBackendRole(Enum):
class TerminationMode(Enum):
ANY = "any"
ALL = "all"
+
+
+class StorageType(Enum):
+ MILVUS = "milvus"
+ QDRANT = "qdrant"
diff --git a/test/retrievers/test_auto_retriever.py b/test/retrievers/test_auto_retriever.py
new file mode 100644
index 000000000..e4cd43b24
--- /dev/null
+++ b/test/retrievers/test_auto_retriever.py
@@ -0,0 +1,90 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+import shutil
+from datetime import datetime
+from unittest.mock import patch
+
+import pytest
+
+from camel.retrievers import AutoRetriever
+from camel.storages import QdrantStorage
+from camel.types import StorageType
+
+
+def temp_storage_path():
+ # Define the path to the temporary storage
+ path = 'test/functions/tempory_storage'
+
+ yield path
+
+ # Remove the files created in the temporary storage
+ if os.path.exists(path):
+ shutil.rmtree(path)
+
+
+def auto_retriever(temp_storage_path):
+ return AutoRetriever(vector_storage_local_path=temp_storage_path,
+ storage_type=StorageType.QDRANT)
+
+
+def test__initialize_vector_storage(auto_retriever):
+ # with tempfile.TemporaryDirectory() as tmpdir:
+ storage_custom = auto_retriever._initialize_vector_storage("collection")
+ assert isinstance(storage_custom, QdrantStorage)
+
+
+def test_get_file_modified_date_from_file(auto_retriever):
+ with patch('os.path.getmtime') as mocked_getmtime:
+ mocked_getmtime.return_value = 1234567890
+ mod_date = auto_retriever._get_file_modified_date_from_file(
+ "/path/to/file")
+ expected_date = datetime.fromtimestamp(1234567890).strftime(
+ '%Y-%m-%dT%H:%M:%S')
+ assert mod_date == expected_date
+
+ with pytest.raises(FileNotFoundError):
+ auto_retriever._get_file_modified_date_from_file(
+ "/path/to/nonexistent/file")
+
+
+def test_run_vector_retriever(auto_retriever):
+ # Define mock data for testing
+ query_related = "what is camel"
+ query_unrealted = "unrelated query"
+ content_input_paths = "https://www.camel-ai.org/"
+ top_k = 1
+ similarity_threshold = 0.75
+
+ # Test with query related to the content in mock data
+ result_related = auto_retriever.run_vector_retriever(
+ query_related, content_input_paths, top_k, similarity_threshold,
+ return_detailed_info=True)
+
+ assert "similarity score" in result_related, \
+ "result_related missing 'similarity score'"
+ assert "content path" in result_related, \
+ "result_related missing 'content path'"
+ assert "metadata" in result_related, \
+ "result_related missing 'metadata'"
+ assert "text" in result_related, \
+ "result_related missing 'text'"
+
+ # Test with query unrelated to the content in mock data
+ result_unrelated = auto_retriever.run_vector_retriever(
+ query_unrealted, content_input_paths, top_k, similarity_threshold)
+
+ assert "No suitable information retrieved from" in result_unrelated