🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
fix: retriever module follow LSP, utils function use `GPT_3_5_TURBO` (#532)
Wendong-Fan344725855d7a
diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.pyindex 0c2d6c9a9..ef7f620a0 100644--- a/camel/retrievers/auto_retriever.py+++ b/camel/retrievers/auto_retriever.py@@ -278,11 +278,18 @@ class AutoRetriever:# Clear the vector storagevector_storage_instance.clear()# Process and store the content to the vector storage- vr.process(content_input_path, vector_storage_instance)+ vr = VectorRetriever(+ storage=vector_storage_instance,+ similarity_threshold=similarity_threshold,+ )+ vr.process(content_input_path)+ else:+ vr = VectorRetriever(+ storage=vector_storage_instance,+ similarity_threshold=similarity_threshold,+ )# Retrieve info by given query from the vector storage- retrieved_info = vr.query(- query, vector_storage_instance, top_k, similarity_threshold- )+ retrieved_info = vr.query(query, top_k)# Reorganize the retrieved info with original queryfor info in retrieved_info:retrieved_infos += "\n" + str(info)diff --git a/camel/retrievers/base.py b/camel/retrievers/base.pyindex eb103a7c9..1f7f18652 100644--- a/camel/retrievers/base.py+++ b/camel/retrievers/base.py@@ -12,53 +12,58 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========from abc import ABC, abstractmethod-from typing import Any, Dict, List+from typing import Any, CallableDEFAULT_TOP_K_RESULTS = 1-class BaseRetriever(ABC):- r"""Abstract base class for implementing various types of information- retrievers.+def _query_unimplemented(self, *input: Any) -> None:+ r"""Defines the query behavior performed at every call.++ Query the results. Subclasses should implement this+ method according to their specific needs.++ It should be overridden by all subclasses.++ .. note::+ Although the recipe for forward pass needs to be defined within+ this function, one should call the :class:`BaseRetriever` instance+ afterwards instead of this since the former takes care of running the+ registered hooks while the latter silently ignores them."""+ raise NotImplementedError(+ f"Retriever [{type(self).__name__}] is missing the required \"query\" function"+ )- @abstractmethod- def __init__(self) -> None:- pass- @abstractmethod- def process(- self,- content_input_path: str,- chunk_type: str = "chunk_by_title",- **kwargs: Any,- ) -> None:- r"""Processes content from a file or URL, divides it into chunks by+def _process_unimplemented(self, *input: Any) -> None:+ r"""Defines the process behavior performed at every call.++ Processes content from a file or URL, divides it into chunks byusing `Unstructured IO`,then stored internally. This method must becalled before executing queries with the retriever.- Args:- content_input_path (str): File path or URL of the content to be- processed.- chunk_type (str): Type of chunking going to apply. Defaults to- "chunk_by_title".- **kwargs (Any): Additional keyword arguments for content parsing.- """- pass+ Should be overridden by all subclasses.- @abstractmethod- def query(- self, query: str, top_k: int = DEFAULT_TOP_K_RESULTS, **kwargs: Any- ) -> List[Dict[str, Any]]:- r"""Query the results. Subclasses should implement this- method according to their specific needs.+ .. note::+ Although the recipe for forward pass needs to be defined within+ this function, one should call the :class:`BaseRetriever` instance+ afterwards instead of this since the former takes care of running the+ registered hooks while the latter silently ignores them.+ """+ raise NotImplementedError(+ f"Retriever [{type(self).__name__}] is missing the required \"process\" function"+ )+++class BaseRetriever(ABC):+ r"""Abstract base class for implementing various types of information+ retrievers.+ """- Args:- query (str): Query string for information retriever.- top_k (int, optional): The number of top results to return during- retriever. Must be a positive integer. Defaults to- `DEFAULT_TOP_K_RESULTS`.- **kwargs (Any): Flexible keyword arguments for additional- parameters, like `similarity_threshold`.- """+ @abstractmethod+ def __init__(self) -> None:pass++ process: Callable[..., Any] = _process_unimplemented+ query: Callable[..., Any] = _query_unimplementeddiff --git a/camel/retrievers/bm25_retriever.py b/camel/retrievers/bm25_retriever.pyindex 246be640a..e48b6caf0 100644--- a/camel/retrievers/bm25_retriever.py+++ b/camel/retrievers/bm25_retriever.py@@ -33,8 +33,8 @@ class BM25Retriever(BaseRetriever):calculating document scores.content_input_path (str): The path to the content that has beenprocessed and stored.- chunks (List[Any]): A list of document chunks processed from the- input content.+ unstructured_modules (UnstructuredIO): A module for parsing files and+ URLs and chunking content based on specified parameters.References:https://github.com/dorianbrown/rank_bm25@@ -47,13 +47,12 @@ class BM25Retriever(BaseRetriever):from rank_bm25 import BM25Okapiexcept ImportError as e:raise ImportError(- "Package `rank_bm25` not installed, install by running"- " 'pip install rank_bm25'"+ "Package `rank_bm25` not installed, install by running 'pip install rank_bm25'") from eself.bm25: BM25Okapi = Noneself.content_input_path: str = ""- self.chunks: List[Any] = []+ self.unstructured_modules: UnstructuredIO = UnstructuredIO()def process(self,@@ -76,11 +75,10 @@ class BM25Retriever(BaseRetriever):# Load and preprocess documentsself.content_input_path = content_input_path- unstructured_modules = UnstructuredIO()- elements = unstructured_modules.parse_file_or_url(+ elements = self.unstructured_modules.parse_file_or_url(content_input_path, **kwargs)- self.chunks = unstructured_modules.chunk_elements(+ self.chunks = self.unstructured_modules.chunk_elements(chunk_type=chunk_type, elements=elements)@@ -88,7 +86,7 @@ class BM25Retriever(BaseRetriever):tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]self.bm25 = BM25Okapi(tokenized_corpus)- def query( # type: ignore[override]+ def query(self,query: str,top_k: int = DEFAULT_TOP_K_RESULTS,@@ -106,22 +104,15 @@ class BM25Retriever(BaseRetriever):Raises:ValueError: If `top_k` is less than or equal to 0, if the BM25- model has not been initialized by calling `process_and_store`+ model has not been initialized by calling `process`first.-- Note:- `storage` and `kwargs` parameters are included to maintain- compatibility with the `BaseRetriever` interface but are not used- in this implementation."""if top_k <= 0:raise ValueError("top_k must be a positive integer.")-- if self.bm25 is None:+ if self.bm25 is None or not self.chunks:raise ValueError(- "BM25 model is not initialized. Call `process_and_store`"- " first."+ "BM25 model is not initialized. Call `process` first.")# Preprocess query similarly to how documents were processeddiff --git a/camel/retrievers/vector_retriever.py b/camel/retrievers/vector_retriever.pyindex f94a94d79..2c74580b7 100644--- a/camel/retrievers/vector_retriever.py+++ b/camel/retrievers/vector_retriever.py@@ -16,7 +16,12 @@ from typing import Any, Dict, List, Optionalfrom camel.embeddings import BaseEmbedding, OpenAIEmbeddingfrom camel.functions import UnstructuredIOfrom camel.retrievers.base import BaseRetriever-from camel.storages import BaseVectorStorage, VectorDBQuery, VectorRecord+from camel.storages import (+ BaseVectorStorage,+ QdrantStorage,+ VectorDBQuery,+ VectorRecord,+)DEFAULT_TOP_K_RESULTS = 1DEFAULT_SIMILARITY_THRESHOLD = 0.75@@ -32,21 +37,41 @@ class VectorRetriever(BaseRetriever):Attributes:embedding_model (BaseEmbedding): Embedding model used to generatevector embeddings.+ storage (BaseVectorStorage): Vector storage to query.+ similarity_threshold (float, optional): The similarity threshold+ for filtering results. Defaults to `DEFAULT_SIMILARITY_THRESHOLD`.+ unstructured_modules (UnstructuredIO): A module for parsing files and+ URLs and chunking content based on specified parameters."""- def __init__(self, embedding_model: Optional[BaseEmbedding] = None) -> None:+ def __init__(+ self,+ similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,+ embedding_model: Optional[BaseEmbedding] = None,+ storage: Optional[BaseVectorStorage] = None,+ ) -> None:r"""Initializes the retriever class with an optional embedding model.Args:+ similarity_threshold (float, optional): The similarity threshold+ for filtering results. Defaults to+ `DEFAULT_SIMILARITY_THRESHOLD`.embedding_model (Optional[BaseEmbedding]): The embedding modelinstance. Defaults to `OpenAIEmbedding` if not provided.+ storage (BaseVectorStorage): Vector storage to query."""self.embedding_model = embedding_model or OpenAIEmbedding()+ self.storage = (+ storage+ if storage is not None+ else QdrantStorage(vector_dim=self.embedding_model.get_output_dim())+ )+ self.similarity_threshold = similarity_threshold+ self.unstructured_modules: UnstructuredIO = UnstructuredIO()- def process( # type: ignore[override]+ def process(self,content_input_path: str,- storage: BaseVectorStorage,chunk_type: str = "chunk_by_title",**kwargs: Any,) -> None:@@ -59,12 +84,13 @@ class VectorRetriever(BaseRetriever):processed.chunk_type (str): Type of chunking going to apply. Defaults to"chunk_by_title".- **kwargs (Any): Additional keyword arguments for elements chunking.+ **kwargs (Any): Additional keyword arguments for content parsing."""- unstructured_modules = UnstructuredIO()- elements = unstructured_modules.parse_file_or_url(content_input_path)- chunks = unstructured_modules.chunk_elements(- chunk_type=chunk_type, elements=elements, **kwargs+ elements = self.unstructured_modules.parse_file_or_url(+ content_input_path, **kwargs+ )+ chunks = self.unstructured_modules.chunk_elements(+ chunk_type=chunk_type, elements=elements)# Iterate to process and store embeddings, set batch of 50for i in range(0, len(chunks), 50):@@ -90,28 +116,20 @@ class VectorRetriever(BaseRetriever):VectorRecord(vector=vector, payload=combined_dict))- storage.add(records=records)+ self.storage.add(records=records)- def query( # type: ignore[override]+ def query(self,query: str,- storage: BaseVectorStorage,top_k: int = DEFAULT_TOP_K_RESULTS,- similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,- **kwargs: Any,) -> List[Dict[str, Any]]:r"""Executes a query in vector storage and compiles the retrievedresults into a dictionary.Args:query (str): Query string for information retriever.- storage (BaseVectorStorage): Vector storage to query.top_k (int, optional): The number of top results to return duringretriever. Must be a positive integer. Defaults to 1.- similarity_threshold (float, optional): The similarity threshold- for filtering results. Defaults to 0.75.- **kwargs (Any): Additional keyword arguments for vector storage- query.Returns:List[Dict[str, Any]]: Concatenated list of the query results.@@ -125,23 +143,22 @@ class VectorRetriever(BaseRetriever):raise ValueError("top_k must be a positive integer.")# Load the storage incase it's hosted remote- storage.load()+ self.storage.load()query_vector = self.embedding_model.embed(obj=query)db_query = VectorDBQuery(query_vector=query_vector, top_k=top_k)- query_results = storage.query(query=db_query, **kwargs)+ query_results = self.storage.query(query=db_query)if query_results[0].record.payload is None:raise ValueError(- "Payload of vector storage is None, please check"- " the collection."+ "Payload of vector storage is None, please check the collection.")# format the resultsformatted_results = []for result in query_results:if (- result.similarity >= similarity_threshold+ result.similarity >= self.similarity_thresholdand result.record.payload is not None):result_dict = {@@ -160,7 +177,7 @@ class VectorRetriever(BaseRetriever):return [{'text': f"""No suitable information retrieved from {content_path} \- with similarity_threshold = {similarity_threshold}."""+ with similarity_threshold = {self.similarity_threshold}."""}]return formatted_resultsdiff --git a/camel/storages/vectordb_storages/qdrant.py b/camel/storages/vectordb_storages/qdrant.pyindex 31bad5952..7d67abe2e 100644--- a/camel/storages/vectordb_storages/qdrant.py+++ b/camel/storages/vectordb_storages/qdrant.py@@ -185,7 +185,9 @@ class QdrantStorage(BaseVectorStorage):VectorDistance.COSINE: Distance.COSINE,VectorDistance.EUCLIDEAN: Distance.EUCLID,}- self._client.recreate_collection(+ # Since `recreate_collection` method will be removed in the future+ # by Qdrant, `create_collection` is recommended instead.+ self._client.create_collection(collection_name=collection_name,vectors_config=VectorParams(size=size,diff --git a/camel/utils/commons.py b/camel/utils/commons.pyindex 24e716a28..ef5d40bea 100644--- a/camel/utils/commons.py+++ b/camel/utils/commons.py@@ -52,7 +52,7 @@ def get_lazy_imported_functions_module():def get_lazy_imported_types_module():from camel.types import ModelType- return ModelType.GPT_4_TURBO+ return ModelType.GPT_3_5_TURBOdef api_key_required(func: F) -> F:diff --git a/test/retrievers/test_bm25_retriever.py b/test/retrievers/test_bm25_retriever.pyindex d20c24d12..10fbafb52 100644--- a/test/retrievers/test_bm25_retriever.py+++ b/test/retrievers/test_bm25_retriever.py@@ -19,28 +19,35 @@ import pytestfrom camel.retrievers import BM25Retriever+def mock_unstructured_modules():+ with patch('camel.retrievers.bm25_retriever.UnstructuredIO') as mock:+ yield mock++def test_bm25retriever_initialization():retriever = BM25Retriever()assert retriever.bm25 is Noneassert retriever.content_input_path == ""- assert retriever.chunks == []-@patch('camel.loaders.UnstructuredIO')def test_process(mock_unstructured_modules):- mock_unstructured_modules.return_value.parse_file_or_url.return_value = [- 'Your parsed content'- ]- mock_unstructured_modules.return_value.chunk_elements.return_value = [- 'Chunk 1',- 'Chunk 2',- ]+ mock_instance = mock_unstructured_modules.return_value- retriever = BM25Retriever()- retriever.process('https://www.camel-ai.org/')+ # Create a mock chunk with metadata+ mock_chunk = MagicMock()+ mock_chunk.metadata.to_dict.return_value = {'mock_key': 'mock_value'}++ # Setup mock behavior+ mock_instance.parse_file_or_url.return_value = ["mock_element"]+ mock_instance.chunk_elements.return_value = [mock_chunk]++ bm25_retriever = BM25Retriever()+ bm25_retriever.process(content_input_path="mock_path")- assert retriever.content_input_path == 'https://www.camel-ai.org/'- assert len(retriever.chunks) == 4+ # Assert that methods are called as expected+ mock_instance.parse_file_or_url.assert_called_once_with("mock_path")+ mock_instance.chunk_elements.assert_called_once()@patch('camel.retrievers.BM25Retriever')diff --git a/test/retrievers/test_vector_retriever.py b/test/retrievers/test_vector_retriever.pyindex a8eadb17e..49a31b9ce 100644--- a/test/retrievers/test_vector_retriever.py+++ b/test/retrievers/test_vector_retriever.py@@ -35,8 +35,16 @@ def mock_vector_storage():@pytest.fixture-def vector_retriever(mock_embedding_model):- return VectorRetriever(embedding_model=mock_embedding_model)+def vector_retriever(mock_embedding_model, mock_vector_storage):+ return VectorRetriever(+ embedding_model=mock_embedding_model, storage=mock_vector_storage+ )+++def mock_unstructured_modules():+ with patch('camel.retrievers.vector_retriever.UnstructuredIO') as mock:+ yield mock# Test initialization with a custom embedding model@@ -53,39 +61,36 @@ def test_initialization_with_default_embedding():# Test process method-@patch('camel.retrievers.vector_retriever.UnstructuredIO')-def test_process(- mock_unstructured_modules, vector_retriever, mock_vector_storage-):+def test_process(mock_unstructured_modules):+ mock_instance = mock_unstructured_modules.return_value+# Create a mock chunk with metadatamock_chunk = MagicMock()mock_chunk.metadata.to_dict.return_value = {'mock_key': 'mock_value'}# Setup mock behavior- mock_unstructured_instance = mock_unstructured_modules.return_value- mock_unstructured_instance.parse_file_or_url.return_value = ["mock_element"]- mock_unstructured_instance.chunk_elements.return_value = [mock_chunk]+ mock_instance.parse_file_or_url.return_value = ["mock_element"]+ mock_instance.chunk_elements.return_value = [mock_chunk]- vector_retriever.embedding_model.embed_list.return_value = [[0.1, 0.2, 0.3]]+ vector_retriever = VectorRetriever()- vector_retriever.process("mock_path", mock_vector_storage)+ vector_retriever.process(content_input_path="mock_path")# Assert that methods are called as expected- mock_unstructured_instance.parse_file_or_url.assert_called_once_with(- "mock_path"- )- mock_unstructured_instance.chunk_elements.assert_called_once()- mock_vector_storage.add.assert_called_once()+ mock_instance.parse_file_or_url.assert_called_once_with("mock_path")+ mock_instance.chunk_elements.assert_called_once()# Test query-def test_query(vector_retriever, mock_vector_storage):+def test_query(vector_retriever):+ query = "test query"+ top_k = 1# Setup mock behavior for vector storage query- mock_vector_storage.query.return_value = [- Mock(similarity=0.8, record=Mock(payload={"text": "mock_result"}))+ vector_retriever.storage.load = Mock()+ vector_retriever.storage.query.return_value = [+ Mock(similarity=0.8, record=Mock(payload={"text1": "mock_result1"}))]- result = vector_retriever.query("mock_query", mock_vector_storage)-- # Assert that the result is as expected- assert any(d.get('text') == 'mock_result' for d in result)+ results = vector_retriever.query(query, top_k=top_k)+ assert len(results) == 1+ assert results[0]['similarity score'] == '0.8'