🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Change auto_retriever output format (#827)
Asher-hssb874c63cda18
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.ymlindex 3a469e450..4ec019f93 100644--- a/.github/ISSUE_TEMPLATE/bug_report.yml+++ b/.github/ISSUE_TEMPLATE/bug_report.yml@@ -27,7 +27,7 @@ body:attributes:label: What version of camel are you using?description: Run command `python3 -c 'print(__import__("camel").__version__)'` in your shell and paste the output here.- placeholder: E.g., 0.1.6.4+ placeholder: E.g., 0.1.6.5validations:required: truediff --git a/README.md b/README.mdindex a7fe1a7a6..8a213b2fd 100644--- a/README.md+++ b/README.md@@ -110,7 +110,7 @@ conda create --name camel python=3.9conda activate camel# Clone github repo-git clone -b v0.1.6.4 https://github.com/camel-ai/camel.git+git clone -b v0.1.6.5 https://github.com/camel-ai/camel.git# Change directory into project directorycd cameldiff --git a/camel/__init__.py b/camel/__init__.pyindex d07c9c115..c49f2ba06 100644--- a/camel/__init__.py+++ b/camel/__init__.py@@ -12,7 +12,7 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========-__version__ = '0.1.6.4'+__version__ = '0.1.6.5'__all__ = ['__version__',diff --git a/camel/configs/gemini_config.py b/camel/configs/gemini_config.pyindex 67702080f..959c5baa0 100644--- a/camel/configs/gemini_config.py+++ b/camel/configs/gemini_config.py@@ -81,7 +81,6 @@ class GeminiConfig(BaseConfig):response_mime_type: Optional[str] = Noneresponse_schema: Optional[Any] = Nonesafety_settings: Optional[Any] = None- tools: Optional[Any] = Nonetool_config: Optional[Any] = Nonerequest_options: Optional[Any] = Nonediff --git a/camel/configs/groq_config.py b/camel/configs/groq_config.pyindex 5c945e8c8..fe3f2f90a 100644--- a/camel/configs/groq_config.py+++ b/camel/configs/groq_config.py@@ -99,7 +99,7 @@ class GroqConfig(BaseConfig):response_format: Union[dict, NotGiven] = NOT_GIVENfrequency_penalty: float = 0.0user: str = ""- tool_choice: Optional[Union[dict[str, str], str]] = "none"+ tool_choice: Optional[Union[dict[str, str], str]] = "auto"GROQ_API_PARAMS = {param for param in GroqConfig.model_fields.keys()}diff --git a/camel/configs/mistral_config.py b/camel/configs/mistral_config.pyindex a3dbcdac4..4b248e680 100644--- a/camel/configs/mistral_config.py+++ b/camel/configs/mistral_config.py@@ -74,7 +74,7 @@ class MistralConfig(BaseConfig):if not isinstance(response_format, ResponseFormat):raise ValueError(f"The tool {response_format} should be an instance "- "of `mistralai.models.chat_completion.ResponseFormat`."+ "of `mistralai.models.ResponseFormat`.")return response_formatdiff --git a/camel/loaders/unstructured_io.py b/camel/loaders/unstructured_io.pyindex 2e03545b4..e9a864e48 100644--- a/camel/loaders/unstructured_io.py+++ b/camel/loaders/unstructured_io.py@@ -12,6 +12,7 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========import uuid+import warningsfrom typing import (Any,Dict,@@ -91,7 +92,7 @@ class UnstructuredIO:def parse_file_or_url(input_path: str,**kwargs: Any,- ) -> List[Element]:+ ) -> Union[List[Element], None]:r"""Loads a file or a URL and parses its contents into elements.Args:@@ -99,12 +100,12 @@ class UnstructuredIO:**kwargs: Extra kwargs passed to the partition function.Returns:- List[Element]: List of elements after parsing the file or URL.+ Union[List[Element],None]: List of elements after parsing the file+ or URL if success.Raises:FileNotFoundError: If the file does not exist at the pathspecified.- Exception: For any other issues during file or URL parsing.Notes:Available document types:@@ -128,8 +129,9 @@ class UnstructuredIO:try:elements = partition_html(url=input_path, **kwargs)return elements- except Exception as e:- raise Exception("Failed to parse the URL.") from e+ except Exception:+ warnings.warn(f"Failed to parse the URL: {input_path}")+ return Noneelse:# Handling file@@ -146,10 +148,9 @@ class UnstructuredIO:with open(input_path, "rb") as f:elements = partition(file=f, **kwargs)return elements- except Exception as e:- raise Exception(- "Failed to parse the unstructured file."- ) from e+ except Exception:+ warnings.warn(f"Failed to partition the file: {input_path}")+ return None@staticmethoddef clean_text_data(diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.pyindex 10d55d6c1..b84679f7f 100644--- a/camel/retrievers/auto_retriever.py+++ b/camel/retrievers/auto_retriever.py@@ -15,7 +15,7 @@ import datetimeimport osimport refrom pathlib import Path-from typing import List, Optional, Tuple, Union+from typing import Collection, List, Optional, Sequence, Tuple, Unionfrom urllib.parse import urlparsefrom camel.embeddings import BaseEmbedding, OpenAIEmbedding@@ -197,7 +197,7 @@ class AutoRetriever:top_k: int = DEFAULT_TOP_K_RESULTS,similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,return_detailed_info: bool = False,- ) -> str:+ ) -> dict[str, Sequence[Collection[str]]]:r"""Executes the automatic vector retriever process using vectorstorage.@@ -216,9 +216,10 @@ class AutoRetriever:metadata. Defaults to `False`.Returns:- string: By default, returns only the text information. If- `return_detailed_info` is `True`, return detailed information- including similarity score, content path and metadata.+ dict[str, Sequence[Collection[str]]]: By default, returns+ only the text information. If `return_detailed_info` is+ `True`, return detailed information including similarity+ score, content path and metadata.Raises:ValueError: If there's an vector storage existing with content@@ -308,20 +309,17 @@ class AutoRetriever:# Select the 'top_k' resultsall_retrieved_info = all_retrieved_info_sorted[:top_k]- retrieved_infos = "\n".join(str(info) for info in all_retrieved_info)- retrieved_infos_text = "\n".join(- info['text'] for info in all_retrieved_info if 'text' in info- )+ text_retrieved_info = [item['text'] for item in all_retrieved_info]- detailed_info = (- f"Original Query:\n{{ {query} }}\n"- f"Retrieved Context:\n{retrieved_infos}"- )+ detailed_info = {+ "Original Query": query,+ "Retrieved Context": all_retrieved_info,+ }- text_info = (- f"Original Query:\n{{ {query} }}\n"- f"Retrieved Context:\n{retrieved_infos_text}"- )+ text_info = {+ "Original Query": query,+ "Retrieved Context": text_retrieved_info,+ }if return_detailed_info:return detailed_infodiff --git a/camel/retrievers/bm25_retriever.py b/camel/retrievers/bm25_retriever.pyindex 6a31024bb..4d085b10c 100644--- a/camel/retrievers/bm25_retriever.py+++ b/camel/retrievers/bm25_retriever.py@@ -74,13 +74,16 @@ class BM25Retriever(BaseRetriever):elements = self.unstructured_modules.parse_file_or_url(content_input_path, **kwargs)- self.chunks = self.unstructured_modules.chunk_elements(- chunk_type=chunk_type, elements=elements- )+ if elements:+ self.chunks = self.unstructured_modules.chunk_elements(+ chunk_type=chunk_type, elements=elements+ )- # Convert chunks to a list of strings for tokenization- tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]- self.bm25 = BM25Okapi(tokenized_corpus)+ # Convert chunks to a list of strings for tokenization+ tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]+ self.bm25 = BM25Okapi(tokenized_corpus)+ else:+ self.bm25 = Nonedef query(self,diff --git a/camel/retrievers/vector_retriever.py b/camel/retrievers/vector_retriever.pyindex 4cddb1acf..082c4370e 100644--- a/camel/retrievers/vector_retriever.py+++ b/camel/retrievers/vector_retriever.py@@ -12,6 +12,7 @@# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========import os+import warningsfrom typing import Any, Dict, List, Optionalfrom urllib.parse import urlparse@@ -89,9 +90,15 @@ class VectorRetriever(BaseRetriever):elements = self.uio.parse_file_or_url(content, **kwargs)else:elements = [self.uio.create_element_from_text(text=content)]- chunks = self.uio.chunk_elements(- chunk_type=chunk_type, elements=elements- )+ if elements:+ chunks = self.uio.chunk_elements(+ chunk_type=chunk_type, elements=elements+ )+ if not elements:+ warnings.warn(+ f"No elements were extracted from the content: {content}"+ )+ return# Iterate to process and store embeddings, set batch of 50for i in range(0, len(chunks), 50):batch_chunks = chunks[i : i + 50]diff --git a/camel/toolkits/retrieval_toolkit.py b/camel/toolkits/retrieval_toolkit.pyindex 24486addc..50b8dd431 100644--- a/camel/toolkits/retrieval_toolkit.py+++ b/camel/toolkits/retrieval_toolkit.py@@ -57,7 +57,7 @@ class RetrievalToolkit(BaseToolkit):retrieved_info = auto_retriever.run_vector_retriever(query=query, contents=contents, top_k=3)- return retrieved_info+ return str(retrieved_info)def get_tools(self) -> List[OpenAIFunction]:r"""Returns a list of OpenAIFunction objects representing thediff --git a/docs/conf.py b/docs/conf.pyindex a83b27230..4386728d7 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -27,7 +27,7 @@ sys.path.insert(0, os.path.abspath('..'))project = 'CAMEL'copyright = '2023, CAMEL-AI.org'author = 'CAMEL-AI.org'-release = '0.1.6.4'+release = '0.1.6.5'html_favicon = ('https://raw.githubusercontent.com/camel-ai/camel/master/misc/favicon.png'diff --git a/docs/get_started/setup.md b/docs/get_started/setup.mdindex 80aa09777..f23b0654a 100644--- a/docs/get_started/setup.md+++ b/docs/get_started/setup.md@@ -61,7 +61,7 @@ conda create --name camel python=3.10conda activate camel# Clone github repo-git clone -b v0.1.6.4 https://github.com/camel-ai/camel.git+git clone -b v0.1.6.5 https://github.com/camel-ai/camel.git# Change directory into project directorycd cameldiff --git a/pyproject.toml b/pyproject.tomlindex 746ec8762..0dd8227bf 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"[tool.poetry]name = "camel-ai"-version = "0.1.6.4"+version = "0.1.6.5"authors = ["CAMEL-AI.org"]description = "Communicative Agents for AI Society Study"readme = "README.md"diff --git a/test/loaders/test_unstructured_io.py b/test/loaders/test_unstructured_io.pyindex d302eb071..f99847847 100644--- a/test/loaders/test_unstructured_io.py+++ b/test/loaders/test_unstructured_io.py@@ -132,9 +132,10 @@ def test_stage_elements_for_csv(unstructured_instance: UnstructuredIO):"philadelphia-eagles-spt-intl/index.html")test_elements = unstructured_instance.parse_file_or_url(test_url)- staged_element: Any = unstructured_instance.stage_elements(- elements=test_elements, stage_type="stage_for_baseplate"- )+ if test_elements:+ staged_element: Any = unstructured_instance.stage_elements(+ elements=test_elements, stage_type="stage_for_baseplate"+ )assert staged_element['rows'][0] == {'data': {'type': 'UncategorizedText',@@ -155,11 +156,12 @@ def test_stage_elements_for_csv(unstructured_instance: UnstructuredIO):# Test with an invalid stage option (should raise ValueError)test_stage_type = "invalid_stageing_option"- with pytest.raises(ValueError):- unstructured_instance.stage_elements(- elements=test_elements,- stage_type=test_stage_type, # type: ignore[arg-type]- )+ if test_elements:+ with pytest.raises(ValueError):+ unstructured_instance.stage_elements(+ elements=test_elements,+ stage_type=test_stage_type, # type: ignore[arg-type]+ )# Test the chunk_elements method@@ -170,14 +172,16 @@ def test_chunk_elements(unstructured_instance: UnstructuredIO):"philadelphia-eagles-spt-intl/index.html")test_elements = unstructured_instance.parse_file_or_url(test_url)- chunked_sections = unstructured_instance.chunk_elements(- elements=test_elements, chunk_type="chunk_by_title"- )+ if test_elements:+ chunked_sections = unstructured_instance.chunk_elements(+ elements=test_elements, chunk_type="chunk_by_title"+ )assert len(chunked_sections) == 7 # Check the number of chunks# Test with an invalid chunk option (should raise ValueError)test_chunk_type = "chunk_by_invalid_option"- with pytest.raises(ValueError):- unstructured_instance.chunk_elements(- elements=test_elements, chunk_type=test_chunk_type- )+ if test_elements:+ with pytest.raises(ValueError):+ unstructured_instance.chunk_elements(+ elements=test_elements, chunk_type=test_chunk_type+ )diff --git a/test/retrievers/test_auto_retriever.py b/test/retrievers/test_auto_retriever.pyindex 6538f42c7..c41324d0d 100644--- a/test/retrievers/test_auto_retriever.py+++ b/test/retrievers/test_auto_retriever.py@@ -83,18 +83,20 @@ def test_run_vector_retriever(auto_retriever):return_detailed_info=True,)- assert (- "similarity score" in result_related+ assert "similarity score" in str(+ result_related), "result_related missing 'similarity score'"- assert (- "content path" in result_related+ assert "content path" in str(+ result_related), "result_related missing 'content path'"- assert "metadata" in result_related, "result_related missing 'metadata'"- assert "text" in result_related, "result_related missing 'text'"+ assert "metadata" in str(+ result_related+ ), "result_related missing 'metadata'"+ assert "text" in str(result_related), "result_related missing 'text'"# Test with query unrelated to the content in mock dataresult_unrelated = auto_retriever.run_vector_retriever(query_unrealted, contents, top_k, similarity_threshold)- assert "No suitable information retrieved from" in result_unrelated+ assert "No suitable information retrieved from" in str(result_unrelated)