Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Change auto_retriever output format (#827)

Asher-hssb874c63cda18
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
index 3a469e450..4ec019f93 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -27,7 +27,7 @@ body:
attributes:
label: What version of camel are you using?
description: Run command `python3 -c 'print(__import__("camel").__version__)'` in your shell and paste the output here.
- placeholder: E.g., 0.1.6.4
+ placeholder: E.g., 0.1.6.5
validations:
required: true
diff --git a/README.md b/README.md
index a7fe1a7a6..8a213b2fd 100644
--- a/README.md
+++ b/README.md
@@ -110,7 +110,7 @@ conda create --name camel python=3.9
conda activate camel
# Clone github repo
-git clone -b v0.1.6.4 https://github.com/camel-ai/camel.git
+git clone -b v0.1.6.5 https://github.com/camel-ai/camel.git
# Change directory into project directory
cd camel
diff --git a/camel/__init__.py b/camel/__init__.py
index d07c9c115..c49f2ba06 100644
--- a/camel/__init__.py
+++ b/camel/__init__.py
@@ -12,7 +12,7 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-__version__ = '0.1.6.4'
+__version__ = '0.1.6.5'
__all__ = [
'__version__',
diff --git a/camel/configs/gemini_config.py b/camel/configs/gemini_config.py
index 67702080f..959c5baa0 100644
--- a/camel/configs/gemini_config.py
+++ b/camel/configs/gemini_config.py
@@ -81,7 +81,6 @@ class GeminiConfig(BaseConfig):
response_mime_type: Optional[str] = None
response_schema: Optional[Any] = None
safety_settings: Optional[Any] = None
- tools: Optional[Any] = None
tool_config: Optional[Any] = None
request_options: Optional[Any] = None
diff --git a/camel/configs/groq_config.py b/camel/configs/groq_config.py
index 5c945e8c8..fe3f2f90a 100644
--- a/camel/configs/groq_config.py
+++ b/camel/configs/groq_config.py
@@ -99,7 +99,7 @@ class GroqConfig(BaseConfig):
response_format: Union[dict, NotGiven] = NOT_GIVEN
frequency_penalty: float = 0.0
user: str = ""
- tool_choice: Optional[Union[dict[str, str], str]] = "none"
+ tool_choice: Optional[Union[dict[str, str], str]] = "auto"
GROQ_API_PARAMS = {param for param in GroqConfig.model_fields.keys()}
diff --git a/camel/configs/mistral_config.py b/camel/configs/mistral_config.py
index a3dbcdac4..4b248e680 100644
--- a/camel/configs/mistral_config.py
+++ b/camel/configs/mistral_config.py
@@ -74,7 +74,7 @@ class MistralConfig(BaseConfig):
if not isinstance(response_format, ResponseFormat):
raise ValueError(
f"The tool {response_format} should be an instance "
- "of `mistralai.models.chat_completion.ResponseFormat`."
+ "of `mistralai.models.ResponseFormat`."
)
return response_format
diff --git a/camel/loaders/unstructured_io.py b/camel/loaders/unstructured_io.py
index 2e03545b4..e9a864e48 100644
--- a/camel/loaders/unstructured_io.py
+++ b/camel/loaders/unstructured_io.py
@@ -12,6 +12,7 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import uuid
+import warnings
from typing import (
Any,
Dict,
@@ -91,7 +92,7 @@ class UnstructuredIO:
def parse_file_or_url(
input_path: str,
**kwargs: Any,
- ) -> List[Element]:
+ ) -> Union[List[Element], None]:
r"""Loads a file or a URL and parses its contents into elements.
Args:
@@ -99,12 +100,12 @@ class UnstructuredIO:
**kwargs: Extra kwargs passed to the partition function.
Returns:
- List[Element]: List of elements after parsing the file or URL.
+ Union[List[Element],None]: List of elements after parsing the file
+ or URL if success.
Raises:
FileNotFoundError: If the file does not exist at the path
specified.
- Exception: For any other issues during file or URL parsing.
Notes:
Available document types:
@@ -128,8 +129,9 @@ class UnstructuredIO:
try:
elements = partition_html(url=input_path, **kwargs)
return elements
- except Exception as e:
- raise Exception("Failed to parse the URL.") from e
+ except Exception:
+ warnings.warn(f"Failed to parse the URL: {input_path}")
+ return None
else:
# Handling file
@@ -146,10 +148,9 @@ class UnstructuredIO:
with open(input_path, "rb") as f:
elements = partition(file=f, **kwargs)
return elements
- except Exception as e:
- raise Exception(
- "Failed to parse the unstructured file."
- ) from e
+ except Exception:
+ warnings.warn(f"Failed to partition the file: {input_path}")
+ return None
@staticmethod
def clean_text_data(
diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.py
index 10d55d6c1..b84679f7f 100644
--- a/camel/retrievers/auto_retriever.py
+++ b/camel/retrievers/auto_retriever.py
@@ -15,7 +15,7 @@ import datetime
import os
import re
from pathlib import Path
-from typing import List, Optional, Tuple, Union
+from typing import Collection, List, Optional, Sequence, Tuple, Union
from urllib.parse import urlparse
from camel.embeddings import BaseEmbedding, OpenAIEmbedding
@@ -197,7 +197,7 @@ class AutoRetriever:
top_k: int = DEFAULT_TOP_K_RESULTS,
similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
return_detailed_info: bool = False,
- ) -> str:
+ ) -> dict[str, Sequence[Collection[str]]]:
r"""Executes the automatic vector retriever process using vector
storage.
@@ -216,9 +216,10 @@ class AutoRetriever:
metadata. Defaults to `False`.
Returns:
- string: By default, returns only the text information. If
- `return_detailed_info` is `True`, return detailed information
- including similarity score, content path and metadata.
+ dict[str, Sequence[Collection[str]]]: By default, returns
+ only the text information. If `return_detailed_info` is
+ `True`, return detailed information including similarity
+ score, content path and metadata.
Raises:
ValueError: If there's an vector storage existing with content
@@ -308,20 +309,17 @@ class AutoRetriever:
# Select the 'top_k' results
all_retrieved_info = all_retrieved_info_sorted[:top_k]
- retrieved_infos = "\n".join(str(info) for info in all_retrieved_info)
- retrieved_infos_text = "\n".join(
- info['text'] for info in all_retrieved_info if 'text' in info
- )
+ text_retrieved_info = [item['text'] for item in all_retrieved_info]
- detailed_info = (
- f"Original Query:\n{{ {query} }}\n"
- f"Retrieved Context:\n{retrieved_infos}"
- )
+ detailed_info = {
+ "Original Query": query,
+ "Retrieved Context": all_retrieved_info,
+ }
- text_info = (
- f"Original Query:\n{{ {query} }}\n"
- f"Retrieved Context:\n{retrieved_infos_text}"
- )
+ text_info = {
+ "Original Query": query,
+ "Retrieved Context": text_retrieved_info,
+ }
if return_detailed_info:
return detailed_info
diff --git a/camel/retrievers/bm25_retriever.py b/camel/retrievers/bm25_retriever.py
index 6a31024bb..4d085b10c 100644
--- a/camel/retrievers/bm25_retriever.py
+++ b/camel/retrievers/bm25_retriever.py
@@ -74,13 +74,16 @@ class BM25Retriever(BaseRetriever):
elements = self.unstructured_modules.parse_file_or_url(
content_input_path, **kwargs
)
- self.chunks = self.unstructured_modules.chunk_elements(
- chunk_type=chunk_type, elements=elements
- )
+ if elements:
+ self.chunks = self.unstructured_modules.chunk_elements(
+ chunk_type=chunk_type, elements=elements
+ )
- # Convert chunks to a list of strings for tokenization
- tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]
- self.bm25 = BM25Okapi(tokenized_corpus)
+ # Convert chunks to a list of strings for tokenization
+ tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]
+ self.bm25 = BM25Okapi(tokenized_corpus)
+ else:
+ self.bm25 = None
def query(
self,
diff --git a/camel/retrievers/vector_retriever.py b/camel/retrievers/vector_retriever.py
index 4cddb1acf..082c4370e 100644
--- a/camel/retrievers/vector_retriever.py
+++ b/camel/retrievers/vector_retriever.py
@@ -12,6 +12,7 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
import os
+import warnings
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
@@ -89,9 +90,15 @@ class VectorRetriever(BaseRetriever):
elements = self.uio.parse_file_or_url(content, **kwargs)
else:
elements = [self.uio.create_element_from_text(text=content)]
- chunks = self.uio.chunk_elements(
- chunk_type=chunk_type, elements=elements
- )
+ if elements:
+ chunks = self.uio.chunk_elements(
+ chunk_type=chunk_type, elements=elements
+ )
+ if not elements:
+ warnings.warn(
+ f"No elements were extracted from the content: {content}"
+ )
+ return
# Iterate to process and store embeddings, set batch of 50
for i in range(0, len(chunks), 50):
batch_chunks = chunks[i : i + 50]
diff --git a/camel/toolkits/retrieval_toolkit.py b/camel/toolkits/retrieval_toolkit.py
index 24486addc..50b8dd431 100644
--- a/camel/toolkits/retrieval_toolkit.py
+++ b/camel/toolkits/retrieval_toolkit.py
@@ -57,7 +57,7 @@ class RetrievalToolkit(BaseToolkit):
retrieved_info = auto_retriever.run_vector_retriever(
query=query, contents=contents, top_k=3
)
- return retrieved_info
+ return str(retrieved_info)
def get_tools(self) -> List[OpenAIFunction]:
r"""Returns a list of OpenAIFunction objects representing the
diff --git a/docs/conf.py b/docs/conf.py
index a83b27230..4386728d7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,7 +27,7 @@ sys.path.insert(0, os.path.abspath('..'))
project = 'CAMEL'
copyright = '2023, CAMEL-AI.org'
author = 'CAMEL-AI.org'
-release = '0.1.6.4'
+release = '0.1.6.5'
html_favicon = (
'https://raw.githubusercontent.com/camel-ai/camel/master/misc/favicon.png'
diff --git a/docs/get_started/setup.md b/docs/get_started/setup.md
index 80aa09777..f23b0654a 100644
--- a/docs/get_started/setup.md
+++ b/docs/get_started/setup.md
@@ -61,7 +61,7 @@ conda create --name camel python=3.10
conda activate camel
# Clone github repo
-git clone -b v0.1.6.4 https://github.com/camel-ai/camel.git
+git clone -b v0.1.6.5 https://github.com/camel-ai/camel.git
# Change directory into project directory
cd camel
diff --git a/pyproject.toml b/pyproject.toml
index 746ec8762..0dd8227bf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "camel-ai"
-version = "0.1.6.4"
+version = "0.1.6.5"
authors = ["CAMEL-AI.org"]
description = "Communicative Agents for AI Society Study"
readme = "README.md"
diff --git a/test/loaders/test_unstructured_io.py b/test/loaders/test_unstructured_io.py
index d302eb071..f99847847 100644
--- a/test/loaders/test_unstructured_io.py
+++ b/test/loaders/test_unstructured_io.py
@@ -132,9 +132,10 @@ def test_stage_elements_for_csv(unstructured_instance: UnstructuredIO):
"philadelphia-eagles-spt-intl/index.html"
)
test_elements = unstructured_instance.parse_file_or_url(test_url)
- staged_element: Any = unstructured_instance.stage_elements(
- elements=test_elements, stage_type="stage_for_baseplate"
- )
+ if test_elements:
+ staged_element: Any = unstructured_instance.stage_elements(
+ elements=test_elements, stage_type="stage_for_baseplate"
+ )
assert staged_element['rows'][0] == {
'data': {
'type': 'UncategorizedText',
@@ -155,11 +156,12 @@ def test_stage_elements_for_csv(unstructured_instance: UnstructuredIO):
# Test with an invalid stage option (should raise ValueError)
test_stage_type = "invalid_stageing_option"
- with pytest.raises(ValueError):
- unstructured_instance.stage_elements(
- elements=test_elements,
- stage_type=test_stage_type, # type: ignore[arg-type]
- )
+ if test_elements:
+ with pytest.raises(ValueError):
+ unstructured_instance.stage_elements(
+ elements=test_elements,
+ stage_type=test_stage_type, # type: ignore[arg-type]
+ )
# Test the chunk_elements method
@@ -170,14 +172,16 @@ def test_chunk_elements(unstructured_instance: UnstructuredIO):
"philadelphia-eagles-spt-intl/index.html"
)
test_elements = unstructured_instance.parse_file_or_url(test_url)
- chunked_sections = unstructured_instance.chunk_elements(
- elements=test_elements, chunk_type="chunk_by_title"
- )
+ if test_elements:
+ chunked_sections = unstructured_instance.chunk_elements(
+ elements=test_elements, chunk_type="chunk_by_title"
+ )
assert len(chunked_sections) == 7 # Check the number of chunks
# Test with an invalid chunk option (should raise ValueError)
test_chunk_type = "chunk_by_invalid_option"
- with pytest.raises(ValueError):
- unstructured_instance.chunk_elements(
- elements=test_elements, chunk_type=test_chunk_type
- )
+ if test_elements:
+ with pytest.raises(ValueError):
+ unstructured_instance.chunk_elements(
+ elements=test_elements, chunk_type=test_chunk_type
+ )
diff --git a/test/retrievers/test_auto_retriever.py b/test/retrievers/test_auto_retriever.py
index 6538f42c7..c41324d0d 100644
--- a/test/retrievers/test_auto_retriever.py
+++ b/test/retrievers/test_auto_retriever.py
@@ -83,18 +83,20 @@ def test_run_vector_retriever(auto_retriever):
return_detailed_info=True,
)
- assert (
- "similarity score" in result_related
+ assert "similarity score" in str(
+ result_related
), "result_related missing 'similarity score'"
- assert (
- "content path" in result_related
+ assert "content path" in str(
+ result_related
), "result_related missing 'content path'"
- assert "metadata" in result_related, "result_related missing 'metadata'"
- assert "text" in result_related, "result_related missing 'text'"
+ assert "metadata" in str(
+ result_related
+ ), "result_related missing 'metadata'"
+ assert "text" in str(result_related), "result_related missing 'text'"
# Test with query unrelated to the content in mock data
result_unrelated = auto_retriever.run_vector_retriever(
query_unrealted, contents, top_k, similarity_threshold
)
- assert "No suitable information retrieved from" in result_unrelated
+ assert "No suitable information retrieved from" in str(result_unrelated)