Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Retrieval module support string as input (#817)

Wendong-Fan03612e3d9f1a
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
index 96967a815..4a128103a 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -27,7 +27,7 @@ body:
attributes:
label: What version of camel are you using?
description: Run command `python3 -c 'print(__import__("camel").__version__)'` in your shell and paste the output here.
- placeholder: E.g., 0.1.6.2
+ placeholder: E.g., 0.1.6.3
validations:
required: true
diff --git a/README.md b/README.md
index 6ee49032b..b57c76263 100644
--- a/README.md
+++ b/README.md
@@ -110,7 +110,7 @@ conda create --name camel python=3.9
conda activate camel
# Clone github repo
-git clone -b v0.1.6.2 https://github.com/camel-ai/camel.git
+git clone -b v0.1.6.3 https://github.com/camel-ai/camel.git
# Change directory into project directory
cd camel
diff --git a/camel/__init__.py b/camel/__init__.py
index b48ac378d..9bd37e96b 100644
--- a/camel/__init__.py
+++ b/camel/__init__.py
@@ -12,7 +12,7 @@
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-__version__ = '0.1.6.2'
+__version__ = '0.1.6.3'
__all__ = [
'__version__',
diff --git a/camel/retrievers/auto_retriever.py b/camel/retrievers/auto_retriever.py
index 82783d829..10d55d6c1 100644
--- a/camel/retrievers/auto_retriever.py
+++ b/camel/retrievers/auto_retriever.py
@@ -97,36 +97,36 @@ class AutoRetriever:
f"Unsupported vector storage type: {self.storage_type}"
)
- def _collection_name_generator(self, content_input_path: str) -> str:
+ def _collection_name_generator(self, content: str) -> str:
r"""Generates a valid collection name from a given file path or URL.
Args:
- content_input_path: str. The input URL or file path from which to
- generate the collection name.
+ contents (str): Local file path, remote URL or string content.
Returns:
str: A sanitized, valid collection name suitable for use.
"""
- # Check path type
- parsed_url = urlparse(content_input_path)
- self.is_url = all([parsed_url.scheme, parsed_url.netloc])
+ # Check if the content is URL
+ parsed_url = urlparse(content)
+ is_url = all([parsed_url.scheme, parsed_url.netloc])
# Convert given path into a collection name, ensuring it only
# contains numbers, letters, and underscores
- if self.is_url:
+ if is_url:
# For URLs, remove https://, replace /, and any characters not
# allowed by Milvus with _
collection_name = re.sub(
r'[^0-9a-zA-Z]+',
'_',
- content_input_path.replace("https://", ""),
+ content.replace("https://", ""),
)
- else:
+ elif os.path.exists(content):
# For file paths, get the stem and replace spaces with _, also
# ensuring only allowed characters are present
- collection_name = re.sub(
- r'[^0-9a-zA-Z]+', '_', Path(content_input_path).stem
- )
+ collection_name = re.sub(r'[^0-9a-zA-Z]+', '_', Path(content).stem)
+ else:
+ # the content is string input
+ collection_name = content[:10]
# Ensure the collection name does not start or end with underscore
collection_name = collection_name.strip("_")
@@ -193,7 +193,7 @@ class AutoRetriever:
def run_vector_retriever(
self,
query: str,
- content_input_paths: Union[str, List[str]],
+ contents: Union[str, List[str]],
top_k: int = DEFAULT_TOP_K_RESULTS,
similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
return_detailed_info: bool = False,
@@ -203,8 +203,8 @@ class AutoRetriever:
Args:
query (str): Query string for information retriever.
- content_input_paths (Union[str, List[str]]): Paths to local
- files or remote URLs.
+ contents (Union[str, List[str]]): Local file paths, remote URLs or
+ string contents.
top_k (int, optional): The number of top results to return during
retrieve. Must be a positive integer. Defaults to
`DEFAULT_TOP_K_RESULTS`.
@@ -223,24 +223,18 @@ class AutoRetriever:
Raises:
ValueError: If there's an vector storage existing with content
name in the vector path but the payload is None. If
- `content_input_paths` is empty.
+ `contents` is empty.
RuntimeError: If any errors occur during the retrieve process.
"""
- if not content_input_paths:
- raise ValueError("content_input_paths cannot be empty.")
+ if not contents:
+ raise ValueError("content cannot be empty.")
- content_input_paths = (
- [content_input_paths]
- if isinstance(content_input_paths, str)
- else content_input_paths
- )
+ contents = [contents] if isinstance(contents, str) else contents
all_retrieved_info = []
- for content_input_path in content_input_paths:
+ for content in contents:
# Generate a valid collection name
- collection_name = self._collection_name_generator(
- content_input_path
- )
+ collection_name = self._collection_name_generator(content)
try:
vector_storage_instance = self._initialize_vector_storage(
collection_name
@@ -251,13 +245,11 @@ class AutoRetriever:
file_is_modified = False # initialize with a default value
if (
vector_storage_instance.status().vector_count != 0
- and not self.is_url
+ and os.path.exists(content)
):
# Get original modified date from file
modified_date_from_file = (
- self._get_file_modified_date_from_file(
- content_input_path
- )
+ self._get_file_modified_date_from_file(content)
)
# Get modified date from vector storage
modified_date_from_storage = (
@@ -280,18 +272,16 @@ class AutoRetriever:
# Process and store the content to the vector storage
vr = VectorRetriever(
storage=vector_storage_instance,
- similarity_threshold=similarity_threshold,
embedding_model=self.embedding_model,
)
- vr.process(content_input_path)
+ vr.process(content)
else:
vr = VectorRetriever(
storage=vector_storage_instance,
- similarity_threshold=similarity_threshold,
embedding_model=self.embedding_model,
)
# Retrieve info by given query from the vector storage
- retrieved_info = vr.query(query, top_k)
+ retrieved_info = vr.query(query, top_k, similarity_threshold)
all_retrieved_info.extend(retrieved_info)
except Exception as e:
raise RuntimeError(
diff --git a/camel/retrievers/vector_retriever.py b/camel/retrievers/vector_retriever.py
index 251775d6b..4cddb1acf 100644
--- a/camel/retrievers/vector_retriever.py
+++ b/camel/retrievers/vector_retriever.py
@@ -11,7 +11,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
from camel.embeddings import BaseEmbedding, OpenAIEmbedding
from camel.loaders import UnstructuredIO
@@ -38,24 +40,18 @@ class VectorRetriever(BaseRetriever):
embedding_model (BaseEmbedding): Embedding model used to generate
vector embeddings.
storage (BaseVectorStorage): Vector storage to query.
- similarity_threshold (float, optional): The similarity threshold
- for filtering results. Defaults to `DEFAULT_SIMILARITY_THRESHOLD`.
unstructured_modules (UnstructuredIO): A module for parsing files and
URLs and chunking content based on specified parameters.
"""
def __init__(
self,
- similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
embedding_model: Optional[BaseEmbedding] = None,
storage: Optional[BaseVectorStorage] = None,
) -> None:
r"""Initializes the retriever class with an optional embedding model.
Args:
- similarity_threshold (float, optional): The similarity threshold
- for filtering results. Defaults to
- `DEFAULT_SIMILARITY_THRESHOLD`.
embedding_model (Optional[BaseEmbedding]): The embedding model
instance. Defaults to `OpenAIEmbedding` if not provided.
storage (BaseVectorStorage): Vector storage to query.
@@ -68,12 +64,11 @@ class VectorRetriever(BaseRetriever):
vector_dim=self.embedding_model.get_output_dim()
)
)
- self.similarity_threshold = similarity_threshold
- self.unstructured_modules: UnstructuredIO = UnstructuredIO()
+ self.uio: UnstructuredIO = UnstructuredIO()
def process(
self,
- content_input_path: str,
+ content: str,
chunk_type: str = "chunk_by_title",
**kwargs: Any,
) -> None:
@@ -82,16 +77,19 @@ class VectorRetriever(BaseRetriever):
vector storage.
Args:
- content_input_path (str): File path or URL of the content to be
- processed.
+ contents (str): Local file path, remote URL or string content.
chunk_type (str): Type of chunking going to apply. Defaults to
"chunk_by_title".
**kwargs (Any): Additional keyword arguments for content parsing.
"""
- elements = self.unstructured_modules.parse_file_or_url(
- content_input_path, **kwargs
- )
- chunks = self.unstructured_modules.chunk_elements(
+ # Check if the content is URL
+ parsed_url = urlparse(content)
+ is_url = all([parsed_url.scheme, parsed_url.netloc])
+ if is_url or os.path.exists(content):
+ elements = self.uio.parse_file_or_url(content, **kwargs)
+ else:
+ elements = [self.uio.create_element_from_text(text=content)]
+ chunks = self.uio.chunk_elements(
chunk_type=chunk_type, elements=elements
)
# Iterate to process and store embeddings, set batch of 50
@@ -105,7 +103,7 @@ class VectorRetriever(BaseRetriever):
# Prepare the payload for each vector record, includes the content
# path, chunk metadata, and chunk text
for vector, chunk in zip(batch_vectors, batch_chunks):
- content_path_info = {"content path": content_input_path}
+ content_path_info = {"content path": content}
chunk_metadata = {"metadata": chunk.metadata.to_dict()}
chunk_text = {"text": str(chunk)}
combined_dict = {
@@ -124,12 +122,16 @@ class VectorRetriever(BaseRetriever):
self,
query: str,
top_k: int = DEFAULT_TOP_K_RESULTS,
+ similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
) -> List[Dict[str, Any]]:
r"""Executes a query in vector storage and compiles the retrieved
results into a dictionary.
Args:
query (str): Query string for information retriever.
+ similarity_threshold (float, optional): The similarity threshold
+ for filtering results. Defaults to
+ `DEFAULT_SIMILARITY_THRESHOLD`.
top_k (int, optional): The number of top results to return during
retriever. Must be a positive integer. Defaults to 1.
@@ -161,7 +163,7 @@ class VectorRetriever(BaseRetriever):
formatted_results = []
for result in query_results:
if (
- result.similarity >= self.similarity_threshold
+ result.similarity >= similarity_threshold
and result.record.payload is not None
):
result_dict = {
@@ -182,7 +184,7 @@ class VectorRetriever(BaseRetriever):
'text': (
f"No suitable information retrieved "
f"from {content_path} with similarity_threshold"
- f" = {self.similarity_threshold}."
+ f" = {similarity_threshold}."
)
}
]
diff --git a/camel/toolkits/retrieval_toolkit.py b/camel/toolkits/retrieval_toolkit.py
index 034e1769a..24486addc 100644
--- a/camel/toolkits/retrieval_toolkit.py
+++ b/camel/toolkits/retrieval_toolkit.py
@@ -27,7 +27,7 @@ class RetrievalToolkit(BaseToolkit):
"""
def information_retrieval(
- self, query: str, content_input_paths: Union[str, List[str]]
+ self, query: str, contents: Union[str, List[str]]
) -> str:
r"""Retrieves information from a local vector storage based on the
specified query. This function connects to a local vector storage
@@ -37,8 +37,8 @@ class RetrievalToolkit(BaseToolkit):
Args:
query (str): The question or query for which an answer is required.
- content_input_paths (Union[str, List[str]]): Paths to local
- files or remote URLs.
+ contents (Union[str, List[str]]): Local file paths, remote URLs or
+ string contents.
Returns:
str: The information retrieved in response to the query, aggregated
@@ -47,7 +47,7 @@ class RetrievalToolkit(BaseToolkit):
Example:
# Retrieve information about CAMEL AI.
information_retrieval(query = "what is CAMEL AI?",
- content_input_paths="https://www.camel-ai.org/")
+ contents="https://www.camel-ai.org/")
"""
auto_retriever = AutoRetriever(
vector_storage_local_path="camel/temp_storage",
@@ -55,7 +55,7 @@ class RetrievalToolkit(BaseToolkit):
)
retrieved_info = auto_retriever.run_vector_retriever(
- query=query, content_input_paths=content_input_paths, top_k=3
+ query=query, contents=contents, top_k=3
)
return retrieved_info
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.py
index f794d6293..bac9423cb 100644
--- a/camel/toolkits/search_toolkit.py
+++ b/camel/toolkits/search_toolkit.py
@@ -65,7 +65,7 @@ class SearchToolkit(BaseToolkit):
return result
def search_duckduckgo(
- self, query: str, source: str = "text", max_results: int = 10
+ self, query: str, source: str = "text", max_results: int = 5
) -> List[Dict[str, Any]]:
r"""Use DuckDuckGo search engine to search information for
the given query.
@@ -78,7 +78,7 @@ class SearchToolkit(BaseToolkit):
query (str): The query to be searched.
source (str): The type of information to query (e.g., "text",
"images", "videos"). Defaults to "text".
- max_results (int): Max number of results, defaults to `10`.
+ max_results (int): Max number of results, defaults to `5`.
Returns:
List[Dict[str, Any]]: A list of dictionaries where each dictionary
@@ -152,7 +152,7 @@ class SearchToolkit(BaseToolkit):
return responses
def search_google(
- self, query: str, num_result_pages: int = 10
+ self, query: str, num_result_pages: int = 5
) -> List[Dict[str, Any]]:
r"""Use Google search engine to search information for the given query.
@@ -196,7 +196,7 @@ class SearchToolkit(BaseToolkit):
# Different language may get different result
search_language = "en"
# How many pages to return
- num_result_pages = 10
+ num_result_pages = num_result_pages
# Constructing the URL
# Doc: https://developers.google.com/custom-search/v1/using_rest
url = (
diff --git a/docs/conf.py b/docs/conf.py
index c905fbb42..0e7416681 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -27,7 +27,7 @@ sys.path.insert(0, os.path.abspath('..'))
project = 'CAMEL'
copyright = '2023, CAMEL-AI.org'
author = 'CAMEL-AI.org'
-release = '0.1.6.2'
+release = '0.1.6.3'
html_favicon = (
'https://raw.githubusercontent.com/camel-ai/camel/master/misc/favicon.png'
diff --git a/docs/get_started/setup.md b/docs/get_started/setup.md
index 583b4c02a..5fdfba08b 100644
--- a/docs/get_started/setup.md
+++ b/docs/get_started/setup.md
@@ -61,7 +61,7 @@ conda create --name camel python=3.10
conda activate camel
# Clone github repo
-git clone -b v0.1.6.2 https://github.com/camel-ai/camel.git
+git clone -b v0.1.6.3 https://github.com/camel-ai/camel.git
# Change directory into project directory
cd camel
diff --git a/docs/key_modules/retrievers.md b/docs/key_modules/retrievers.md
index 1aaab2ca8..de2392e14 100644
--- a/docs/key_modules/retrievers.md
+++ b/docs/key_modules/retrievers.md
@@ -89,7 +89,7 @@ ar = AutoRetriever(vector_storage_local_path="camel/retrievers",storage_type=Sto
# Run the auto vector retriever
retrieved_info = ar.run_vector_retriever(
- content_input_paths=[
+ contents=[
"https://www.camel-ai.org/", # Example remote url
],
query="What is CAMEL-AI",
diff --git a/docs/tutorials_and_cookbooks/agents_with_rag.md b/docs/tutorials_and_cookbooks/agents_with_rag.md
index 194a5192e..caf415b25 100644
--- a/docs/tutorials_and_cookbooks/agents_with_rag.md
+++ b/docs/tutorials_and_cookbooks/agents_with_rag.md
@@ -63,7 +63,7 @@ vector_retriever = VectorRetriever(embedding_model=embedding_instance)
We use integrated `Unstructured Module` to splite the content into small chunks, the content will be splited automacitlly with its `chunk_by_title` function, the max character for each chunk is 500 characters, which is a suitable length for `OpenAIEmbedding`. All the text in the chunks will be embed and stored to the vector storage instance, it will take some time, please wait..
```python
vector_retriever.process(
- content_input_path="local_data/camel paper.pdf",
+ content="local_data/camel paper.pdf",
storage=storage_instance,
)
```
@@ -120,7 +120,7 @@ auto_retriever = AutoRetriever(
retrieved_info = auto_retriever.run_vector_retriever(
query="What is CAMEL-AI",
- content_input_paths=[
+ contents=[
"local_data/camel paper.pdf", # example local path
"https://www.camel-ai.org/", # example remote url
],
@@ -166,7 +166,7 @@ def single_agent(query: str) ->str :
retrieved_info = auto_retriever.run_vector_retriever(
query=query,
- content_input_paths=[
+ contents=[
"local_data/camel paper.pdf", # example local path
"https://www.camel-ai.org/", # example remote url
],
@@ -221,7 +221,7 @@ def local_retriever(query: str) -> str:
retrieved_info = auto_retriever.run_vector_retriever(
query=query,
- content_input_paths=[
+ contents=[
"local_data/camel paper.pdf", # example local path
],
top_k=1
diff --git a/examples/bots/discord_bot.py b/examples/bots/discord_bot.py
index 735f2d8af..a87ecd885 100644
--- a/examples/bots/discord_bot.py
+++ b/examples/bots/discord_bot.py
@@ -116,7 +116,7 @@ class DiscordBot:
if self.auto_retriever:
retrieved_content = self.auto_retriever.run_vector_retriever(
query=user_raw_msg,
- vector_storage_local_path=self.content_input_paths,
+ contents=self.contents,
top_k=self.top_k,
return_detailed_info=self.return_detailed_info,
)
diff --git a/examples/function_call/github_examples.py b/examples/function_call/github_examples.py
index 4dc6fb2fb..af965a194 100644
--- a/examples/function_call/github_examples.py
+++ b/examples/function_call/github_examples.py
@@ -119,7 +119,7 @@ def solve_issue(
).as_dict()
model = ModelFactory.create(
- model_platform=ModelPlatformType.OpenAI,
+ model_platform=ModelPlatformType.OPENAI,
model_type=ModelType.GPT_3_5_TURBO,
model_config_dict=assistant_model_config_dict,
)
diff --git a/examples/translation/translator.py b/examples/translation/translator.py
index e3d205f5a..9eb2f48b0 100644
--- a/examples/translation/translator.py
+++ b/examples/translation/translator.py
@@ -119,7 +119,7 @@ def translate_content(
model_config = ChatGPTConfig(stream=True)
model = ModelFactory.create(
- model_platform=ModelPlatformType.OpenAI,
+ model_platform=ModelPlatformType.OPENAI,
model_type=ModelType.GPT_3_5_TURBO,
model_config=model_config,
)
diff --git a/pyproject.toml b/pyproject.toml
index 06b41d105..b6f13f0a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "camel-ai"
-version = "0.1.6.2"
+version = "0.1.6.3"
authors = ["CAMEL-AI.org"]
description = "Communicative Agents for AI Society Study"
readme = "README.md"
diff --git a/test/retrievers/test_auto_retriever.py b/test/retrievers/test_auto_retriever.py
index 6271a522b..6538f42c7 100644
--- a/test/retrievers/test_auto_retriever.py
+++ b/test/retrievers/test_auto_retriever.py
@@ -70,14 +70,14 @@ def test_run_vector_retriever(auto_retriever):
# Define mock data for testing
query_related = "what is camel ai"
query_unrealted = "unrelated query"
- content_input_paths = "https://www.camel-ai.org/"
+ contents = "https://www.camel-ai.org/"
top_k = 1
similarity_threshold = 0.5
# Test with query related to the content in mock data
result_related = auto_retriever.run_vector_retriever(
query_related,
- content_input_paths,
+ contents,
top_k,
similarity_threshold,
return_detailed_info=True,
@@ -94,7 +94,7 @@ def test_run_vector_retriever(auto_retriever):
# Test with query unrelated to the content in mock data
result_unrelated = auto_retriever.run_vector_retriever(
- query_unrealted, content_input_paths, top_k, similarity_threshold
+ query_unrealted, contents, top_k, similarity_threshold
)
assert "No suitable information retrieved from" in result_unrelated
diff --git a/test/retrievers/test_vector_retriever.py b/test/retrievers/test_vector_retriever.py
index 718e5c9e2..1c61c2ec9 100644
--- a/test/retrievers/test_vector_retriever.py
+++ b/test/retrievers/test_vector_retriever.py
@@ -76,10 +76,12 @@ def test_process(mock_unstructured_modules):
vector_retriever = VectorRetriever()
- vector_retriever.process(content_input_path="mock_path")
+ vector_retriever.process(content="https://www.camel-ai.org/")
# Assert that methods are called as expected
- mock_instance.parse_file_or_url.assert_called_once_with("mock_path")
+ mock_instance.parse_file_or_url.assert_called_once_with(
+ "https://www.camel-ai.org/"
+ )
mock_instance.chunk_elements.assert_called_once()