🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add duckduckgo search and enhance the text extraction from the website (#440)
jjyaoao3a56dc50a27f
diff --git a/camel/agents/__init__.py b/camel/agents/__init__.pyindex a94c98c09..df63e0cbc 100644--- a/camel/agents/__init__.py+++ b/camel/agents/__init__.py@@ -17,6 +17,7 @@ from .critic_agent import CriticAgentfrom .embodied_agent import EmbodiedAgentfrom .knowledge_graph_agent import KnowledgeGraphAgentfrom .role_assignment_agent import RoleAssignmentAgent+from .search_agent import SearchAgentfrom .task_agent import (TaskCreationAgent,TaskPlannerAgent,@@ -38,5 +39,6 @@ __all__ = ['HuggingFaceToolAgent','EmbodiedAgent','RoleAssignmentAgent',+ 'SearchAgent','KnowledgeGraphAgent',]diff --git a/camel/agents/role_assignment_agent.py b/camel/agents/role_assignment_agent.pyindex 4b3a79fec..38be7e1c3 100644--- a/camel/agents/role_assignment_agent.py+++ b/camel/agents/role_assignment_agent.py@@ -22,6 +22,7 @@ from camel.types import ModelType, RoleTypeclass RoleAssignmentAgent(ChatAgent):r"""An agent that generates role names based on the task prompt.+Attributes:role_assignment_prompt (TextPrompt): A prompt for the agent to generaterole names.diff --git a/camel/agents/search_agent.py b/camel/agents/search_agent.pynew file mode 100644index 000000000..d66c8e1c1--- /dev/null+++ b/camel/agents/search_agent.py@@ -0,0 +1,122 @@+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+# Licensed under the Apache License, Version 2.0 (the “License”);+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an “AS IS” BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from typing import Any, Optional++from camel.agents.chat_agent import ChatAgent+from camel.messages import BaseMessage+from camel.prompts import TextPrompt+from camel.types import ModelType, RoleType+from camel.utils import create_chunks+++class SearchAgent(ChatAgent):+ r"""An agent that summarizes text based on a query and evaluates the+ relevance of an answer.++ Args:+ model_type (ModelType, optional): The type of model to use for the+ agent. (default: :obj:`ModelType.GPT_3_5_TURBO`)+ model_config (Any, optional): The configuration for the model.+ (default: :obj:`None`)+ """++ def __init__(+ self,+ model_type: ModelType = ModelType.GPT_3_5_TURBO,+ model_config: Optional[Any] = None,+ ) -> None:+ system_message = BaseMessage(+ role_name="Assistant",+ role_type=RoleType.ASSISTANT,+ meta_dict=None,+ content="You are a helpful assistant.",+ )+ super().__init__(system_message, model_type, model_config)++ def summarize_text(self, text: str, query: str) -> str:+ r"""Summarize the information from the text, base on the query.++ Args:+ text (str): Text to summarize.+ query (str): What information you want.++ Returns:+ str: Strings with information.+ """+ self.reset()++ summary_prompt = TextPrompt(+ '''Gather information from this text that relative to the+ question, but do not directly answer the question.\nquestion:+ {query}\ntext '''+ )+ summary_prompt = summary_prompt.format(query=query)+ # Max length of each chunk+ max_len = 3000+ results = ""+ chunks = create_chunks(text, max_len)+ # Summarize+ for i, chunk in enumerate(chunks, start=1):+ prompt = summary_prompt + str(i) + ": " + chunk+ user_msg = BaseMessage.make_user_message(+ role_name="User",+ content=prompt,+ )+ result = self.step(user_msg).msg.content+ results += result + "\n"++ # Final summarise+ final_prompt = TextPrompt(+ '''Here are some summarized texts which split from one text. Using+ the information to answer the question. If can't find the answer,+ you must answer "I can not find the answer to the query" and+ explain why.\n Query:\n{query}.\n\nText:\n'''+ )+ final_prompt = final_prompt.format(query=query)+ prompt = final_prompt + results++ user_msg = BaseMessage.make_user_message(+ role_name="User",+ content=prompt,+ )+ response = self.step(user_msg).msg.content++ return response++ def continue_search(self, query: str, answer: str) -> bool:+ r"""Ask whether to continue search or not based on the provided answer.++ Args:+ query (str): The question.+ answer (str): The answer to the question.++ Returns:+ bool: `True` if the user want to continue search, `False`+ otherwise.+ """+ prompt = TextPrompt(+ "Do you think the ANSWER can answer the QUERY? "+ "Use only 'yes' or 'no' to answer.\n"+ "===== QUERY =====\n{query}\n\n"+ "===== ANSWER =====\n{answer}"+ )+ prompt = prompt.format(query=query, answer=answer)+ user_msg = BaseMessage.make_user_message(+ role_name="User",+ content=prompt,+ )+ response = self.step(user_msg).msg.content+ if "yes" in str(response).lower():+ return False+ return Truediff --git a/camel/functions/search_functions.py b/camel/functions/search_functions.pyindex cf1cd3d57..e983edfb8 100644--- a/camel/functions/search_functions.py+++ b/camel/functions/search_functions.py@@ -14,15 +14,12 @@import osfrom typing import Any, Dict, List-from camel.agents import ChatAgentfrom camel.functions.openai_function import OpenAIFunction-from camel.messages import BaseMessage-from camel.prompts import TextPromptdef search_wiki(entity: str) -> str:- r"""Search the entity in WikiPedia and return the summary of the- required page, containing factual information about the given entity.+ r"""Search the entity in WikiPedia and return the summary of the required+ page, containing factual information about the given entity.Args:entity (str): The entity to be searched.@@ -59,11 +56,101 @@ def search_wiki(entity: str) -> str:return result-def search_google(query: str) -> List[Dict[str, Any]]:+def search_duckduckgo(+ query: str, source: str = "text", max_results: int = 10+) -> List[Dict[str, Any]]:+ r"""Use DuckDuckGo search engine to search information for the given query.++ This function queries the DuckDuckGo API for related topics to the given+ search term. The results are formatted into a list of dictionaries, each+ representing a search result.++ Args:+ query (str): The query to be searched.+ source (str): The type of information to query (e.g., "text",+ "images", "videos"). Defaults to "text".+ max_results (int): Max number of results, defaults to `10`.++ Returns:+ List[Dict[str, Any]]: A list of dictionaries where each dictionary+ represents a search result.+ """+ from duckduckgo_search import DDGS+ from requests.exceptions import RequestException++ ddgs = DDGS()+ responses: List[Dict[str, Any]] = []++ if source == "text":+ try:+ results = ddgs.text(keywords=query, max_results=max_results)+ except RequestException as e:+ # Handle specific exceptions or general request exceptions+ responses.append({"error": f"duckduckgo search failed.{e}"})++ # Iterate over results found+ for i, result in enumerate(results, start=1):+ # Creating a response object with a similar structure+ response = {+ "result_id": i,+ "title": result["title"],+ "description": result["body"],+ "url": result["href"],+ }+ responses.append(response)++ elif source == "images":+ try:+ results = ddgs.images(keywords=query, max_results=max_results)+ except RequestException as e:+ # Handle specific exceptions or general request exceptions+ responses.append({"error": f"duckduckgo search failed.{e}"})++ # Iterate over results found+ for i, result in enumerate(results, start=1):+ # Creating a response object with a similar structure+ response = {+ "result_id": i,+ "title": result["title"],+ "image": result["image"],+ "url": result["url"],+ "source": result["source"],+ }+ responses.append(response)++ elif source == "videos":+ try:+ results = ddgs.videos(keywords=query, max_results=max_results)+ except RequestException as e:+ # Handle specific exceptions or general request exceptions+ responses.append({"error": f"duckduckgo search failed.{e}"})++ # Iterate over results found+ for i, result in enumerate(results, start=1):+ # Creating a response object with a similar structure+ response = {+ "result_id": i,+ "title": result["title"],+ "description": result["description"],+ "embed_url": result["embed_url"],+ "publisher": result["publisher"],+ "duration": result["duration"],+ "published": result["published"],+ }+ responses.append(response)++ # If no answer found, return an empty list+ return responses+++def search_google(+ query: str, num_result_pages: int = 10+) -> List[Dict[str, Any]]:r"""Use Google search engine to search information for the given query.Args:query (str): The query to be searched.+ num_result_pages (int): The number of result pages to retrieve.Returns:List[Dict[str, Any]]: A list of dictionaries where each dictionary@@ -82,7 +169,7 @@ def search_google(query: str) -> List[Dict[str, Any]]:'description': 'An organization focused on ensuring thatartificial general intelligence benefits all of humanity.','long_description': 'OpenAI is a non-profit artificial- intelligence research company. Our goal is to advance digital+ intelligence research company. Our goal is to advance digitalintelligence in the way that is most likely to benefit humanityas a whole','url': 'https://www.openai.com'@@ -148,172 +235,12 @@ def search_google(query: str) -> List[Dict[str, Any]]:responses.append({"error": "google search failed."})except requests.RequestException:+ # Handle specific exceptions or general request exceptionsresponses.append({"error": "google search failed."})-+ # If no answer found, return an empty listreturn responses-def text_extract_from_web(url: str) -> str:- r"""Get the text information from given url.-- Args:- url (str): The website you want to search.-- Returns:- str: All texts extract from the web.- """- import requests- from bs4 import BeautifulSoup-- try:- # Request the target page- response_text = requests.get(url).text-- # Parse the obtained page- soup = BeautifulSoup(response_text, features="html.parser")-- for script in soup(["script", "style"]):- script.extract()-- text = soup.get_text()- # Strip text- lines = (line.strip() for line in text.splitlines())- chunks = (- phrase.strip() for line in lines for phrase in line.split(" ")- )- text = ".".join(chunk for chunk in chunks if chunk)-- except requests.RequestException:- text = f"can't access {url}"-- return text---# Split a text into smaller chunks of size n-def create_chunks(text: str, n: int) -> List[str]:- r"""Returns successive n-sized chunks from provided text."-- Args:- text (str): The text to be split.- n (int): The max length of a single chunk.-- Returns:- List[str]: A list of split texts.- """-- chunks = []- i = 0- while i < len(text):- # Find the nearest end of sentence within a range of 0.5 * n- # and 1.5 * n tokens- j = min(i + int(1.2 * n), len(text))- while j > i + int(0.8 * n):- # Decode the tokens and check for full stop or newline- chunk = text[i:j]- if chunk.endswith(".") or chunk.endswith("\n"):- break- j -= 1- # If no end of sentence found, use n tokens as the chunk size- if j == i + int(0.8 * n):- j = min(i + n, len(text))- chunks.append(text[i:j])- i = j- return chunks---def prompt_single_step_agent(prompt: str) -> str:- """Prompt a single-step agent to summarize texts or answer a question."""-- assistant_sys_msg = BaseMessage.make_assistant_message(- role_name="Assistant",- content="You are a helpful assistant.",- )- agent = ChatAgent(assistant_sys_msg)- agent.reset()-- user_msg = BaseMessage.make_user_message(- role_name="User",- content=prompt,- )- assistant_response = agent.step(user_msg)- if assistant_response.msgs is not None:- return assistant_response.msg.content- return ""---def summarize_text(text: str, query: str) -> str:- r"""Summarize the information from the text, base on the query if query is- given.-- Args:- text (str): Text to summarize.- query (str): What information you want.-- Returns:- str: Strings with information.- """- summary_prompt = TextPrompt(- '''Gather information from this text that relative to the question, but- do not directly answer the question.\nquestion: {query}\ntext '''- )- summary_prompt = summary_prompt.format(query=query)- # Max length of each chunk- max_len = 3000- results = ""- chunks = create_chunks(text, max_len)- # Summarize- for i, chunk in enumerate(chunks, start=1):- prompt = summary_prompt + str(i) + ": " + chunk- result = prompt_single_step_agent(prompt)- results += result + "\n"-- # Final summarise- final_prompt = TextPrompt(- '''Here are some summarized texts which split from one text, Using the- information to answer the question: {query}.\n\nText: '''- )- final_prompt = final_prompt.format(query=query)- prompt = final_prompt + results-- response = prompt_single_step_agent(prompt)-- return response---def search_google_and_summarize(query: str) -> str:- r"""Search webs for information. Given a query, this function will use- the Google search engine to search for related information from the- internet, and then return a summarized answer.-- Args:- query (str): Question you want to be answered.-- Returns:- str: Summarized information from webs.- """- # Google search will return a list of urls- responses = search_google(query)- for item in responses:- if "url" in item:- url = item.get("url")- # Extract text- text = text_extract_from_web(str(url))- # Using chatgpt summarise text- answer = summarize_text(text, query)-- # Let chatgpt decide whether to continue search or not- prompt = TextPrompt(- '''Do you think the answer: {answer} can answer the query:- {query}. Use only 'yes' or 'no' to answer.'''- )- prompt = prompt.format(answer=answer, query=query)- reply = prompt_single_step_agent(prompt)- if "yes" in str(reply).lower():- return answer-- return "Failed to find the answer from google search."--def query_wolfram_alpha(query: str, is_detailed: bool) -> str:r"""Queries Wolfram|Alpha and returns the result. Wolfram|Alpha is ananswer engine developed by Wolfram Research. It is offered as an online@@ -371,5 +298,10 @@ def query_wolfram_alpha(query: str, is_detailed: bool) -> str:SEARCH_FUNCS: List[OpenAIFunction] = [OpenAIFunction(func) # type: ignore[arg-type]- for func in [search_wiki, search_google_and_summarize, query_wolfram_alpha]+ for func in [+ search_wiki,+ search_google,+ search_duckduckgo,+ query_wolfram_alpha,+ ]]diff --git a/camel/utils/__init__.py b/camel/utils/__init__.pyindex 32d474412..c0e9b7db3 100644--- a/camel/utils/__init__.py+++ b/camel/utils/__init__.py@@ -11,10 +11,12 @@# See the License for the specific language governing permissions and# limitations under the License.# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========+from .commons import (PYDANTIC_V2,api_keys_required,check_server_running,+ create_chunks,dependencies_required,download_tasks,get_first_int,@@ -23,6 +25,7 @@ from .commons import (get_task_list,model_api_key_required,print_text_animated,+ text_extract_from_web,to_pascal,)from .token_counting import (@@ -49,6 +52,8 @@ __all__ = ['BaseTokenCounter','OpenAITokenCounter','OpenSourceTokenCounter',+ 'text_extract_from_web',+ 'create_chunks','dependencies_required','api_keys_required',]diff --git a/camel/utils/commons.py b/camel/utils/commons.pyindex bab9b3faf..3372006a3 100644--- a/camel/utils/commons.py+++ b/camel/utils/commons.py@@ -332,3 +332,63 @@ def to_pascal(snake: str) -> str:PYDANTIC_V2 = pydantic.VERSION.startswith("2.")+++def text_extract_from_web(url: str) -> str:+ r"""Get the text information from given url.++ Args:+ url (str): The website you want to search.++ Returns:+ str: All texts extract from the web.+ """+ try:+ import requests+ from newspaper import Article++ # Request the target page+ article = Article(url)+ article.download()+ article.parse()+ text = article.text++ except requests.RequestException as e:+ text = f"Can't access {url}, error: {e}"++ except Exception as e:+ text = f"Can't extract text from {url}, error: {e}"++ return text+++def create_chunks(text: str, n: int) -> List[str]:+ r"""Returns successive n-sized chunks from provided text. Split a text+ into smaller chunks of size n".++ Args:+ text (str): The text to be split.+ n (int): The max length of a single chunk.++ Returns:+ List[str]: A list of split texts.+ """++ chunks = []+ i = 0+ while i < len(text):+ # Find the nearest end of sentence within a range of 0.5 * n+ # and 1.5 * n tokens+ j = min(i + int(1.2 * n), len(text))+ while j > i + int(0.8 * n):+ # Decode the tokens and check for full stop or newline+ chunk = text[i:j]+ if chunk.endswith(".") or chunk.endswith("\n"):+ break+ j -= 1+ # If no end of sentence found, use n tokens as the chunk size+ if j == i + int(0.8 * n):+ j = min(i + n, len(text))+ chunks.append(text[i:j])+ i = j+ return chunksdiff --git a/poetry.lock b/poetry.lockindex 782d07ba1..bdf984d66 100644--- a/poetry.lock+++ b/poetry.lock@@ -381,7 +381,7 @@ files = [name = "cffi"version = "1.16.0"description = "Foreign Function Interface for Python calling C code."-optional = true+optional = falsepython-versions = ">=3.8"files = [{file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"},@@ -829,6 +829,42 @@ ssh = ["bcrypt (>=3.1.5)"]test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"]test-randomorder = ["pytest-randomly"]+[[package]]+name = "cssselect"+version = "1.2.0"+description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0"+optional = true+python-versions = ">=3.7"+files = [+ {file = "cssselect-1.2.0-py2.py3-none-any.whl", hash = "sha256:da1885f0c10b60c03ed5eccbb6b68d6eff248d91976fcde348f395d54c9fd35e"},+ {file = "cssselect-1.2.0.tar.gz", hash = "sha256:666b19839cfaddb9ce9d36bfe4c969132c647b92fc9088c4e23f786b30f1b3dc"},+]++[[package]]+name = "curl-cffi"+version = "0.6.2"+description = "libcurl ffi bindings for Python, with impersonation support"+optional = false+python-versions = ">=3.8"+files = [+ {file = "curl_cffi-0.6.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:23b8a2872b160718c04b06b1f8aa4fb1a2f4f94bce7040493515e081a27cad19"},+ {file = "curl_cffi-0.6.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ad3c1cf5360810825ec4bc3da425f26ee4098878a615dab9d309a99afd883ba9"},+ {file = "curl_cffi-0.6.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d01de6ed737ad1924aaa0198195b9020c38e77ce90ea3d72b9eacf4938c7adf"},+ {file = "curl_cffi-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37e513cc149d024a2d625e202f2cc9d4423d2937343ea2e06f797d99779e62dc"},+ {file = "curl_cffi-0.6.2-cp38-abi3-win32.whl", hash = "sha256:12e829af97cbf7c1d5afef177e786f6f404ddf163b08897a1ed087cadbeb4837"},+ {file = "curl_cffi-0.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:3791b7a9ae4cb1298165300f2dc2d60a86779f055570ae83163fc2d8a74bf714"},+ {file = "curl_cffi-0.6.2.tar.gz", hash = "sha256:9ee519e960b5fc6e0bbf13d0ecba9ce5f6306cb929354504bf03cc30f59a8f63"},+]++[package.dependencies]+certifi = "*"+cffi = ">=1.12.0"++[package.extras]+build = ["cibuildwheel", "wheel"]+dev = ["autoflake (==1.4)", "coverage (==6.4.1)", "cryptography (==38.0.3)", "flake8 (==6.0.0)", "flake8-bugbear (==22.7.1)", "flake8-pie (==0.15.0)", "httpx (==0.23.1)", "mypy (==0.971)", "nest-asyncio (==1.6.0)", "pytest (==7.1.2)", "pytest-asyncio (==0.19.0)", "pytest-trio (==0.7.0)", "ruff (==0.1.14)", "trio (==0.21.0)", "trio-typing (==0.7.0)", "trustme (==0.9.0)", "types-certifi (==2021.10.8.2)", "uvicorn (==0.18.3)", "websockets (==11.0.3)"]+test = ["cryptography (==38.0.3)", "fastapi (==0.100.0)", "httpx (==0.23.1)", "nest-asyncio (==1.6.0)", "proxy.py (==2.4.3)", "pytest (==7.1.2)", "pytest-asyncio (==0.19.0)", "pytest-trio (==0.7.0)", "python-multipart (==0.0.6)", "trio (==0.21.0)", "trio-typing (==0.7.0)", "trustme (==0.9.0)", "types-certifi (==2021.10.8.2)", "uvicorn (==0.18.3)", "websockets (==11.0.3)"]+[[package]]name = "cycler"version = "0.12.1"@@ -1036,6 +1072,26 @@ files = [{file = "docx2txt-0.8.tar.gz", hash = "sha256:2c06d98d7cfe2d3947e5760a57d924e3ff07745b379c8737723922e7009236e5"},]+[[package]]+name = "duckduckgo-search"+version = "6.1.5"+description = "Search for words, documents, images, news, maps and text translation using the DuckDuckGo.com search engine."+optional = true+python-versions = ">=3.8"+files = [+ {file = "duckduckgo_search-6.1.5-py3-none-any.whl", hash = "sha256:f0a18fe5f20323ba6bb11865ce32d4520bb90086a6ae62f5da510865f5a7dca8"},+ {file = "duckduckgo_search-6.1.5.tar.gz", hash = "sha256:10e5c4d09a4243fd9d85007dc4fe637456c3c3995fd2e1e9b49ffd6f75bb0afb"},+]++[package.dependencies]+click = ">=8.1.7"+orjson = ">=3.10.3"+pyreqwest-impersonate = ">=0.4.7"++[package.extras]+dev = ["mypy (>=1.10.0)", "pytest (>=8.2.0)", "pytest-asyncio (>=0.23.6)", "ruff (>=0.4.4)"]+lxml = ["lxml (>=5.2.2)"]+[[package]]name = "effdet"version = "0.4.1"@@ -1222,6 +1278,35 @@ lz4 = ["lz4"]snappy = ["cramjam"]zstandard = ["zstandard"]+[[package]]+name = "feedfinder2"+version = "0.0.4"+description = "Find the feed URLs for a website."+optional = true+python-versions = "*"+files = [+ {file = "feedfinder2-0.0.4.tar.gz", hash = "sha256:3701ee01a6c85f8b865a049c30ba0b4608858c803fe8e30d1d289fdbe89d0efe"},+]++[package.dependencies]+beautifulsoup4 = "*"+requests = "*"+six = "*"++[[package]]+name = "feedparser"+version = "6.0.11"+description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"+optional = true+python-versions = ">=3.6"+files = [+ {file = "feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45"},+ {file = "feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5"},+]++[package.dependencies]+sgmllib3k = "*"+[[package]]name = "ffmpy"version = "0.3.2"@@ -1984,6 +2069,16 @@ files = [docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]testing = ["portend", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"]+[[package]]+name = "jieba3k"+version = "0.35.1"+description = "Chinese Words Segementation Utilities"+optional = true+python-versions = "*"+files = [+ {file = "jieba3k-0.35.1.zip", hash = "sha256:980a4f2636b778d312518066be90c7697d410dd5a472385f5afced71a2db1c10"},+]+[[package]]name = "jinja2"version = "3.1.4"@@ -2932,6 +3027,32 @@ doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx-extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"]test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"]+[[package]]+name = "newspaper3k"+version = "0.2.8"+description = "Simplified python article discovery & extraction."+optional = true+python-versions = "*"+files = [+ {file = "newspaper3k-0.2.8-py3-none-any.whl", hash = "sha256:44a864222633d3081113d1030615991c3dbba87239f6bbf59d91240f71a22e3e"},+ {file = "newspaper3k-0.2.8.tar.gz", hash = "sha256:9f1bd3e1fb48f400c715abf875cc7b0a67b7ddcd87f50c9aeeb8fcbbbd9004fb"},+]++[package.dependencies]+beautifulsoup4 = ">=4.4.1"+cssselect = ">=0.9.2"+feedfinder2 = ">=0.0.4"+feedparser = ">=5.2.1"+jieba3k = ">=0.35.1"+lxml = ">=3.6.0"+nltk = ">=3.2.1"+Pillow = ">=3.3.0"+python-dateutil = ">=2.5.3"+PyYAML = ">=3.11"+requests = ">=2.10.0"+tinysegmenter = "0.3"+tldextract = ">=2.0.1"+[[package]]name = "nltk"version = "3.8.1"@@ -3820,7 +3941,7 @@ numpy = "*"name = "pycparser"version = "2.22"description = "C parser in Python"-optional = true+optional = falsepython-versions = ">=3.8"files = [{file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},@@ -4216,6 +4337,66 @@ files = [{file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"},]+[[package]]+name = "pyreqwest-impersonate"+version = "0.4.7"+description = "HTTP client that can impersonate web browsers, mimicking their headers and `TLS/JA3/JA4/HTTP2` fingerprints"+optional = true+python-versions = ">=3.8"+files = [+ {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c175dfc429c4231a6ce03841630b236f50995ca613ff1eea26fa4c75c730b562"},+ {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b3f83c50cef2d5ed0a9246318fd3ef3bfeabe286d4eabf92df4835c05a0be7dc"},+ {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f34930113aa42f47e0542418f6a67bdb2c23fe0e2fa1866f60b29280a036b829"},+ {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88d2792df548b845edd409a3e4284f76cb4fc2510fe4a69fde9e39d54910b935"},+ {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27622d5183185dc63bcab9a7dd1de566688c63b844812b1d9366da7c459a494"},+ {file = "pyreqwest_impersonate-0.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b7bf13d49ef127e659ed134129336e94f7107023ed0138c81a46321b9a580428"},+ {file = "pyreqwest_impersonate-0.4.7-cp310-none-win_amd64.whl", hash = "sha256:0cba006b076b85a875814a4b5dd8cb27f483ebeeb0de83984a3786060fe18e0d"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:370a8cb7a92b15749cbbe3ce7a9f09d35aac7d2a74505eb447f45419ea8ef2ff"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:33244ea10ccee08bac7a7ccdc3a8e6bef6e28f2466ed61de551fa24b76ee4b6a"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dba24fb6db822cbd9cbac32539893cc19cc06dd1820e03536e685b9fd2a2ffdd"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e001ed09fc364cc00578fd31c0ae44d543cf75daf06b2657c7a82dcd99336ce"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:608525535f078e85114fcd4eeba0f0771ffc7093c29208e9c0a55147502723bf"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38daedba0fc997e29cbc25c684a42a04aed38bfbcf85d8f1ffe8f87314d5f72f"},+ {file = "pyreqwest_impersonate-0.4.7-cp311-none-win_amd64.whl", hash = "sha256:d21f3e93ee0aecdc43d2914800bdf23501bde858d70ac7c0b06168f85f95bf22"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:5caeee29370a06a322ea6951730d21ec3c641ce46417fd2b5805b283564f2fef"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1c7aa4b428ed58370975d828a95eaf10561712e79a4e2eafca1746a4654a34a8"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:887249adcab35487a44a5428ccab2a6363642785b36649a732d5e649df568b8e"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60f932de8033c15323ba79a7470406ca8228e07aa60078dee5a18e89f0a9fc88"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a2e6332fd6d78623a22f4e747688fe9e6005b61b6f208936d5428d2a65d34b39"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:349b005eef323195685ba5cb2b6f302da0db481e59f03696ef57099f232f0c1f"},+ {file = "pyreqwest_impersonate-0.4.7-cp312-none-win_amd64.whl", hash = "sha256:5620025ac138a10c46a9b14c91b6f58114d50063ff865a2d02ad632751b67b29"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:ebf954e09b3dc800a7576c7bde9827b00064531364c7817356c7cc58eb4b46b2"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:112d9561f136548bd67d31cadb6b78d4c31751e526e62e09c6e581c2f1711455"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05213f5f014ecc6732d859a0f51b3dff0424748cc6e2d0d9a42aa1f7108b4eaa"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10fa70529a60fc043650ce03481fab7714e7519c3b06f5e81c95206b8b60aec6"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5b1288881eada1891db7e862c69b673fb159834a41f823b9b00fc52d0f096ccc"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:57ca562229c40615074f36e7f1ae5e57b8164f604eddb042132467c3a00fc2c5"},+ {file = "pyreqwest_impersonate-0.4.7-cp38-none-win_amd64.whl", hash = "sha256:c098ef1333511ea9a43be9a818fcc0866bd2caa63cdc9cf4ab48450ace675646"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:39d961330190bf2d59983ad16dafb4b42d5adcdfe7531ad099c8f3ab53f8d906"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0d793591784b89953422b1efaa17460f57f6116de25b3e3065d9fa6cf220ef18"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:945116bb9ffb7e45a87e313f47de28c4da889b14bda620aebc5ba9c3600425cf"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b96a0955c49f346786ee997c755561fecf33b7886cecef861fe4db15c7b23ad3"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ed997197f907ccce9b86a75163b5e78743bc469d2ddcf8a22d4d90c2595573cb"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1f54788f6fb0ee8b31c1eaadba81fb003efb406a768844e2a1a50b855f4806bf"},+ {file = "pyreqwest_impersonate-0.4.7-cp39-none-win_amd64.whl", hash = "sha256:0a679e81b0175dcc670a5ed47a5c184d7031ce16b5c58bf6b2c650ab9f2496c8"},+ {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bddb07e04e4006a2184608c44154983fdfa0ce2e230b0a7cec81cd4ba88dd07"},+ {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:780c53bfd2fbda151081165733fba5d5b1e17dd61999360110820942e351d011"},+ {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4bfa8ea763e6935e7660f8e885f1b00713b0d22f79a526c6ae6932b1856d1343"},+ {file = "pyreqwest_impersonate-0.4.7-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:96b23b0688a63cbd6c39237461baa95162a69a15e9533789163aabcaf3f572fb"},+ {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b0eb56a8ad9d48952c613903d3ef6d8762d48dcec9807a509fee2a43e94ccac"},+ {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9330176494e260521ea0eaae349ca06128dc527400248c57b378597c470d335c"},+ {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:6343bc3392781ff470e5dc47fea9f77bb61d8831b07e901900d31c46decec5d1"},+ {file = "pyreqwest_impersonate-0.4.7-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ecd598e16020a165029647ca80078311bf079e8317bf61c1b2fa824b8967e0db"},+ {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a38f3014ac31b08f5fb1ef4e1eb6c6e810f51f6cb815d0066ab3f34ec0f82d98"},+ {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db76a97068e5145f5b348037e09a91b2bed9c8eab92e79a3297b1306429fa839"},+ {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1596a8ef8f20bbfe606a90ad524946747846611c8633cbdfbad0a4298b538218"},+ {file = "pyreqwest_impersonate-0.4.7-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:dcee18bc350b3d3a0455422c446f1f03f00eb762b3e470066e2bc4664fd7110d"},+ {file = "pyreqwest_impersonate-0.4.7.tar.gz", hash = "sha256:74ba7e6e4f4f753da4f71a7e5dc12625b296bd7d6ddd64093a1fbff14d8d5df7"},+]++[package.extras]+dev = ["pytest (>=8.1.1)"]+[[package]]name = "pysocks"version = "1.7.1"@@ -4779,6 +4960,20 @@ urllib3 = ">=1.21.1,<3"socks = ["PySocks (>=1.5.6,!=1.5.7)"]use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]+[[package]]+name = "requests-file"+version = "2.1.0"+description = "File transport adapter for Requests"+optional = true+python-versions = "*"+files = [+ {file = "requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c"},+ {file = "requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658"},+]++[package.dependencies]+requests = ">=1.0.0"+[[package]]name = "requests-oauthlib"version = "1.3.1"@@ -5364,6 +5559,16 @@ files = [docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]+[[package]]+name = "sgmllib3k"+version = "1.0.0"+description = "Py3k port of sgmllib."+optional = true+python-versions = "*"+files = [+ {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"},+]+[[package]]name = "shellingham"version = "1.5.4"@@ -5727,6 +5932,37 @@ safetensors = "*"torch = "*"torchvision = "*"+[[package]]+name = "tinysegmenter"+version = "0.3"+description = "Very compact Japanese tokenizer"+optional = true+python-versions = "*"+files = [+ {file = "tinysegmenter-0.3.tar.gz", hash = "sha256:ed1f6d2e806a4758a73be589754384cbadadc7e1a414c81a166fc9adf2d40c6d"},+]++[[package]]+name = "tldextract"+version = "5.1.2"+description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well."+optional = true+python-versions = ">=3.8"+files = [+ {file = "tldextract-5.1.2-py3-none-any.whl", hash = "sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46"},+ {file = "tldextract-5.1.2.tar.gz", hash = "sha256:c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200"},+]++[package.dependencies]+filelock = ">=3.0.8"+idna = "*"+requests = ">=2.1.0"+requests-file = ">=1.4"++[package.extras]+release = ["build", "twine"]+testing = ["black", "mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "types-filelock", "types-requests"]+[[package]]name = "tokenizers"version = "0.19.1"@@ -7095,16 +7331,16 @@ doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linketest = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"][extras]-all = ["PyMuPDF", "accelerate", "beautifulsoup4", "cohere", "datasets", "diffusers", "docx2txt", "googlemaps", "neo4j", "openapi-spec-validator", "opencv-python", "pillow", "prance", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "slack-sdk", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]+all = ["PyMuPDF", "accelerate", "beautifulsoup4", "cohere", "datasets", "diffusers", "docx2txt", "duckduckgo-search", "googlemaps", "neo4j", "newspaper3k", "openapi-spec-validator", "opencv-python", "pillow", "prance", "pydub", "pygithub", "pymilvus", "pyowm", "qdrant-client", "rank-bm25", "requests_oauthlib", "sentence-transformers", "sentencepiece", "slack-sdk", "soundfile", "torch", "transformers", "unstructured", "wikipedia", "wolframalpha"]encoders = ["sentence-transformers"]graph-storages = ["neo4j"]huggingface-agent = ["accelerate", "datasets", "diffusers", "opencv-python", "sentencepiece", "soundfile", "torch", "transformers"]retrievers = ["cohere", "rank-bm25"]test = ["mock", "pytest", "pytest-asyncio"]-tools = ["PyMuPDF", "beautifulsoup4", "docx2txt", "googlemaps", "openapi-spec-validator", "pillow", "prance", "pydub", "pygithub", "pyowm", "requests_oauthlib", "slack-sdk", "unstructured", "wikipedia", "wolframalpha"]+tools = ["PyMuPDF", "beautifulsoup4", "docx2txt", "duckduckgo-search", "googlemaps", "newspaper3k", "openapi-spec-validator", "pillow", "prance", "pydub", "pygithub", "pyowm", "requests_oauthlib", "slack-sdk", "unstructured", "wikipedia", "wolframalpha"]vector-databases = ["pymilvus", "qdrant-client"][metadata]lock-version = "2.0"python-versions = ">=3.8.1,<3.12"-content-hash = "061fbd7b32abc78cfa857f9517b138b5e6b67cde96261f6f50fe8f27f5b3e71c"+content-hash = "30e5ed3148a38de70436af019c87f99516067ffdd2c8a3e172d723efed6a1402"diff --git a/pyproject.toml b/pyproject.tomlindex 7e9ba0ad1..994ec54c3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -38,6 +38,7 @@ pathlib = "^1.0.1"anthropic = "^0.21.3"docstring-parser = "^0.15"pydantic = ">=1.9,<3"+curl_cffi = "0.6.2"# huggingface-agenttransformers = { version = "^4", optional = true }@@ -54,6 +55,8 @@ beautifulsoup4 = { version = "^4", optional = true }docx2txt = { version = "^0.8", optional = true }PyMuPDF = { version = "^1.22.5", optional = true }wikipedia = { version = "^1", optional = true }+duckduckgo-search = { version = "^6.1.0", optional = true }+newspaper3k = {version = "^0.2.8", optional = true}wolframalpha = { version = "^5.0.0", optional = true }pyowm = { version = "^3.3.0", optional = true }googlemaps = { version = "^4.10.0", optional = true }@@ -109,6 +112,8 @@ tools = ["docx2txt","PyMuPDF","wikipedia",+ "duckduckgo-search",+ "newspaper3k","wolframalpha","pyowm","googlemaps",@@ -152,6 +157,8 @@ all = ["pygithub","PyMuPDF","wikipedia",+ "duckduckgo-search",+ "newspaper3k","wolframalpha","pyowm","googlemaps",@@ -270,6 +277,8 @@ module = ["huggingface_hub","huggingface_hub.utils._errors","wikipedia",+ "duckduckgo_search",+ "newspaper","wolframalpha","pyowm","googlemaps",diff --git a/test/functions/test_search_functions.py b/test/functions/test_search_functions.pyindex fb091a4e7..46f883b05 100644--- a/test/functions/test_search_functions.py+++ b/test/functions/test_search_functions.py@@ -19,7 +19,8 @@ import wikipediafrom camel.functions.search_functions import (query_wolfram_alpha,- search_google_and_summarize,+ search_duckduckgo,+ search_google,search_wiki,)@@ -67,10 +68,21 @@ def test_google_api():assert result.status_code == 200+def test_duckduckgo_api():+ # Test DuckDuckGo Instant Answer API++ url = "https://api.duckduckgo.com/"+ params = {"q": "test", "format": "json"}+ result = requests.get(url, params=params)++ assert result.status_code == 200++def test_web_search():query = "What big things are happening in 2023?"- answer = search_google_and_summarize(query)-+ answer = search_google(query)+ assert answer is not None+ answer = search_duckduckgo(query)assert answer is not None