🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add baidu search (#1856)
Neil Johnson62301476d0be
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.pyindex 389d32cce..176cda8e4 100644--- a/camel/toolkits/search_toolkit.py+++ b/camel/toolkits/search_toolkit.py@@ -766,6 +766,75 @@ class SearchToolkit(BaseToolkit):except requests.exceptions.RequestException as e:return {"error": f"Bocha AI search failed: {e!s}"}+ def search_baidu(self, query: str, max_results: int = 5) -> Dict[str, Any]:+ r"""Search Baidu using web scraping to retrieve relevant search+ results. This method queries Baidu's search engine and extracts search+ results including titles, descriptions, and URLs.++ Args:+ query (str): Search query string to submit to Baidu.+ max_results (int): Maximum number of results to return.+ (default: :obj:`5`)++ Returns:+ Dict[str, Any]: A dictionary containing search results or error+ message.+ """+ from bs4 import BeautifulSoup++ try:+ url = "https://www.baidu.com/s"+ headers = {+ "User-Agent": (+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+ "AppleWebKit/537.36 (KHTML, like Gecko) "+ "Chrome/120.0.0.0 Safari/537.36"+ ),+ "Referer": "https://www.baidu.com",+ }+ params = {"wd": query, "rn": str(max_results)}++ response = requests.get(url, headers=headers, params=params)+ response.encoding = "utf-8"++ soup = BeautifulSoup(response.text, "html.parser")++ results = []+ for idx, item in enumerate(soup.select(".result"), 1):+ title_element = item.select_one("h3 > a")+ title = (+ title_element.get_text(strip=True) if title_element else ""+ )++ link = title_element["href"] if title_element else ""++ desc_element = item.select_one(".c-abstract, .c-span-last")+ desc = (+ desc_element.get_text(strip=True) if desc_element else ""+ )++ results.append(+ {+ "result_id": idx,+ "title": title,+ "description": desc,+ "url": link,+ }+ )+ if len(results) >= max_results:+ break++ if not results:+ print(+ "Warning: No results found. Check "+ "if Baidu HTML structure has changed."+ )++ return {"results": results}++ except Exception as e:+ return {"error": f"Baidu scraping error: {e!s}"}+def get_tools(self) -> List[FunctionTool]:r"""Returns a list of FunctionTool objects representing thefunctions in the toolkit.@@ -783,4 +852,5 @@ class SearchToolkit(BaseToolkit):FunctionTool(self.tavily_search),FunctionTool(self.search_brave),FunctionTool(self.search_bocha),+ FunctionTool(self.search_baidu),]diff --git a/docs/key_modules/tools.md b/docs/key_modules/tools.mdindex 113103b75..7a39b15b4 100644--- a/docs/key_modules/tools.md+++ b/docs/key_modules/tools.md@@ -173,7 +173,7 @@ CAMEL provides a variety of built-in toolkits that you can use right away. Here'| OpenBBToolkit | A toolkit for accessing and analyzing financial market data through the OpenBB Platform, including stocks, ETFs, cryptocurrencies, and economic indicators. || RedditToolkit | A toolkit for Reddit operations including collecting top posts, performing sentiment analysis on comments, and tracking keyword discussions. || RetrievalToolkit | A toolkit for retrieving information from local vector storage systems based on specified queries. |-| SearchToolkit | A toolkit for performing web searches using various search engines like Google, DuckDuckGo, Wikipedia, and Wolfram Alpha. |+| SearchToolkit | A toolkit for performing web searches using various search engines like Google, DuckDuckGo, Wikipedia, Bing, BaiDu and Wolfram Alpha. || SemanticScholarToolkit | A toolkit for interacting with the Semantic Scholar API to fetch paper and author data from academic publications. || SlackToolkit | A toolkit for Slack operations including creating channels, joining channels, and managing channel membership. || StripeToolkit | A toolkit for processing payments and managing financial transactions via Stripe. |diff --git a/examples/toolkits/search_toolkit.py b/examples/toolkits/search_toolkit.pyindex 8c5dc9fa4..c3dd9d6a6 100644--- a/examples/toolkits/search_toolkit.py+++ b/examples/toolkits/search_toolkit.py@@ -17,6 +17,39 @@ from pydantic import BaseModelfrom camel.agents import ChatAgentfrom camel.toolkits import FunctionTool, SearchToolkit+agent = ChatAgent(+ system_message="""You are a helpful assistant that can use baidu search+ engine to answer questions.""",+ tools=[FunctionTool(SearchToolkit().search_baidu)],+)++usr_msg = "今天北京的天气如何"++response = agent.step(input_message=usr_msg, response_format=None)++print(response.msgs[0].content)++"""+===============================================================================+今天北京的天气信息可以通过以下链接查看:++1. [中国天气网 - 北京天气预报](http://www.baidu.com/link?+url=AJhE9PhEO3TmkJ70CUcRsR3NVB3m6wxN5Imdp0ZVsEBK1t8YhtM6YMxrQy3_vRN6dJv4FLHkBCe+fZURnzHTm9gio-dS4-4MwGVgJe40m7prOoggce2eB0h-3DsllbKMm)+2. [中国天气网 - 北京天气预报](http://www.baidu.com/link?+url=1vhNOfl9tV65_104GMQbDnU_fdCZPXDV2BtTJelxdd6isdSZjAHvtoXqOWG3n7D1N-m9zAmOhQG+c-jEGqiXe9K)+3. [中国天气网 - 北京天气预报](http://www.baidu.com/link?+url=Q0URfpodXDpUe1TKBPpToKIyIuCcjSGUR5jorx81g8Pni5XH-Tbc6AXMa7EwCWjBG3jysTZb43S+6ZCsJOKvPw2EbIlQ_bMu42-5sCraqXlS)+4. [中国天气网 - 北京天气预报一周](http://www.baidu.com/link?+url=TtFe8QryJFuwX1kx50YF5WijRcd2TMJRhPudDQvqW7TG4siah68gUZd_frsVWPi1xkYvrxoYL87+QMH0wSjDYOq)++请点击链接查看详细的天气预报信息。+===============================================================================+"""+res_simple = SearchToolkit().query_wolfram_alpha(query="solve 3x-7=11", is_detailed=False)diff --git a/pyproject.toml b/pyproject.tomlindex 0a2dea985..aee6fe8d3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -96,6 +96,7 @@ web_tools = ["fastapi>=0.115.11","playwright>=1.50.0","html2text>=2024.2.26",+ "beautifulsoup4>=4,<5",]document_tools = ["beautifulsoup4>=4,<5",diff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.pyindex 803c5850d..c68d2dcc2 100644--- a/test/toolkits/test_search_functions.py+++ b/test/toolkits/test_search_functions.py@@ -184,6 +184,70 @@ def test_search_duckduckgo_images():)+@patch('requests.get')+def test_search_baidu(mock_get, search_toolkit):+ # Mock the response from Baidu search+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.encoding = "utf-8"+ mock_response.text = """+ <html>+ <head><title>Baidu Search</title></head>+ <body>+ <div class="result c-container">+ <h3 class="t">+ <a href="https://example1.com">Test Title 1</a>+ </h3>+ <div class="c-abstract">Test Abstract 1</div>+ </div>+ <div class="result c-container">+ <h3 class="t">+ <a href="https://example2.com">Test Title 2</a>+ </h3>+ <div class="c-abstract">Test Abstract 2</div>+ </div>+ </body>+ </html>+ """+ mock_get.return_value = mock_response++ # Call the function under test+ result = search_toolkit.search_baidu(query="test query", max_results=5)++ # Expected output+ expected_output = {+ "results": [+ {+ "result_id": 1,+ "title": "Test Title 1",+ "description": "Test Abstract 1",+ "url": "https://example1.com",+ },+ {+ "result_id": 2,+ "title": "Test Title 2",+ "description": "Test Abstract 2",+ "url": "https://example2.com",+ },+ ]+ }++ # Assertions+ assert result == expected_output+ mock_get.assert_called_once_with(+ "https://www.baidu.com/s",+ headers={+ "User-Agent": (+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+ "AppleWebKit/537.36 (KHTML, like Gecko) "+ "Chrome/120.0.0.0 Safari/537.36"+ ),+ "Referer": "https://www.baidu.com",+ },+ params={"wd": "test query", "rn": "5"},+ )++@patch('requests.get')@patch('wolframalpha.Client')@patch('os.environ.get')diff --git a/uv.lock b/uv.lockindex d2660a77d..745afe3a1 100644--- a/uv.lock+++ b/uv.lock@@ -759,6 +759,7 @@ test = []web-tools = [{ name = "apify-client" },+ { name = "beautifulsoup4" },{ name = "dappier" },{ name = "duckduckgo-search" },{ name = "fastapi" },@@ -796,6 +797,7 @@ requires-dist = [{ name = "azure-storage-blob", marker = "extra == 'storage'", specifier = ">=12.21.0,<13" },{ name = "beautifulsoup4", marker = "extra == 'all'", specifier = ">=4,<5" },{ name = "beautifulsoup4", marker = "extra == 'document-tools'", specifier = ">=4,<5" },+ { name = "beautifulsoup4", marker = "extra == 'web-tools'", specifier = ">=4,<5" },{ name = "botocore", marker = "extra == 'all'", specifier = ">=1.35.3,<2" },{ name = "botocore", marker = "extra == 'storage'", specifier = ">=1.35.3,<2" },{ name = "cohere", marker = "extra == 'all'", specifier = ">=5.11.0,<6" },