Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add baidu search (#1856)

Neil Johnson62301476d0be
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.py
index 389d32cce..176cda8e4 100644
--- a/camel/toolkits/search_toolkit.py
+++ b/camel/toolkits/search_toolkit.py
@@ -766,6 +766,75 @@ class SearchToolkit(BaseToolkit):
except requests.exceptions.RequestException as e:
return {"error": f"Bocha AI search failed: {e!s}"}
+ def search_baidu(self, query: str, max_results: int = 5) -> Dict[str, Any]:
+ r"""Search Baidu using web scraping to retrieve relevant search
+ results. This method queries Baidu's search engine and extracts search
+ results including titles, descriptions, and URLs.
+
+ Args:
+ query (str): Search query string to submit to Baidu.
+ max_results (int): Maximum number of results to return.
+ (default: :obj:`5`)
+
+ Returns:
+ Dict[str, Any]: A dictionary containing search results or error
+ message.
+ """
+ from bs4 import BeautifulSoup
+
+ try:
+ url = "https://www.baidu.com/s"
+ headers = {
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/120.0.0.0 Safari/537.36"
+ ),
+ "Referer": "https://www.baidu.com",
+ }
+ params = {"wd": query, "rn": str(max_results)}
+
+ response = requests.get(url, headers=headers, params=params)
+ response.encoding = "utf-8"
+
+ soup = BeautifulSoup(response.text, "html.parser")
+
+ results = []
+ for idx, item in enumerate(soup.select(".result"), 1):
+ title_element = item.select_one("h3 > a")
+ title = (
+ title_element.get_text(strip=True) if title_element else ""
+ )
+
+ link = title_element["href"] if title_element else ""
+
+ desc_element = item.select_one(".c-abstract, .c-span-last")
+ desc = (
+ desc_element.get_text(strip=True) if desc_element else ""
+ )
+
+ results.append(
+ {
+ "result_id": idx,
+ "title": title,
+ "description": desc,
+ "url": link,
+ }
+ )
+ if len(results) >= max_results:
+ break
+
+ if not results:
+ print(
+ "Warning: No results found. Check "
+ "if Baidu HTML structure has changed."
+ )
+
+ return {"results": results}
+
+ except Exception as e:
+ return {"error": f"Baidu scraping error: {e!s}"}
+
def get_tools(self) -> List[FunctionTool]:
r"""Returns a list of FunctionTool objects representing the
functions in the toolkit.
@@ -783,4 +852,5 @@ class SearchToolkit(BaseToolkit):
FunctionTool(self.tavily_search),
FunctionTool(self.search_brave),
FunctionTool(self.search_bocha),
+ FunctionTool(self.search_baidu),
]
diff --git a/docs/key_modules/tools.md b/docs/key_modules/tools.md
index 113103b75..7a39b15b4 100644
--- a/docs/key_modules/tools.md
+++ b/docs/key_modules/tools.md
@@ -173,7 +173,7 @@ CAMEL provides a variety of built-in toolkits that you can use right away. Here'
| OpenBBToolkit | A toolkit for accessing and analyzing financial market data through the OpenBB Platform, including stocks, ETFs, cryptocurrencies, and economic indicators. |
| RedditToolkit | A toolkit for Reddit operations including collecting top posts, performing sentiment analysis on comments, and tracking keyword discussions. |
| RetrievalToolkit | A toolkit for retrieving information from local vector storage systems based on specified queries. |
-| SearchToolkit | A toolkit for performing web searches using various search engines like Google, DuckDuckGo, Wikipedia, and Wolfram Alpha. |
+| SearchToolkit | A toolkit for performing web searches using various search engines like Google, DuckDuckGo, Wikipedia, Bing, BaiDu and Wolfram Alpha. |
| SemanticScholarToolkit | A toolkit for interacting with the Semantic Scholar API to fetch paper and author data from academic publications. |
| SlackToolkit | A toolkit for Slack operations including creating channels, joining channels, and managing channel membership. |
| StripeToolkit | A toolkit for processing payments and managing financial transactions via Stripe. |
diff --git a/examples/toolkits/search_toolkit.py b/examples/toolkits/search_toolkit.py
index 8c5dc9fa4..c3dd9d6a6 100644
--- a/examples/toolkits/search_toolkit.py
+++ b/examples/toolkits/search_toolkit.py
@@ -17,6 +17,39 @@ from pydantic import BaseModel
from camel.agents import ChatAgent
from camel.toolkits import FunctionTool, SearchToolkit
+agent = ChatAgent(
+ system_message="""You are a helpful assistant that can use baidu search
+ engine to answer questions.""",
+ tools=[FunctionTool(SearchToolkit().search_baidu)],
+)
+
+usr_msg = "今天北京的天气如何"
+
+response = agent.step(input_message=usr_msg, response_format=None)
+
+print(response.msgs[0].content)
+
+"""
+===============================================================================
+今天北京的天气信息可以通过以下链接查看:
+
+1. [中国天气网 - 北京天气预报](http://www.baidu.com/link?
+url=AJhE9PhEO3TmkJ70CUcRsR3NVB3m6wxN5Imdp0ZVsEBK1t8YhtM6YMxrQy3_vRN6dJv4FLHkBCe
+fZURnzHTm9gio-dS4-4MwGVgJe40m7prOoggce2eB0h-3DsllbKMm)
+2. [中国天气网 - 北京天气预报](http://www.baidu.com/link?
+url=1vhNOfl9tV65_104GMQbDnU_fdCZPXDV2BtTJelxdd6isdSZjAHvtoXqOWG3n7D1N-m9zAmOhQG
+c-jEGqiXe9K)
+3. [中国天气网 - 北京天气预报](http://www.baidu.com/link?
+url=Q0URfpodXDpUe1TKBPpToKIyIuCcjSGUR5jorx81g8Pni5XH-Tbc6AXMa7EwCWjBG3jysTZb43S
+6ZCsJOKvPw2EbIlQ_bMu42-5sCraqXlS)
+4. [中国天气网 - 北京天气预报一周](http://www.baidu.com/link?
+url=TtFe8QryJFuwX1kx50YF5WijRcd2TMJRhPudDQvqW7TG4siah68gUZd_frsVWPi1xkYvrxoYL87
+QMH0wSjDYOq)
+
+请点击链接查看详细的天气预报信息。
+===============================================================================
+"""
+
res_simple = SearchToolkit().query_wolfram_alpha(
query="solve 3x-7=11", is_detailed=False
)
diff --git a/pyproject.toml b/pyproject.toml
index 0a2dea985..aee6fe8d3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -96,6 +96,7 @@ web_tools = [
"fastapi>=0.115.11",
"playwright>=1.50.0",
"html2text>=2024.2.26",
+ "beautifulsoup4>=4,<5",
]
document_tools = [
"beautifulsoup4>=4,<5",
diff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.py
index 803c5850d..c68d2dcc2 100644
--- a/test/toolkits/test_search_functions.py
+++ b/test/toolkits/test_search_functions.py
@@ -184,6 +184,70 @@ def test_search_duckduckgo_images():
)
+@patch('requests.get')
+def test_search_baidu(mock_get, search_toolkit):
+ # Mock the response from Baidu search
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.encoding = "utf-8"
+ mock_response.text = """
+ <html>
+ <head><title>Baidu Search</title></head>
+ <body>
+ <div class="result c-container">
+ <h3 class="t">
+ <a href="https://example1.com">Test Title 1</a>
+ </h3>
+ <div class="c-abstract">Test Abstract 1</div>
+ </div>
+ <div class="result c-container">
+ <h3 class="t">
+ <a href="https://example2.com">Test Title 2</a>
+ </h3>
+ <div class="c-abstract">Test Abstract 2</div>
+ </div>
+ </body>
+ </html>
+ """
+ mock_get.return_value = mock_response
+
+ # Call the function under test
+ result = search_toolkit.search_baidu(query="test query", max_results=5)
+
+ # Expected output
+ expected_output = {
+ "results": [
+ {
+ "result_id": 1,
+ "title": "Test Title 1",
+ "description": "Test Abstract 1",
+ "url": "https://example1.com",
+ },
+ {
+ "result_id": 2,
+ "title": "Test Title 2",
+ "description": "Test Abstract 2",
+ "url": "https://example2.com",
+ },
+ ]
+ }
+
+ # Assertions
+ assert result == expected_output
+ mock_get.assert_called_once_with(
+ "https://www.baidu.com/s",
+ headers={
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/120.0.0.0 Safari/537.36"
+ ),
+ "Referer": "https://www.baidu.com",
+ },
+ params={"wd": "test query", "rn": "5"},
+ )
+
+
@patch('requests.get')
@patch('wolframalpha.Client')
@patch('os.environ.get')
diff --git a/uv.lock b/uv.lock
index d2660a77d..745afe3a1 100644
--- a/uv.lock
+++ b/uv.lock
@@ -759,6 +759,7 @@ test = [
]
web-tools = [
{ name = "apify-client" },
+ { name = "beautifulsoup4" },
{ name = "dappier" },
{ name = "duckduckgo-search" },
{ name = "fastapi" },
@@ -796,6 +797,7 @@ requires-dist = [
{ name = "azure-storage-blob", marker = "extra == 'storage'", specifier = ">=12.21.0,<13" },
{ name = "beautifulsoup4", marker = "extra == 'all'", specifier = ">=4,<5" },
{ name = "beautifulsoup4", marker = "extra == 'document-tools'", specifier = ">=4,<5" },
+ { name = "beautifulsoup4", marker = "extra == 'web-tools'", specifier = ">=4,<5" },
{ name = "botocore", marker = "extra == 'all'", specifier = ">=1.35.3,<2" },
{ name = "botocore", marker = "extra == 'storage'", specifier = ">=1.35.3,<2" },
{ name = "cohere", marker = "extra == 'all'", specifier = ">=5.11.0,<6" },