Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

enhance: Add Serper.dev search support to SearchToolkit PR3539 (#3617)

Wendong-Fanf3da19fdb76a
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.py
index db577a2fa..7c3641424 100644
--- a/camel/toolkits/search_toolkit.py
+++ b/camel/toolkits/search_toolkit.py
@@ -63,14 +63,15 @@ class SearchToolkit(BaseToolkit):
def search_serper(
self,
query: str,
- page: int = 1,
+ page: int = 10,
location: str = "United States",
) -> Dict[str, Any]:
r"""Use Serper.dev API to perform Google search.
Args:
query (str): The search query.
- page (int): The page number of results to retrieve. (default: :obj:`1`)
+ page (int): The page number of results to retrieve.
+ (default: :obj:`10`)
location (str): The location for the search results.
(default: :obj:`"United States"`)
@@ -78,19 +79,15 @@ class SearchToolkit(BaseToolkit):
Dict[str, Any]: The search result dictionary containing 'organic',
'peopleAlsoAsk', etc.
"""
- import json
-
SERPER_API_KEY = os.getenv("SERPER_API_KEY")
url = "https://google.serper.dev/search"
- payload = json.dumps(
- {
- "q": query,
- "location": location,
- "page": page,
- }
- )
+ payload = {
+ "q": query,
+ "location": location,
+ "page": page,
+ }
headers = {
"X-API-KEY": SERPER_API_KEY,
@@ -98,11 +95,19 @@ class SearchToolkit(BaseToolkit):
}
try:
- response = requests.post(url, headers=headers, data=payload)
- response.raise_for_status()
+ response = requests.post(
+ url, headers=headers, json=payload, timeout=self.timeout
+ )
+ if response.status_code != 200:
+ return {
+ "error": (
+ f"Serper API failed with status {response.status_code}: "
+ f"{response.text}"
+ )
+ }
return response.json()
except requests.exceptions.RequestException as e:
- raise RuntimeError(f"Error making request to Serper: {e}")
+ return {"error": f"Serper search failed: {e!s}"}
@dependencies_required("wikipedia")
def search_wiki(self, entity: str) -> str:
@@ -458,7 +463,9 @@ class SearchToolkit(BaseToolkit):
}
params = {k: v for k, v in params.items() if v is not None}
- response = requests.get(url, headers=headers, params=params)
+ response = requests.get(
+ url, headers=headers, params=params, timeout=self.timeout
+ )
try:
response.raise_for_status()
except requests.HTTPError as e:
@@ -628,7 +635,7 @@ class SearchToolkit(BaseToolkit):
# Fetch the results given the URL
try:
# Make the get
- result = requests.get(url)
+ result = requests.get(url, timeout=self.timeout)
data = result.json()
# Get the result items
@@ -834,7 +841,9 @@ class SearchToolkit(BaseToolkit):
ensure_ascii=False,
)
try:
- response = requests.post(url, headers=headers, data=payload)
+ response = requests.post(
+ url, headers=headers, data=payload, timeout=self.timeout
+ )
if response.status_code != 200:
return {
"error": (
@@ -878,7 +887,9 @@ class SearchToolkit(BaseToolkit):
}
params = {"wd": query, "rn": str(number_of_result_pages)}
- response = requests.get(url, headers=headers, params=params)
+ response = requests.get(
+ url, headers=headers, params=params, timeout=self.timeout
+ )
response.encoding = "utf-8"
soup = BeautifulSoup(response.text, "html.parser")
@@ -962,8 +973,7 @@ class SearchToolkit(BaseToolkit):
"Chrome/120.0.0.0 Safari/537.36"
),
}
- # Add timeout to prevent hanging
- response = requests.get(url, headers=headers, timeout=10)
+ response = requests.get(url, headers=headers, timeout=self.timeout)
# Check if the request was successful
if response.status_code != 200:
@@ -1248,7 +1258,7 @@ class SearchToolkit(BaseToolkit):
try:
# Send GET request with proper typing for params
response = requests.get(
- base_url, headers=headers, params=params, timeout=10
+ base_url, headers=headers, params=params, timeout=self.timeout
)
# Check response status
diff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.py
index 596ecf957..3eccea78f 100644
--- a/test/toolkits/test_search_functions.py
+++ b/test/toolkits/test_search_functions.py
@@ -508,18 +508,19 @@ def test_search_baidu(mock_get, search_toolkit):
# Assertions
assert result == expected_output
- mock_get.assert_called_once_with(
- "https://www.baidu.com/s",
- headers={
- "User-Agent": (
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
- "AppleWebKit/537.36 (KHTML, like Gecko) "
- "Chrome/120.0.0.0 Safari/537.36"
- ),
- "Referer": "https://www.baidu.com",
- },
- params={"wd": "test query", "rn": "10"},
- )
+ mock_get.assert_called_once()
+ args, kwargs = mock_get.call_args
+ assert args[0] == "https://www.baidu.com/s"
+ assert kwargs['headers'] == {
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/120.0.0.0 Safari/537.36"
+ ),
+ "Referer": "https://www.baidu.com",
+ }
+ assert kwargs['params'] == {"wd": "test query", "rn": "10"}
+ assert 'timeout' in kwargs
@patch('requests.get')
@@ -570,17 +571,17 @@ def test_search_bing(mock_get, search_toolkit):
# Assertions
assert result == expected_output
- mock_get.assert_called_once_with(
- "https://cn.bing.com/search?q=test+query",
- headers={
- "User-Agent": (
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
- "AppleWebKit/537.36 (KHTML, like Gecko) "
- "Chrome/120.0.0.0 Safari/537.36"
- ),
- },
- timeout=10,
- )
+ mock_get.assert_called_once()
+ args, kwargs = mock_get.call_args
+ assert args[0] == "https://cn.bing.com/search?q=test+query"
+ assert kwargs['headers'] == {
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/120.0.0.0 Safari/537.36"
+ ),
+ }
+ assert 'timeout' in kwargs
class MockSearchResult:
@@ -750,19 +751,19 @@ def test_search_alibaba_tongxiao(mock_get, search_toolkit):
)
# Verify the request was made correctly
- mock_get.assert_called_once_with(
- "https://cloud-iqs.aliyuncs.com/search/genericSearch",
- headers={"X-API-Key": "fake_api_key"},
- params={
- "query": "test query",
- "timeRange": "NoLimit",
- "page": 10,
- "returnMainText": "false",
- "returnMarkdownText": "true",
- "enableRerank": "true",
- },
- timeout=10,
- )
+ mock_get.assert_called_once()
+ args, kwargs = mock_get.call_args
+ assert args[0] == "https://cloud-iqs.aliyuncs.com/search/genericSearch"
+ assert kwargs['headers'] == {"X-API-Key": "fake_api_key"}
+ assert kwargs['params'] == {
+ "query": "test query",
+ "timeRange": "NoLimit",
+ "page": 10,
+ "returnMainText": "false",
+ "returnMarkdownText": "true",
+ "enableRerank": "true",
+ }
+ assert 'timeout' in kwargs
# Check if the result is as expected
assert result == {
@@ -904,8 +905,6 @@ def test_search_metaso_invalid_json(mock_https_connection, search_toolkit):
@patch('requests.post')
def test_search_serper_success(mock_post, search_toolkit):
"""Test successful Serper search."""
- import json
-
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
@@ -940,22 +939,39 @@ def test_search_serper_success(mock_post, search_toolkit):
"X-API-KEY": "test_key",
"Content-Type": "application/json",
}
- # Verify payload structure matches exactly what user requested
- # Note: requests.post(json=payload) sets data to json dump
- assert json.loads(kwargs['data']) == {
+ # Verify payload structure - using json= parameter now
+ assert kwargs['json'] == {
"q": "apple inc",
"location": "United States",
- "page": 1,
+ "page": 10,
}
+ # Verify timeout is passed
+ assert 'timeout' in kwargs
@patch('requests.post')
-def test_search_serper_error(mock_post, search_toolkit):
- """Test error handling in Serper search."""
+def test_search_serper_request_error(mock_post, search_toolkit):
+ """Test request error handling in Serper search."""
mock_post.side_effect = requests.exceptions.RequestException("API Error")
with patch.dict(os.environ, {'SERPER_API_KEY': 'test_key'}):
- with pytest.raises(RuntimeError) as excinfo:
- search_toolkit.search_serper(query="test")
+ result = search_toolkit.search_serper(query="test")
+
+ assert "error" in result
+ assert "Serper search failed" in result["error"]
+
- assert "Error making request to Serper" in str(excinfo.value)
+@patch('requests.post')
+def test_search_serper_http_error(mock_post, search_toolkit):
+ """Test HTTP error handling in Serper search."""
+ mock_response = MagicMock()
+ mock_response.status_code = 401
+ mock_response.text = '{"error": "Invalid API key"}'
+ mock_post.return_value = mock_response
+
+ with patch.dict(os.environ, {'SERPER_API_KEY': 'invalid_key'}):
+ result = search_toolkit.search_serper(query="test")
+
+ assert "error" in result
+ assert "Serper API failed with status 401" in result["error"]
+ assert "Invalid API key" in result["error"]