🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
enhance: Add Serper.dev search support to SearchToolkit PR3539 (#3617)
Wendong-Fanf3da19fdb76a
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.pyindex db577a2fa..7c3641424 100644--- a/camel/toolkits/search_toolkit.py+++ b/camel/toolkits/search_toolkit.py@@ -63,14 +63,15 @@ class SearchToolkit(BaseToolkit):def search_serper(self,query: str,- page: int = 1,+ page: int = 10,location: str = "United States",) -> Dict[str, Any]:r"""Use Serper.dev API to perform Google search.Args:query (str): The search query.- page (int): The page number of results to retrieve. (default: :obj:`1`)+ page (int): The page number of results to retrieve.+ (default: :obj:`10`)location (str): The location for the search results.(default: :obj:`"United States"`)@@ -78,19 +79,15 @@ class SearchToolkit(BaseToolkit):Dict[str, Any]: The search result dictionary containing 'organic','peopleAlsoAsk', etc."""- import json-SERPER_API_KEY = os.getenv("SERPER_API_KEY")url = "https://google.serper.dev/search"- payload = json.dumps(- {- "q": query,- "location": location,- "page": page,- }- )+ payload = {+ "q": query,+ "location": location,+ "page": page,+ }headers = {"X-API-KEY": SERPER_API_KEY,@@ -98,11 +95,19 @@ class SearchToolkit(BaseToolkit):}try:- response = requests.post(url, headers=headers, data=payload)- response.raise_for_status()+ response = requests.post(+ url, headers=headers, json=payload, timeout=self.timeout+ )+ if response.status_code != 200:+ return {+ "error": (+ f"Serper API failed with status {response.status_code}: "+ f"{response.text}"+ )+ }return response.json()except requests.exceptions.RequestException as e:- raise RuntimeError(f"Error making request to Serper: {e}")+ return {"error": f"Serper search failed: {e!s}"}@dependencies_required("wikipedia")def search_wiki(self, entity: str) -> str:@@ -458,7 +463,9 @@ class SearchToolkit(BaseToolkit):}params = {k: v for k, v in params.items() if v is not None}- response = requests.get(url, headers=headers, params=params)+ response = requests.get(+ url, headers=headers, params=params, timeout=self.timeout+ )try:response.raise_for_status()except requests.HTTPError as e:@@ -628,7 +635,7 @@ class SearchToolkit(BaseToolkit):# Fetch the results given the URLtry:# Make the get- result = requests.get(url)+ result = requests.get(url, timeout=self.timeout)data = result.json()# Get the result items@@ -834,7 +841,9 @@ class SearchToolkit(BaseToolkit):ensure_ascii=False,)try:- response = requests.post(url, headers=headers, data=payload)+ response = requests.post(+ url, headers=headers, data=payload, timeout=self.timeout+ )if response.status_code != 200:return {"error": (@@ -878,7 +887,9 @@ class SearchToolkit(BaseToolkit):}params = {"wd": query, "rn": str(number_of_result_pages)}- response = requests.get(url, headers=headers, params=params)+ response = requests.get(+ url, headers=headers, params=params, timeout=self.timeout+ )response.encoding = "utf-8"soup = BeautifulSoup(response.text, "html.parser")@@ -962,8 +973,7 @@ class SearchToolkit(BaseToolkit):"Chrome/120.0.0.0 Safari/537.36"),}- # Add timeout to prevent hanging- response = requests.get(url, headers=headers, timeout=10)+ response = requests.get(url, headers=headers, timeout=self.timeout)# Check if the request was successfulif response.status_code != 200:@@ -1248,7 +1258,7 @@ class SearchToolkit(BaseToolkit):try:# Send GET request with proper typing for paramsresponse = requests.get(- base_url, headers=headers, params=params, timeout=10+ base_url, headers=headers, params=params, timeout=self.timeout)# Check response statusdiff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.pyindex 596ecf957..3eccea78f 100644--- a/test/toolkits/test_search_functions.py+++ b/test/toolkits/test_search_functions.py@@ -508,18 +508,19 @@ def test_search_baidu(mock_get, search_toolkit):# Assertionsassert result == expected_output- mock_get.assert_called_once_with(- "https://www.baidu.com/s",- headers={- "User-Agent": (- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "- "AppleWebKit/537.36 (KHTML, like Gecko) "- "Chrome/120.0.0.0 Safari/537.36"- ),- "Referer": "https://www.baidu.com",- },- params={"wd": "test query", "rn": "10"},- )+ mock_get.assert_called_once()+ args, kwargs = mock_get.call_args+ assert args[0] == "https://www.baidu.com/s"+ assert kwargs['headers'] == {+ "User-Agent": (+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+ "AppleWebKit/537.36 (KHTML, like Gecko) "+ "Chrome/120.0.0.0 Safari/537.36"+ ),+ "Referer": "https://www.baidu.com",+ }+ assert kwargs['params'] == {"wd": "test query", "rn": "10"}+ assert 'timeout' in kwargs@patch('requests.get')@@ -570,17 +571,17 @@ def test_search_bing(mock_get, search_toolkit):# Assertionsassert result == expected_output- mock_get.assert_called_once_with(- "https://cn.bing.com/search?q=test+query",- headers={- "User-Agent": (- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "- "AppleWebKit/537.36 (KHTML, like Gecko) "- "Chrome/120.0.0.0 Safari/537.36"- ),- },- timeout=10,- )+ mock_get.assert_called_once()+ args, kwargs = mock_get.call_args+ assert args[0] == "https://cn.bing.com/search?q=test+query"+ assert kwargs['headers'] == {+ "User-Agent": (+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "+ "AppleWebKit/537.36 (KHTML, like Gecko) "+ "Chrome/120.0.0.0 Safari/537.36"+ ),+ }+ assert 'timeout' in kwargsclass MockSearchResult:@@ -750,19 +751,19 @@ def test_search_alibaba_tongxiao(mock_get, search_toolkit):)# Verify the request was made correctly- mock_get.assert_called_once_with(- "https://cloud-iqs.aliyuncs.com/search/genericSearch",- headers={"X-API-Key": "fake_api_key"},- params={- "query": "test query",- "timeRange": "NoLimit",- "page": 10,- "returnMainText": "false",- "returnMarkdownText": "true",- "enableRerank": "true",- },- timeout=10,- )+ mock_get.assert_called_once()+ args, kwargs = mock_get.call_args+ assert args[0] == "https://cloud-iqs.aliyuncs.com/search/genericSearch"+ assert kwargs['headers'] == {"X-API-Key": "fake_api_key"}+ assert kwargs['params'] == {+ "query": "test query",+ "timeRange": "NoLimit",+ "page": 10,+ "returnMainText": "false",+ "returnMarkdownText": "true",+ "enableRerank": "true",+ }+ assert 'timeout' in kwargs# Check if the result is as expectedassert result == {@@ -904,8 +905,6 @@ def test_search_metaso_invalid_json(mock_https_connection, search_toolkit):@patch('requests.post')def test_search_serper_success(mock_post, search_toolkit):"""Test successful Serper search."""- import json-mock_response = MagicMock()mock_response.status_code = 200mock_response.json.return_value = {@@ -940,22 +939,39 @@ def test_search_serper_success(mock_post, search_toolkit):"X-API-KEY": "test_key","Content-Type": "application/json",}- # Verify payload structure matches exactly what user requested- # Note: requests.post(json=payload) sets data to json dump- assert json.loads(kwargs['data']) == {+ # Verify payload structure - using json= parameter now+ assert kwargs['json'] == {"q": "apple inc","location": "United States",- "page": 1,+ "page": 10,}+ # Verify timeout is passed+ assert 'timeout' in kwargs@patch('requests.post')-def test_search_serper_error(mock_post, search_toolkit):- """Test error handling in Serper search."""+def test_search_serper_request_error(mock_post, search_toolkit):+ """Test request error handling in Serper search."""mock_post.side_effect = requests.exceptions.RequestException("API Error")with patch.dict(os.environ, {'SERPER_API_KEY': 'test_key'}):- with pytest.raises(RuntimeError) as excinfo:- search_toolkit.search_serper(query="test")+ result = search_toolkit.search_serper(query="test")++ assert "error" in result+ assert "Serper search failed" in result["error"]+- assert "Error making request to Serper" in str(excinfo.value)+@patch('requests.post')+def test_search_serper_http_error(mock_post, search_toolkit):+ """Test HTTP error handling in Serper search."""+ mock_response = MagicMock()+ mock_response.status_code = 401+ mock_response.text = '{"error": "Invalid API key"}'+ mock_post.return_value = mock_response++ with patch.dict(os.environ, {'SERPER_API_KEY': 'invalid_key'}):+ result = search_toolkit.search_serper(query="test")++ assert "error" in result+ assert "Serper API failed with status 401" in result["error"]+ assert "Invalid API key" in result["error"]