🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
Fix:search google (#2979)
Puzhen Zhang2d90236fcf44
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.pyindex 8a226db78..4e3c1585d 100644--- a/camel/toolkits/search_toolkit.py+++ b/camel/toolkits/search_toolkit.py@@ -13,6 +13,7 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import osfrom typing import Any, Dict, List, Literal, Optional, TypeAlias, Union, cast+from urllib.parse import quoteimport requests@@ -516,6 +517,29 @@ class SearchToolkit(BaseToolkit):"""import requests+ # Validate input parameters+ if not isinstance(start_page, int) or start_page < 1:+ raise ValueError("start_page must be a positive integer")++ if (+ not isinstance(number_of_result_pages, int)+ or number_of_result_pages < 1+ ):+ raise ValueError(+ "number_of_result_pages must be a positive integer"+ )++ # Google Custom Search API has a limit of 10 results per request+ if number_of_result_pages > 10:+ logger.warning(+ f"Google API limits results to 10 per request. "+ f"Requested {number_of_result_pages}, using 10 instead."+ )+ number_of_result_pages = 10++ if search_type not in ["web", "image"]:+ raise ValueError("search_type must be either 'web' or 'image'")+# https://developers.google.com/custom-search/v1/overviewGOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")# https://cse.google.com/cse/all@@ -535,11 +559,13 @@ class SearchToolkit(BaseToolkit):modified_query = f"{query} {exclusion_terms}"logger.debug(f"Excluded domains, modified query: {modified_query}")+ encoded_query = quote(modified_query)+# Constructing the URL# Doc: https://developers.google.com/custom-search/v1/using_restbase_url = (f"https://www.googleapis.com/customsearch/v1?"- f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={modified_query}&start="+ f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={encoded_query}&start="f"{start_page_idx}&lr={search_language}&num={number_of_result_pages}")@@ -552,8 +578,8 @@ class SearchToolkit(BaseToolkit):responses = []# Fetch the results given the URLtry:- # Make the get- result = requests.get(url)+ result = requests.get(url, timeout=30)+ result.raise_for_status() # Raise exception for bad status codesdata = result.json()# Get the result items@@ -592,28 +618,31 @@ class SearchToolkit(BaseToolkit):responses.append(response)else:- # Process web search results (existing logic)- # Check metatags are present- if "pagemap" not in search_item:- continue- if "metatags" not in search_item["pagemap"]:- continue+ title = search_item.get("title", "")+ snippet = search_item.get("snippet", "")+ link = search_item.get("link", "")++ long_description = "N/A"if (- "og:description"- in search_item["pagemap"]["metatags"][0]+ "pagemap" in search_item+ and "metatags" in search_item["pagemap"]):- long_description = search_item["pagemap"][- "metatags"- ][0]["og:description"]- else:- long_description = "N/A"- # Get the page title- title = search_item.get("title")- # Page snippet- snippet = search_item.get("snippet")+ metatags = search_item["pagemap"]["metatags"]+ if metatags and len(metatags) > 0:+ meta = metatags[0]+ long_description = (+ meta.get("og:description")+ or meta.get("description")+ or meta.get("twitter:description")+ or "N/A"+ )++ if not title and not snippet and not link:+ logger.debug(+ f"Skipping result {i} due to missing essential fields"+ )+ continue- # Extract the page url- link = search_item.get("link")response = {"result_id": i,"title": title,@@ -623,19 +652,74 @@ class SearchToolkit(BaseToolkit):}responses.append(response)else:+ logger.debug(f"Google API response without items: {data}")error_info = data.get("error", {})- logger.error(- f"Google search failed - API response: {error_info}"- )+ if error_info:+ error_code = error_info.get("code", "Unknown")+ error_message = error_info.get("message", "Unknown error")+ error_details = f"Error {error_code}: {error_message}"+ else:+ error_details = (+ "No results found or API returned empty response"+ )++ logger.error(f"Google search failed - {error_details}")responses.append(- {- "error": f"Google search failed - "- f"API response: {error_info}"- }+ {"error": f"Google search failed - {error_details}"})+ except requests.exceptions.Timeout:+ logger.error("Google search request timed out")+ responses.append(+ {"error": "Google search request timed out after 30 seconds"}+ )+ except requests.exceptions.HTTPError as e:+ error_message = f"Google API HTTP error: {e}"+ try:+ error_data = e.response.json()+ if "error" in error_data and "message" in error_data["error"]:+ api_error_message = error_data["error"]["message"]+ error_code = error_data["error"].get(+ "code", e.response.status_code+ )+ error_message = (+ f"Google API error {error_code}: {api_error_message}"+ )++ if e.response.status_code == 429:+ error_message = f"Google API rate limit exceeded: {api_error_message}"+ elif e.response.status_code == 400:+ if "API key" in api_error_message:+ error_message = (+ f"Invalid Google API key: {api_error_message}"+ )+ else:+ error_message = f"Bad request to Google API: {api_error_message}"+ except (ValueError, KeyError):+ if e.response.status_code == 429:+ error_message = "Google API rate limit exceeded. Please try again later."+ elif e.response.status_code == 400:+ error_message = "Google API bad request. Please check your API key and search engine ID."+ elif e.response.status_code == 403:+ error_message = "Google API access forbidden. Please check your API key permissions."++ logger.error(error_message)+ responses.append({"error": error_message})+ except requests.exceptions.RequestException as e:+ logger.error(f"Google search network error: {e}")+ responses.append(+ {"error": f"Network error during Google search: {e!s}"}+ )+ except ValueError as e:+ logger.error(f"Google search response parsing error: {e}")+ responses.append(+ {"error": f"Failed to parse Google search response: {e!s}"}+ )except Exception as e:- responses.append({"error": f"google search failed: {e!s}"})+ logger.error(f"Unexpected error during Google search: {e}")+ responses.append(+ {"error": f"Unexpected error during Google search: {e!s}"}+ )return responsesdef tavily_search(diff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.pyindex 1be2e2e90..861c13d63 100644--- a/test/toolkits/test_search_functions.py+++ b/test/toolkits/test_search_functions.py@@ -90,6 +90,339 @@ def test_google_api():assert result.status_code == 200+@patch('requests.get')+def test_search_google_web_success(mock_get, search_toolkit):+ """Test successful Google web search."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "items": [+ {+ "title": "Test Title 1",+ "snippet": "Test snippet 1",+ "link": "https://example1.com",+ "pagemap": {+ "metatags": [{"og:description": "Test long description 1"}]+ },+ },+ {+ "title": "Test Title 2",+ "snippet": "Test snippet 2",+ "link": "https://example2.com",+ "pagemap": {+ "metatags": [{"description": "Test long description 2"}]+ },+ },+ ]+ }+ mock_get.return_value = mock_response++ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ result = search_toolkit.search_google(+ query="test query", number_of_result_pages=2+ )++ assert len(result) == 2+ assert result[0] == {+ "result_id": 1,+ "title": "Test Title 1",+ "description": "Test snippet 1",+ "long_description": "Test long description 1",+ "url": "https://example1.com",+ }+ assert result[1] == {+ "result_id": 2,+ "title": "Test Title 2",+ "description": "Test snippet 2",+ "long_description": "Test long description 2",+ "url": "https://example2.com",+ }++ # Verify the request was made with correct parameters+ mock_get.assert_called_once()+ call_args = mock_get.call_args[0][0]+ assert "q=test%20query" in call_args+ assert "num=2" in call_args+++@patch('requests.get')+def test_search_google_image_success(mock_get, search_toolkit):+ """Test successful Google image search."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "items": [+ {+ "title": "Image Title 1",+ "link": "https://example.com/image1.jpg",+ "displayLink": "example.com",+ "image": {+ "contextLink": "https://example.com/page1.html",+ "width": 800,+ "height": 600,+ },+ }+ ]+ }+ mock_get.return_value = mock_response++ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ result = search_toolkit.search_google(+ query="test image", search_type="image", number_of_result_pages=1+ )++ assert len(result) == 1+ assert result[0] == {+ "result_id": 1,+ "title": "Image Title 1",+ "image_url": "https://example.com/image1.jpg",+ "display_link": "example.com",+ "context_url": "https://example.com/page1.html",+ "width": 800,+ "height": 600,+ }++ # Verify searchType parameter was included+ call_args = mock_get.call_args[0][0]+ assert "searchType=image" in call_args+++@patch('requests.get')+def test_search_google_no_results(mock_get, search_toolkit):+ """Test Google search with no results."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "searchInformation": {"totalResults": "0"}+ }+ mock_get.return_value = mock_response++ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ result = search_toolkit.search_google(query="very specific query")++ assert len(result) == 1+ assert "error" in result[0]+ assert "No results found" in result[0]["error"]+++def test_search_google_parameter_validation(search_toolkit):+ """Test parameter validation for Google search."""+ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ # Test invalid start_page+ with pytest.raises(+ ValueError, match="start_page must be a positive integer"+ ):+ search_toolkit.search_google(query="test", start_page=0)++ with pytest.raises(+ ValueError, match="start_page must be a positive integer"+ ):+ search_toolkit.search_google(query="test", start_page=-1)++ # Test invalid number_of_result_pages+ with pytest.raises(+ ValueError,+ match="number_of_result_pages must be a positive integer",+ ):+ search_toolkit.search_google(+ query="test", number_of_result_pages=0+ )++ # Test invalid search_type+ with pytest.raises(+ ValueError, match="search_type must be either 'web' or 'image'"+ ):+ search_toolkit.search_google(query="test", search_type="invalid")+++@patch('requests.get')+def test_search_google_error_handling(mock_get, search_toolkit):+ """Test error handling in Google search."""+ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ # Test timeout error+ mock_get.side_effect = requests.exceptions.Timeout()+ result = search_toolkit.search_google(query="test")+ assert len(result) == 1+ assert "error" in result[0]+ assert "timed out" in result[0]["error"]++ # Test HTTP 429 error (rate limit) with API message+ mock_response = MagicMock()+ mock_error_response = MagicMock()+ mock_error_response.status_code = 429+ mock_error_response.json.return_value = {+ "error": {+ "code": 429,+ "message": "Rate limit exceeded. Try again in 1 minute.",+ }+ }+ mock_response.raise_for_status.side_effect = (+ requests.exceptions.HTTPError(response=mock_error_response)+ )+ mock_get.side_effect = None+ mock_get.return_value = mock_response+ result = search_toolkit.search_google(query="test")+ assert "Google API rate limit exceeded" in result[0]["error"]+ assert "Try again in 1 minute" in result[0]["error"]++ # Test HTTP 400 error (bad request) with API key error+ mock_error_response.status_code = 400+ mock_error_response.json.return_value = {+ "error": {+ "code": 400,+ "message": "API key not valid. Please pass a valid API key.",+ }+ }+ mock_response.raise_for_status.side_effect = (+ requests.exceptions.HTTPError(response=mock_error_response)+ )+ result = search_toolkit.search_google(query="test")+ assert "Invalid Google API key" in result[0]["error"]+ assert "API key not valid" in result[0]["error"]++ # Test HTTP 400 error (bad request) without API key error+ mock_error_response.json.return_value = {+ "error": {+ "code": 400,+ "message": "Request contains an invalid argument.",+ }+ }+ result = search_toolkit.search_google(query="test")+ assert "Bad request to Google API" in result[0]["error"]+ assert "invalid argument" in result[0]["error"]++ # Test HTTP 403 error (forbidden) - fallback when no JSON+ mock_error_response.status_code = 403+ mock_error_response.json.side_effect = ValueError("No JSON")+ mock_response.raise_for_status.side_effect = (+ requests.exceptions.HTTPError(response=mock_error_response)+ )+ result = search_toolkit.search_google(query="test")+ assert "access forbidden" in result[0]["error"]+++@patch('requests.get')+def test_search_google_with_exclude_domains(mock_get):+ """Test Google search with excluded domains."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"items": []}+ mock_get.return_value = mock_response++ # Create toolkit with exclude_domains+ toolkit_with_exclusions = SearchToolkit(+ exclude_domains=["example.com", "test.com"]+ )++ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ toolkit_with_exclusions.search_google(query="test query")++ # Verify the query includes exclusion terms+ call_args = mock_get.call_args[0][0]+ assert "test%20query%20-site%3Aexample.com%20-site%3Atest.com" in call_args+++@patch('requests.get')+def test_search_google_special_characters(mock_get, search_toolkit):+ """Test Google search with special characters in query."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"items": []}+ mock_get.return_value = mock_response++ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ search_toolkit.search_google(query="C++ & Python programming")++ # Verify special characters are properly encoded+ call_args = mock_get.call_args[0][0]+ assert "C%2B%2B%20%26%20Python%20programming" in call_args+++@patch('requests.get')+def test_search_google_without_metatags(mock_get, search_toolkit):+ """Test Google search handling results without metatags."""+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "items": [+ {+ "title": "Test Title Without Metatags",+ "snippet": "Test snippet without metatags",+ "link": "https://example.com/no-metatags",+ # No pagemap/metatags+ },+ {+ "title": "Test Title With Twitter Description",+ "snippet": "Test snippet with twitter description",+ "link": "https://example.com/twitter",+ "pagemap": {+ "metatags": [+ {"twitter:description": "Twitter description only"}+ ]+ },+ },+ ]+ }+ mock_get.return_value = mock_response++ with patch.dict(+ os.environ,+ {+ 'GOOGLE_API_KEY': 'fake_api_key',+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',+ },+ ):+ result = search_toolkit.search_google(query="test query")++ assert len(result) == 2+ # First result should have N/A for long_description+ assert result[0]["long_description"] == "N/A"+ # Second result should use twitter:description+ assert result[1]["long_description"] == "Twitter description only"++search_duckduckgo = SearchToolkit().search_duckduckgo