Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

Fix:search google (#2979)

Puzhen Zhang2d90236fcf44
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.py
index 8a226db78..4e3c1585d 100644
--- a/camel/toolkits/search_toolkit.py
+++ b/camel/toolkits/search_toolkit.py
@@ -13,6 +13,7 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import os
from typing import Any, Dict, List, Literal, Optional, TypeAlias, Union, cast
+from urllib.parse import quote
import requests
@@ -516,6 +517,29 @@ class SearchToolkit(BaseToolkit):
"""
import requests
+ # Validate input parameters
+ if not isinstance(start_page, int) or start_page < 1:
+ raise ValueError("start_page must be a positive integer")
+
+ if (
+ not isinstance(number_of_result_pages, int)
+ or number_of_result_pages < 1
+ ):
+ raise ValueError(
+ "number_of_result_pages must be a positive integer"
+ )
+
+ # Google Custom Search API has a limit of 10 results per request
+ if number_of_result_pages > 10:
+ logger.warning(
+ f"Google API limits results to 10 per request. "
+ f"Requested {number_of_result_pages}, using 10 instead."
+ )
+ number_of_result_pages = 10
+
+ if search_type not in ["web", "image"]:
+ raise ValueError("search_type must be either 'web' or 'image'")
+
# https://developers.google.com/custom-search/v1/overview
GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
# https://cse.google.com/cse/all
@@ -535,11 +559,13 @@ class SearchToolkit(BaseToolkit):
modified_query = f"{query} {exclusion_terms}"
logger.debug(f"Excluded domains, modified query: {modified_query}")
+ encoded_query = quote(modified_query)
+
# Constructing the URL
# Doc: https://developers.google.com/custom-search/v1/using_rest
base_url = (
f"https://www.googleapis.com/customsearch/v1?"
- f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={modified_query}&start="
+ f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={encoded_query}&start="
f"{start_page_idx}&lr={search_language}&num={number_of_result_pages}"
)
@@ -552,8 +578,8 @@ class SearchToolkit(BaseToolkit):
responses = []
# Fetch the results given the URL
try:
- # Make the get
- result = requests.get(url)
+ result = requests.get(url, timeout=30)
+ result.raise_for_status() # Raise exception for bad status codes
data = result.json()
# Get the result items
@@ -592,28 +618,31 @@ class SearchToolkit(BaseToolkit):
responses.append(response)
else:
- # Process web search results (existing logic)
- # Check metatags are present
- if "pagemap" not in search_item:
- continue
- if "metatags" not in search_item["pagemap"]:
- continue
+ title = search_item.get("title", "")
+ snippet = search_item.get("snippet", "")
+ link = search_item.get("link", "")
+
+ long_description = "N/A"
if (
- "og:description"
- in search_item["pagemap"]["metatags"][0]
+ "pagemap" in search_item
+ and "metatags" in search_item["pagemap"]
):
- long_description = search_item["pagemap"][
- "metatags"
- ][0]["og:description"]
- else:
- long_description = "N/A"
- # Get the page title
- title = search_item.get("title")
- # Page snippet
- snippet = search_item.get("snippet")
+ metatags = search_item["pagemap"]["metatags"]
+ if metatags and len(metatags) > 0:
+ meta = metatags[0]
+ long_description = (
+ meta.get("og:description")
+ or meta.get("description")
+ or meta.get("twitter:description")
+ or "N/A"
+ )
+
+ if not title and not snippet and not link:
+ logger.debug(
+ f"Skipping result {i} due to missing essential fields"
+ )
+ continue
- # Extract the page url
- link = search_item.get("link")
response = {
"result_id": i,
"title": title,
@@ -623,19 +652,74 @@ class SearchToolkit(BaseToolkit):
}
responses.append(response)
else:
+ logger.debug(f"Google API response without items: {data}")
error_info = data.get("error", {})
- logger.error(
- f"Google search failed - API response: {error_info}"
- )
+ if error_info:
+ error_code = error_info.get("code", "Unknown")
+ error_message = error_info.get("message", "Unknown error")
+ error_details = f"Error {error_code}: {error_message}"
+ else:
+ error_details = (
+ "No results found or API returned empty response"
+ )
+
+ logger.error(f"Google search failed - {error_details}")
responses.append(
- {
- "error": f"Google search failed - "
- f"API response: {error_info}"
- }
+ {"error": f"Google search failed - {error_details}"}
)
+ except requests.exceptions.Timeout:
+ logger.error("Google search request timed out")
+ responses.append(
+ {"error": "Google search request timed out after 30 seconds"}
+ )
+ except requests.exceptions.HTTPError as e:
+ error_message = f"Google API HTTP error: {e}"
+ try:
+ error_data = e.response.json()
+ if "error" in error_data and "message" in error_data["error"]:
+ api_error_message = error_data["error"]["message"]
+ error_code = error_data["error"].get(
+ "code", e.response.status_code
+ )
+ error_message = (
+ f"Google API error {error_code}: {api_error_message}"
+ )
+
+ if e.response.status_code == 429:
+ error_message = f"Google API rate limit exceeded: {api_error_message}"
+ elif e.response.status_code == 400:
+ if "API key" in api_error_message:
+ error_message = (
+ f"Invalid Google API key: {api_error_message}"
+ )
+ else:
+ error_message = f"Bad request to Google API: {api_error_message}"
+ except (ValueError, KeyError):
+ if e.response.status_code == 429:
+ error_message = "Google API rate limit exceeded. Please try again later."
+ elif e.response.status_code == 400:
+ error_message = "Google API bad request. Please check your API key and search engine ID."
+ elif e.response.status_code == 403:
+ error_message = "Google API access forbidden. Please check your API key permissions."
+
+ logger.error(error_message)
+ responses.append({"error": error_message})
+ except requests.exceptions.RequestException as e:
+ logger.error(f"Google search network error: {e}")
+ responses.append(
+ {"error": f"Network error during Google search: {e!s}"}
+ )
+ except ValueError as e:
+ logger.error(f"Google search response parsing error: {e}")
+ responses.append(
+ {"error": f"Failed to parse Google search response: {e!s}"}
+ )
except Exception as e:
- responses.append({"error": f"google search failed: {e!s}"})
+ logger.error(f"Unexpected error during Google search: {e}")
+ responses.append(
+ {"error": f"Unexpected error during Google search: {e!s}"}
+ )
return responses
def tavily_search(
diff --git a/test/toolkits/test_search_functions.py b/test/toolkits/test_search_functions.py
index 1be2e2e90..861c13d63 100644
--- a/test/toolkits/test_search_functions.py
+++ b/test/toolkits/test_search_functions.py
@@ -90,6 +90,339 @@ def test_google_api():
assert result.status_code == 200
+@patch('requests.get')
+def test_search_google_web_success(mock_get, search_toolkit):
+ """Test successful Google web search."""
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "items": [
+ {
+ "title": "Test Title 1",
+ "snippet": "Test snippet 1",
+ "link": "https://example1.com",
+ "pagemap": {
+ "metatags": [{"og:description": "Test long description 1"}]
+ },
+ },
+ {
+ "title": "Test Title 2",
+ "snippet": "Test snippet 2",
+ "link": "https://example2.com",
+ "pagemap": {
+ "metatags": [{"description": "Test long description 2"}]
+ },
+ },
+ ]
+ }
+ mock_get.return_value = mock_response
+
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ result = search_toolkit.search_google(
+ query="test query", number_of_result_pages=2
+ )
+
+ assert len(result) == 2
+ assert result[0] == {
+ "result_id": 1,
+ "title": "Test Title 1",
+ "description": "Test snippet 1",
+ "long_description": "Test long description 1",
+ "url": "https://example1.com",
+ }
+ assert result[1] == {
+ "result_id": 2,
+ "title": "Test Title 2",
+ "description": "Test snippet 2",
+ "long_description": "Test long description 2",
+ "url": "https://example2.com",
+ }
+
+ # Verify the request was made with correct parameters
+ mock_get.assert_called_once()
+ call_args = mock_get.call_args[0][0]
+ assert "q=test%20query" in call_args
+ assert "num=2" in call_args
+
+
+@patch('requests.get')
+def test_search_google_image_success(mock_get, search_toolkit):
+ """Test successful Google image search."""
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "items": [
+ {
+ "title": "Image Title 1",
+ "link": "https://example.com/image1.jpg",
+ "displayLink": "example.com",
+ "image": {
+ "contextLink": "https://example.com/page1.html",
+ "width": 800,
+ "height": 600,
+ },
+ }
+ ]
+ }
+ mock_get.return_value = mock_response
+
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ result = search_toolkit.search_google(
+ query="test image", search_type="image", number_of_result_pages=1
+ )
+
+ assert len(result) == 1
+ assert result[0] == {
+ "result_id": 1,
+ "title": "Image Title 1",
+ "image_url": "https://example.com/image1.jpg",
+ "display_link": "example.com",
+ "context_url": "https://example.com/page1.html",
+ "width": 800,
+ "height": 600,
+ }
+
+ # Verify searchType parameter was included
+ call_args = mock_get.call_args[0][0]
+ assert "searchType=image" in call_args
+
+
+@patch('requests.get')
+def test_search_google_no_results(mock_get, search_toolkit):
+ """Test Google search with no results."""
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "searchInformation": {"totalResults": "0"}
+ }
+ mock_get.return_value = mock_response
+
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ result = search_toolkit.search_google(query="very specific query")
+
+ assert len(result) == 1
+ assert "error" in result[0]
+ assert "No results found" in result[0]["error"]
+
+
+def test_search_google_parameter_validation(search_toolkit):
+ """Test parameter validation for Google search."""
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ # Test invalid start_page
+ with pytest.raises(
+ ValueError, match="start_page must be a positive integer"
+ ):
+ search_toolkit.search_google(query="test", start_page=0)
+
+ with pytest.raises(
+ ValueError, match="start_page must be a positive integer"
+ ):
+ search_toolkit.search_google(query="test", start_page=-1)
+
+ # Test invalid number_of_result_pages
+ with pytest.raises(
+ ValueError,
+ match="number_of_result_pages must be a positive integer",
+ ):
+ search_toolkit.search_google(
+ query="test", number_of_result_pages=0
+ )
+
+ # Test invalid search_type
+ with pytest.raises(
+ ValueError, match="search_type must be either 'web' or 'image'"
+ ):
+ search_toolkit.search_google(query="test", search_type="invalid")
+
+
+@patch('requests.get')
+def test_search_google_error_handling(mock_get, search_toolkit):
+ """Test error handling in Google search."""
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ # Test timeout error
+ mock_get.side_effect = requests.exceptions.Timeout()
+ result = search_toolkit.search_google(query="test")
+ assert len(result) == 1
+ assert "error" in result[0]
+ assert "timed out" in result[0]["error"]
+
+ # Test HTTP 429 error (rate limit) with API message
+ mock_response = MagicMock()
+ mock_error_response = MagicMock()
+ mock_error_response.status_code = 429
+ mock_error_response.json.return_value = {
+ "error": {
+ "code": 429,
+ "message": "Rate limit exceeded. Try again in 1 minute.",
+ }
+ }
+ mock_response.raise_for_status.side_effect = (
+ requests.exceptions.HTTPError(response=mock_error_response)
+ )
+ mock_get.side_effect = None
+ mock_get.return_value = mock_response
+ result = search_toolkit.search_google(query="test")
+ assert "Google API rate limit exceeded" in result[0]["error"]
+ assert "Try again in 1 minute" in result[0]["error"]
+
+ # Test HTTP 400 error (bad request) with API key error
+ mock_error_response.status_code = 400
+ mock_error_response.json.return_value = {
+ "error": {
+ "code": 400,
+ "message": "API key not valid. Please pass a valid API key.",
+ }
+ }
+ mock_response.raise_for_status.side_effect = (
+ requests.exceptions.HTTPError(response=mock_error_response)
+ )
+ result = search_toolkit.search_google(query="test")
+ assert "Invalid Google API key" in result[0]["error"]
+ assert "API key not valid" in result[0]["error"]
+
+ # Test HTTP 400 error (bad request) without API key error
+ mock_error_response.json.return_value = {
+ "error": {
+ "code": 400,
+ "message": "Request contains an invalid argument.",
+ }
+ }
+ result = search_toolkit.search_google(query="test")
+ assert "Bad request to Google API" in result[0]["error"]
+ assert "invalid argument" in result[0]["error"]
+
+ # Test HTTP 403 error (forbidden) - fallback when no JSON
+ mock_error_response.status_code = 403
+ mock_error_response.json.side_effect = ValueError("No JSON")
+ mock_response.raise_for_status.side_effect = (
+ requests.exceptions.HTTPError(response=mock_error_response)
+ )
+ result = search_toolkit.search_google(query="test")
+ assert "access forbidden" in result[0]["error"]
+
+
+@patch('requests.get')
+def test_search_google_with_exclude_domains(mock_get):
+ """Test Google search with excluded domains."""
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"items": []}
+ mock_get.return_value = mock_response
+
+ # Create toolkit with exclude_domains
+ toolkit_with_exclusions = SearchToolkit(
+ exclude_domains=["example.com", "test.com"]
+ )
+
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ toolkit_with_exclusions.search_google(query="test query")
+
+ # Verify the query includes exclusion terms
+ call_args = mock_get.call_args[0][0]
+ assert "test%20query%20-site%3Aexample.com%20-site%3Atest.com" in call_args
+
+
+@patch('requests.get')
+def test_search_google_special_characters(mock_get, search_toolkit):
+ """Test Google search with special characters in query."""
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"items": []}
+ mock_get.return_value = mock_response
+
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ search_toolkit.search_google(query="C++ & Python programming")
+
+ # Verify special characters are properly encoded
+ call_args = mock_get.call_args[0][0]
+ assert "C%2B%2B%20%26%20Python%20programming" in call_args
+
+
+@patch('requests.get')
+def test_search_google_without_metatags(mock_get, search_toolkit):
+ """Test Google search handling results without metatags."""
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "items": [
+ {
+ "title": "Test Title Without Metatags",
+ "snippet": "Test snippet without metatags",
+ "link": "https://example.com/no-metatags",
+ # No pagemap/metatags
+ },
+ {
+ "title": "Test Title With Twitter Description",
+ "snippet": "Test snippet with twitter description",
+ "link": "https://example.com/twitter",
+ "pagemap": {
+ "metatags": [
+ {"twitter:description": "Twitter description only"}
+ ]
+ },
+ },
+ ]
+ }
+ mock_get.return_value = mock_response
+
+ with patch.dict(
+ os.environ,
+ {
+ 'GOOGLE_API_KEY': 'fake_api_key',
+ 'SEARCH_ENGINE_ID': 'fake_search_engine_id',
+ },
+ ):
+ result = search_toolkit.search_google(query="test query")
+
+ assert len(result) == 2
+ # First result should have N/A for long_description
+ assert result[0]["long_description"] == "N/A"
+ # Second result should use twitter:description
+ assert result[1]["long_description"] == "Twitter description only"
+
+
search_duckduckgo = SearchToolkit().search_duckduckgo