🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: allow search_google toolkit to exclude specified domains (#2901)
Boer-Zhang059e266627eb
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.pyindex 7c58916e3..f7c9d7e88 100644--- a/camel/toolkits/search_toolkit.py+++ b/camel/toolkits/search_toolkit.py@@ -435,7 +435,10 @@ class SearchToolkit(BaseToolkit):])def search_google(- self, query: str, search_type: str = "web"+ self,+ query: str,+ search_type: str = "web",+ exclude_domains: Optional[List[str]] = None,) -> List[Dict[str, Any]]:r"""Use Google search engine to search information for the given query.@@ -443,6 +446,9 @@ class SearchToolkit(BaseToolkit):query (str): The query to be searched.search_type (str): The type of search to perform. Either "web" forweb pages or "image" for image search. (default: "web")+ exclude_domains (Optional[List[str]]): A list of domains to exclude+ from the search results. If provided, the query will be modified.+ (default: :obj:`None`)Returns:List[Dict[str, Any]]: A list of dictionaries where each dictionary@@ -499,11 +505,21 @@ class SearchToolkit(BaseToolkit):start_page_idx = 1# Different language may get different resultsearch_language = "en"++ modified_query = query+ if exclude_domains:+ # Use Google's -site: operator to exclude domains+ exclusion_terms = " ".join(+ [f"-site:{domain}" for domain in exclude_domains]+ )+ modified_query = f"{query} {exclusion_terms}"+ logger.debug(f"Excluded domains, modified query: {modified_query}")+# Constructing the URL# Doc: https://developers.google.com/custom-search/v1/using_restbase_url = (f"https://www.googleapis.com/customsearch/v1?"- f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={query}&start="+ f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={modified_query}&start="f"{start_page_idx}&lr={search_language}&num={self.number_of_result_pages}")diff --git a/examples/toolkits/search_google_with_exclusion.py b/examples/toolkits/search_google_with_exclusion.pynew file mode 100644index 000000000..8c6bb6cd2--- /dev/null+++ b/examples/toolkits/search_google_with_exclusion.py@@ -0,0 +1,39 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from camel.toolkits import SearchToolkit++# This example demonstrates how to use the SearchToolkit to+# perform a Google search while excluding certain domains from the results.++# Google search API Key and Search Engine ID are required to run this example+# API Key: https://developers.google.com/custom-search/v1/overview+# Search Engine ID: https://cse.google.com/cse/all+++toolkit = SearchToolkit(number_of_result_pages=3)+search_result = toolkit.search_google(query="Python Wikipedia")+print("Search results for 'Python Wikipedia':\n")+for result in search_result:+ print(result['url'])+search_result = toolkit.search_google(+ query="Python Wikipedia",+ exclude_domains=[+ "wikipedia.org",+ "wikipedia-api.readthedocs.io",+ "wikipedia.readthedocs.io",+ ],+)+print("\nExcluding specified domains related to wikipedia:\n")+for result in search_result:+ print(result['url'])