Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: allow search_google toolkit to exclude specified domains (#2901)

Boer-Zhang059e266627eb
diff --git a/camel/toolkits/search_toolkit.py b/camel/toolkits/search_toolkit.py
index 7c58916e3..f7c9d7e88 100644
--- a/camel/toolkits/search_toolkit.py
+++ b/camel/toolkits/search_toolkit.py
@@ -435,7 +435,10 @@ class SearchToolkit(BaseToolkit):
]
)
def search_google(
- self, query: str, search_type: str = "web"
+ self,
+ query: str,
+ search_type: str = "web",
+ exclude_domains: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
r"""Use Google search engine to search information for the given query.
@@ -443,6 +446,9 @@ class SearchToolkit(BaseToolkit):
query (str): The query to be searched.
search_type (str): The type of search to perform. Either "web" for
web pages or "image" for image search. (default: "web")
+ exclude_domains (Optional[List[str]]): A list of domains to exclude
+ from the search results. If provided, the query will be modified.
+ (default: :obj:`None`)
Returns:
List[Dict[str, Any]]: A list of dictionaries where each dictionary
@@ -499,11 +505,21 @@ class SearchToolkit(BaseToolkit):
start_page_idx = 1
# Different language may get different result
search_language = "en"
+
+ modified_query = query
+ if exclude_domains:
+ # Use Google's -site: operator to exclude domains
+ exclusion_terms = " ".join(
+ [f"-site:{domain}" for domain in exclude_domains]
+ )
+ modified_query = f"{query} {exclusion_terms}"
+ logger.debug(f"Excluded domains, modified query: {modified_query}")
+
# Constructing the URL
# Doc: https://developers.google.com/custom-search/v1/using_rest
base_url = (
f"https://www.googleapis.com/customsearch/v1?"
- f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={query}&start="
+ f"key={GOOGLE_API_KEY}&cx={SEARCH_ENGINE_ID}&q={modified_query}&start="
f"{start_page_idx}&lr={search_language}&num={self.number_of_result_pages}"
)
diff --git a/examples/toolkits/search_google_with_exclusion.py b/examples/toolkits/search_google_with_exclusion.py
new file mode 100644
index 000000000..8c6bb6cd2
--- /dev/null
+++ b/examples/toolkits/search_google_with_exclusion.py
@@ -0,0 +1,39 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from camel.toolkits import SearchToolkit
+
+# This example demonstrates how to use the SearchToolkit to
+# perform a Google search while excluding certain domains from the results.
+
+# Google search API Key and Search Engine ID are required to run this example
+# API Key: https://developers.google.com/custom-search/v1/overview
+# Search Engine ID: https://cse.google.com/cse/all
+
+
+toolkit = SearchToolkit(number_of_result_pages=3)
+search_result = toolkit.search_google(query="Python Wikipedia")
+print("Search results for 'Python Wikipedia':\n")
+for result in search_result:
+ print(result['url'])
+search_result = toolkit.search_google(
+ query="Python Wikipedia",
+ exclude_domains=[
+ "wikipedia.org",
+ "wikipedia-api.readthedocs.io",
+ "wikipedia.readthedocs.io",
+ ],
+)
+print("\nExcluding specified domains related to wikipedia:\n")
+for result in search_result:
+ print(result['url'])