🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add searxng toolkit (#1963)
Rishabh666cb81b5d92
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.pyindex 7da68879b..d29626f3f 100644--- a/camel/toolkits/__init__.py+++ b/camel/toolkits/__init__.py@@ -61,6 +61,8 @@ from .file_write_toolkit import FileWriteToolkitfrom .terminal_toolkit import TerminalToolkitfrom .pubmed_toolkit import PubMedToolkitfrom .thinking_toolkit import ThinkingToolkit+from .searxng_toolkit import SearxNGToolkit+__all__ = ['BaseToolkit',@@ -110,4 +112,5 @@ __all__ = ['TerminalToolkit','PubMedToolkit','ThinkingToolkit',+ 'SearxNGToolkit',]diff --git a/camel/toolkits/searxng_toolkit.py b/camel/toolkits/searxng_toolkit.pynew file mode 100644index 000000000..4c80a5edf--- /dev/null+++ b/camel/toolkits/searxng_toolkit.py@@ -0,0 +1,207 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from typing import ClassVar, Dict, List, Optional, Union+from urllib.parse import urlparse++import requests++from camel.logger import get_logger+from camel.toolkits.base import BaseToolkit+from camel.toolkits.function_tool import FunctionTool++logger = get_logger(__name__)+++class SearxNGToolkit(BaseToolkit):+ r"""A toolkit for performing web searches using SearxNG search engine.++ This toolkit provides methods to search the web using SearxNG,+ a privacy-respecting metasearch engine. It supports customizable+ search parameters and safe search levels.++ Args:+ searxng_host (str): The URL of the SearxNG instance to use for+ searches. Must be a valid HTTP/HTTPS URL.+ language (str, optional): Search language code for results.+ (default: :obj:`"en"`)+ categories (List[str], optional): List of search categories to use.+ (default: :obj:`None`)+ time_range (str, optional): Time range filter for search results.Valid+ values are "day", "week", "month", "year". (default: :obj:`None`)+ safe_search (int, optional): Safe search level (0: None, 1: Moderate,+ 2: Strict). (default: :obj:`1`)++ Raises:+ ValueError: If searxng_host is not a valid HTTP/HTTPS URL.+ ValueError: If safe_search is not in the valid range [0, 2].+ ValueError: If time_range is provided but not in valid options.+ """++ # Constants for validation+ _SAFE_SEARCH_LEVELS: ClassVar[Dict[int, str]] = {+ 0: "Disabled",+ 1: "Moderate",+ 2: "Strict",+ }+ _VALID_TIME_RANGES: ClassVar[List[str]] = ["day", "week", "month", "year"]+ _DEFAULT_CATEGORY: ClassVar[str] = "general"++ def __init__(+ self,+ searxng_host: str,+ language: str = "en",+ categories: Optional[List[str]] = None,+ time_range: Optional[str] = None,+ safe_search: int = 1,+ ) -> None:+ self._validate_searxng_host(searxng_host)+ self._validate_safe_search(safe_search)+ if time_range is not None:+ self._validate_time_range(time_range)++ self.searxng_host = searxng_host.rstrip('/')+ self.language = language+ self.categories = categories or [self._DEFAULT_CATEGORY]+ self.time_range = time_range+ self.safe_search = safe_search++ logger.info(+ f"Initialized SearxNG toolkit with host: {searxng_host}, "+ f"safe_search: {self._SAFE_SEARCH_LEVELS[safe_search]}"+ )++ def _validate_searxng_host(self, url: str) -> None:+ r"""Validate if the given URL is a proper HTTP/HTTPS URL.++ Args:+ url (str): The URL to validate.++ Raises:+ ValueError: If the URL is not valid.+ """+ try:+ result = urlparse(url)+ is_valid = all(+ [+ result.scheme in ('http', 'https'),+ result.netloc,+ ]+ )+ if not is_valid:+ raise ValueError+ except Exception:+ raise ValueError(+ "Invalid searxng_host URL. Must be a valid HTTP/HTTPS URL."+ )++ def _validate_safe_search(self, level: int) -> None:+ r"""Validate if the safe search level is valid.++ Args:+ level (int): The safe search level to validate.++ Raises:+ ValueError: If the safe search level is not valid.+ """+ if level not in self._SAFE_SEARCH_LEVELS:+ raise ValueError(+ f"Invalid safe_search level: {level}. Must be one of: "+ f"{list(self._SAFE_SEARCH_LEVELS.keys())}"+ )++ def _validate_time_range(self, time_range: str) -> None:+ r"""Validate if the time range is valid.++ Args:+ time_range (str): The time range to validate.++ Raises:+ ValueError: If the time range is not valid.+ """+ if time_range not in self._VALID_TIME_RANGES:+ raise ValueError(+ f"Invalid time_range: {time_range}. Must be one of: "+ f"{self._VALID_TIME_RANGES}"+ )++ def search(+ self,+ query: str,+ num_results: int = 10,+ category: Optional[str] = None,+ ) -> List[Dict[str, str]]:+ r"""Perform a web search using the configured SearxNG instance.++ Args:+ query (str): The search query string to execute.+ num_results (int, optional): Maximum number of results to return.+ (default: :obj:`10`)+ category (str, optional): Specific search category to use. If not+ provided, uses the first category from self.categories.+ (default: :obj:`None`)++ Returns:+ List[Dict[str, str]]: List of search results, where each result is+ dictionary containing 'title', 'link', and 'snippet' keys.+ """+ params: Dict[str, Union[str, int]] = {+ "q": query,+ "format": "json",+ "language": self.language,+ "categories": category or self.categories[0],+ "pageno": 1,+ "safe": self.safe_search,+ }++ if self.time_range:+ params["time_range"] = self.time_range++ try:+ logger.debug(f"Sending search request with query: {query}")+ response = requests.get(+ f"{self.searxng_host}/search",+ params=params, # type: ignore[arg-type]+ headers={"User-Agent": "camel-ai/searxng-toolkit"},+ )+ response.raise_for_status()+ results = response.json().get("results", [])++ formatted_results = []+ for result in results[:num_results]:+ formatted_results.append(+ {+ "title": result.get("title", ""),+ "link": result.get("url", ""),+ "snippet": result.get("content", ""),+ }+ )++ logger.debug(f"Retrieved {len(formatted_results)} results")+ return formatted_results++ except Exception as error:+ logger.error(f"Search failed: {error!s}")+ return []++ def get_tools(self) -> List[FunctionTool]:+ r"""Get the list of available tools in the toolkit.++ Returns:+ List[FunctionTool]: A list of FunctionTool objects representing the+ available functions in the toolkit.+ """+ return [+ FunctionTool(self.search),+ ]diff --git a/examples/toolkits/searxng_toolkit_example.py b/examples/toolkits/searxng_toolkit_example.pynew file mode 100644index 000000000..4d87241bc--- /dev/null+++ b/examples/toolkits/searxng_toolkit_example.py@@ -0,0 +1,97 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import os+from typing import Optional++from camel.agents import ChatAgent+from camel.configs import ChatGPTConfig+from camel.logger import get_logger+from camel.models import ModelFactory+from camel.toolkits import SearxNGToolkit+from camel.types import ModelPlatformType, ModelType++logger = get_logger(__name__)+++def get_searxng_instance() -> Optional[str]:+ r"""Get SearxNG instance URL from environment variable.++ Returns:+ Optional[str]: The SearxNG instance URL if set, None otherwise.+ """+ instance_url = os.getenv('SEARXNG_URL')+ if not instance_url:+ logger.warning(+ "SEARXNG_URL environment variable not set. "+ "Please set it to your SearxNG instance URL."+ )+ return instance_url+++def main() -> None:+ r"""Run the SearxNG toolkit example."""+ # Get SearxNG instance URL+ instance_url = get_searxng_instance()+ if not instance_url:+ logger.error(+ "\nTo run this example:"+ "\n1. Find a SearxNG instance (self-host or use public instance)"+ "\n2. Set the SEARXNG_URL environment variable:"+ "\n export SEARXNG_URL='https://your-searxng-instance.com'"+ "\n\nPublic instances can be found at: https://searx.space"+ )+ return++ # Initialize the toolkit+ searxng_toolkit = SearxNGToolkit(+ searxng_host=instance_url,+ language="en",+ categories=["general", "news"],+ time_range="month",+ safe_search=1,+ )++ # Initialize the model+ model = ModelFactory.create(+ model_type=ModelType.DEFAULT,+ model_platform=ModelPlatformType.DEFAULT,+ model_config_dict=ChatGPTConfig(temperature=0.0).as_dict(),+ )++ # Create chat agent+ system_message = (+ "You are a helpful assistant that can search the web using SearxNG."+ )+ agent = ChatAgent(+ system_message=system_message,+ model=model,+ tools=[*searxng_toolkit.get_tools()],+ )++ # Example search query+ query = "Tell me about the CAMEL AI framework"+ response = agent.step(query)++ # Print the response message content+ print("\nAgent Response:")+ print(response.msgs[0].content)++ # Print tool calls if needed+ print("\nTool Calls:")+ print(response.info['tool_calls'])+++if __name__ == "__main__":+ main()diff --git a/test/toolkits/test_searxng_toolkit.py b/test/toolkits/test_searxng_toolkit.pynew file mode 100644index 000000000..ad36abfc1--- /dev/null+++ b/test/toolkits/test_searxng_toolkit.py@@ -0,0 +1,160 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import pytest++from camel.toolkits import SearxNGToolkit+++class TestSearxNGToolkitInitialization:+ r"""Test cases for SearxNGToolkit initialization."""++ def test_init_with_valid_parameters(self):+ r"""Test initialization with valid parameters."""+ toolkit = SearxNGToolkit(+ searxng_host="https://searx.example.org",+ language="en",+ categories=["general"],+ time_range="day",+ safe_search=1,+ )+ assert toolkit.searxng_host == "https://searx.example.org"+ assert toolkit.language == "en"+ assert toolkit.categories == ["general"]+ assert toolkit.time_range == "day"+ assert toolkit.safe_search == 1++ def test_init_with_default_parameters(self):+ r"""Test initialization with default parameters."""+ toolkit = SearxNGToolkit(searxng_host="https://searx.example.org")+ assert toolkit.searxng_host == "https://searx.example.org"+ assert toolkit.language == "en"+ assert toolkit.categories == ["general"]+ assert toolkit.time_range is None+ assert toolkit.safe_search == 1++ def test_init_with_trailing_slash_in_host(self):+ r"""Test that trailing slashes are removed from the host URL."""+ toolkit = SearxNGToolkit(searxng_host="https://searx.example.org/")+ assert toolkit.searxng_host == "https://searx.example.org"+++class TestSearxNGToolkitTimeRangeValidation:+ r"""Test cases specifically for the time_range validation functionality."""++ @pytest.mark.parametrize(+ "time_range",+ ["day", "week", "month", "year"],+ )+ def test_valid_time_ranges(self, time_range):+ r"""Test that valid time ranges are accepted."""+ # This should not raise an exception+ toolkit = SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range=time_range,+ )+ assert toolkit.time_range == time_range++ @pytest.mark.parametrize(+ "invalid_time_range",+ [+ "hour",+ "daily",+ "monthly",+ "yearly",+ "quarter",+ "decade",+ "century",+ "",+ "DAY", # Case sensitive+ " day", # With whitespace+ "day ",+ 123, # Wrong type (will be caught by type checking)+ ],+ )+ def test_invalid_time_ranges(self, invalid_time_range):+ r"""Test that invalid time ranges raise ValueError."""+ with pytest.raises(ValueError) as excinfo:+ SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range=invalid_time_range,+ )++ # Verify the error message contains useful information+ error_message = str(excinfo.value)+ assert "Invalid time_range" in error_message+ assert str(invalid_time_range) in error_message+ assert "day" in error_message+ assert "week" in error_message+ assert "month" in error_message+ assert "year" in error_message++ def test_none_time_range(self):+ r"""Test that None is accepted as a time_range value."""+ # This should not raise an exception+ toolkit = SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range=None,+ )+ assert toolkit.time_range is None++ def test_direct_method_call(self):+ r"""Test directly calling the _validate_time_range method."""+ toolkit = SearxNGToolkit(searxng_host="https://searx.example.org")++ # Valid time ranges should not raise exceptions+ for valid_range in ["day", "week", "month", "year"]:+ toolkit._validate_time_range(valid_range) # Should not raise++ # Invalid time range should raise ValueError+ with pytest.raises(ValueError):+ toolkit._validate_time_range("invalid_range")+++class TestSearxNGToolkitTimeRangeEdgeCases:+ r"""Test edge cases for time_range validation."""++ def test_time_range_case_sensitivity(self):+ r"""Test that time range validation is case sensitive."""+ with pytest.raises(ValueError):+ SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range="Day", # Capitalized+ )++ with pytest.raises(ValueError):+ SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range="WEEK", # All caps+ )++ def test_time_range_with_whitespace(self):+ r"""Test that time range validation rejects values with whitespace."""+ with pytest.raises(ValueError):+ SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range=" day", # Leading space+ )++ with pytest.raises(ValueError):+ SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range="month ", # Trailing space+ )++ with pytest.raises(ValueError):+ SearxNGToolkit(+ searxng_host="https://searx.example.org",+ time_range=" year ", # Both leading and trailing space+ )