Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add searxng toolkit (#1963)

Rishabh666cb81b5d92
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.py
index 7da68879b..d29626f3f 100644
--- a/camel/toolkits/__init__.py
+++ b/camel/toolkits/__init__.py
@@ -61,6 +61,8 @@ from .file_write_toolkit import FileWriteToolkit
from .terminal_toolkit import TerminalToolkit
from .pubmed_toolkit import PubMedToolkit
from .thinking_toolkit import ThinkingToolkit
+from .searxng_toolkit import SearxNGToolkit
+
__all__ = [
'BaseToolkit',
@@ -110,4 +112,5 @@ __all__ = [
'TerminalToolkit',
'PubMedToolkit',
'ThinkingToolkit',
+ 'SearxNGToolkit',
]
diff --git a/camel/toolkits/searxng_toolkit.py b/camel/toolkits/searxng_toolkit.py
new file mode 100644
index 000000000..4c80a5edf
--- /dev/null
+++ b/camel/toolkits/searxng_toolkit.py
@@ -0,0 +1,207 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import ClassVar, Dict, List, Optional, Union
+from urllib.parse import urlparse
+
+import requests
+
+from camel.logger import get_logger
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+logger = get_logger(__name__)
+
+
+class SearxNGToolkit(BaseToolkit):
+ r"""A toolkit for performing web searches using SearxNG search engine.
+
+ This toolkit provides methods to search the web using SearxNG,
+ a privacy-respecting metasearch engine. It supports customizable
+ search parameters and safe search levels.
+
+ Args:
+ searxng_host (str): The URL of the SearxNG instance to use for
+ searches. Must be a valid HTTP/HTTPS URL.
+ language (str, optional): Search language code for results.
+ (default: :obj:`"en"`)
+ categories (List[str], optional): List of search categories to use.
+ (default: :obj:`None`)
+ time_range (str, optional): Time range filter for search results.Valid
+ values are "day", "week", "month", "year". (default: :obj:`None`)
+ safe_search (int, optional): Safe search level (0: None, 1: Moderate,
+ 2: Strict). (default: :obj:`1`)
+
+ Raises:
+ ValueError: If searxng_host is not a valid HTTP/HTTPS URL.
+ ValueError: If safe_search is not in the valid range [0, 2].
+ ValueError: If time_range is provided but not in valid options.
+ """
+
+ # Constants for validation
+ _SAFE_SEARCH_LEVELS: ClassVar[Dict[int, str]] = {
+ 0: "Disabled",
+ 1: "Moderate",
+ 2: "Strict",
+ }
+ _VALID_TIME_RANGES: ClassVar[List[str]] = ["day", "week", "month", "year"]
+ _DEFAULT_CATEGORY: ClassVar[str] = "general"
+
+ def __init__(
+ self,
+ searxng_host: str,
+ language: str = "en",
+ categories: Optional[List[str]] = None,
+ time_range: Optional[str] = None,
+ safe_search: int = 1,
+ ) -> None:
+ self._validate_searxng_host(searxng_host)
+ self._validate_safe_search(safe_search)
+ if time_range is not None:
+ self._validate_time_range(time_range)
+
+ self.searxng_host = searxng_host.rstrip('/')
+ self.language = language
+ self.categories = categories or [self._DEFAULT_CATEGORY]
+ self.time_range = time_range
+ self.safe_search = safe_search
+
+ logger.info(
+ f"Initialized SearxNG toolkit with host: {searxng_host}, "
+ f"safe_search: {self._SAFE_SEARCH_LEVELS[safe_search]}"
+ )
+
+ def _validate_searxng_host(self, url: str) -> None:
+ r"""Validate if the given URL is a proper HTTP/HTTPS URL.
+
+ Args:
+ url (str): The URL to validate.
+
+ Raises:
+ ValueError: If the URL is not valid.
+ """
+ try:
+ result = urlparse(url)
+ is_valid = all(
+ [
+ result.scheme in ('http', 'https'),
+ result.netloc,
+ ]
+ )
+ if not is_valid:
+ raise ValueError
+ except Exception:
+ raise ValueError(
+ "Invalid searxng_host URL. Must be a valid HTTP/HTTPS URL."
+ )
+
+ def _validate_safe_search(self, level: int) -> None:
+ r"""Validate if the safe search level is valid.
+
+ Args:
+ level (int): The safe search level to validate.
+
+ Raises:
+ ValueError: If the safe search level is not valid.
+ """
+ if level not in self._SAFE_SEARCH_LEVELS:
+ raise ValueError(
+ f"Invalid safe_search level: {level}. Must be one of: "
+ f"{list(self._SAFE_SEARCH_LEVELS.keys())}"
+ )
+
+ def _validate_time_range(self, time_range: str) -> None:
+ r"""Validate if the time range is valid.
+
+ Args:
+ time_range (str): The time range to validate.
+
+ Raises:
+ ValueError: If the time range is not valid.
+ """
+ if time_range not in self._VALID_TIME_RANGES:
+ raise ValueError(
+ f"Invalid time_range: {time_range}. Must be one of: "
+ f"{self._VALID_TIME_RANGES}"
+ )
+
+ def search(
+ self,
+ query: str,
+ num_results: int = 10,
+ category: Optional[str] = None,
+ ) -> List[Dict[str, str]]:
+ r"""Perform a web search using the configured SearxNG instance.
+
+ Args:
+ query (str): The search query string to execute.
+ num_results (int, optional): Maximum number of results to return.
+ (default: :obj:`10`)
+ category (str, optional): Specific search category to use. If not
+ provided, uses the first category from self.categories.
+ (default: :obj:`None`)
+
+ Returns:
+ List[Dict[str, str]]: List of search results, where each result is
+ dictionary containing 'title', 'link', and 'snippet' keys.
+ """
+ params: Dict[str, Union[str, int]] = {
+ "q": query,
+ "format": "json",
+ "language": self.language,
+ "categories": category or self.categories[0],
+ "pageno": 1,
+ "safe": self.safe_search,
+ }
+
+ if self.time_range:
+ params["time_range"] = self.time_range
+
+ try:
+ logger.debug(f"Sending search request with query: {query}")
+ response = requests.get(
+ f"{self.searxng_host}/search",
+ params=params, # type: ignore[arg-type]
+ headers={"User-Agent": "camel-ai/searxng-toolkit"},
+ )
+ response.raise_for_status()
+ results = response.json().get("results", [])
+
+ formatted_results = []
+ for result in results[:num_results]:
+ formatted_results.append(
+ {
+ "title": result.get("title", ""),
+ "link": result.get("url", ""),
+ "snippet": result.get("content", ""),
+ }
+ )
+
+ logger.debug(f"Retrieved {len(formatted_results)} results")
+ return formatted_results
+
+ except Exception as error:
+ logger.error(f"Search failed: {error!s}")
+ return []
+
+ def get_tools(self) -> List[FunctionTool]:
+ r"""Get the list of available tools in the toolkit.
+
+ Returns:
+ List[FunctionTool]: A list of FunctionTool objects representing the
+ available functions in the toolkit.
+ """
+ return [
+ FunctionTool(self.search),
+ ]
diff --git a/examples/toolkits/searxng_toolkit_example.py b/examples/toolkits/searxng_toolkit_example.py
new file mode 100644
index 000000000..4d87241bc
--- /dev/null
+++ b/examples/toolkits/searxng_toolkit_example.py
@@ -0,0 +1,97 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+from typing import Optional
+
+from camel.agents import ChatAgent
+from camel.configs import ChatGPTConfig
+from camel.logger import get_logger
+from camel.models import ModelFactory
+from camel.toolkits import SearxNGToolkit
+from camel.types import ModelPlatformType, ModelType
+
+logger = get_logger(__name__)
+
+
+def get_searxng_instance() -> Optional[str]:
+ r"""Get SearxNG instance URL from environment variable.
+
+ Returns:
+ Optional[str]: The SearxNG instance URL if set, None otherwise.
+ """
+ instance_url = os.getenv('SEARXNG_URL')
+ if not instance_url:
+ logger.warning(
+ "SEARXNG_URL environment variable not set. "
+ "Please set it to your SearxNG instance URL."
+ )
+ return instance_url
+
+
+def main() -> None:
+ r"""Run the SearxNG toolkit example."""
+ # Get SearxNG instance URL
+ instance_url = get_searxng_instance()
+ if not instance_url:
+ logger.error(
+ "\nTo run this example:"
+ "\n1. Find a SearxNG instance (self-host or use public instance)"
+ "\n2. Set the SEARXNG_URL environment variable:"
+ "\n export SEARXNG_URL='https://your-searxng-instance.com'"
+ "\n\nPublic instances can be found at: https://searx.space"
+ )
+ return
+
+ # Initialize the toolkit
+ searxng_toolkit = SearxNGToolkit(
+ searxng_host=instance_url,
+ language="en",
+ categories=["general", "news"],
+ time_range="month",
+ safe_search=1,
+ )
+
+ # Initialize the model
+ model = ModelFactory.create(
+ model_type=ModelType.DEFAULT,
+ model_platform=ModelPlatformType.DEFAULT,
+ model_config_dict=ChatGPTConfig(temperature=0.0).as_dict(),
+ )
+
+ # Create chat agent
+ system_message = (
+ "You are a helpful assistant that can search the web using SearxNG."
+ )
+ agent = ChatAgent(
+ system_message=system_message,
+ model=model,
+ tools=[*searxng_toolkit.get_tools()],
+ )
+
+ # Example search query
+ query = "Tell me about the CAMEL AI framework"
+ response = agent.step(query)
+
+ # Print the response message content
+ print("\nAgent Response:")
+ print(response.msgs[0].content)
+
+ # Print tool calls if needed
+ print("\nTool Calls:")
+ print(response.info['tool_calls'])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/test/toolkits/test_searxng_toolkit.py b/test/toolkits/test_searxng_toolkit.py
new file mode 100644
index 000000000..ad36abfc1
--- /dev/null
+++ b/test/toolkits/test_searxng_toolkit.py
@@ -0,0 +1,160 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import pytest
+
+from camel.toolkits import SearxNGToolkit
+
+
+class TestSearxNGToolkitInitialization:
+ r"""Test cases for SearxNGToolkit initialization."""
+
+ def test_init_with_valid_parameters(self):
+ r"""Test initialization with valid parameters."""
+ toolkit = SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ language="en",
+ categories=["general"],
+ time_range="day",
+ safe_search=1,
+ )
+ assert toolkit.searxng_host == "https://searx.example.org"
+ assert toolkit.language == "en"
+ assert toolkit.categories == ["general"]
+ assert toolkit.time_range == "day"
+ assert toolkit.safe_search == 1
+
+ def test_init_with_default_parameters(self):
+ r"""Test initialization with default parameters."""
+ toolkit = SearxNGToolkit(searxng_host="https://searx.example.org")
+ assert toolkit.searxng_host == "https://searx.example.org"
+ assert toolkit.language == "en"
+ assert toolkit.categories == ["general"]
+ assert toolkit.time_range is None
+ assert toolkit.safe_search == 1
+
+ def test_init_with_trailing_slash_in_host(self):
+ r"""Test that trailing slashes are removed from the host URL."""
+ toolkit = SearxNGToolkit(searxng_host="https://searx.example.org/")
+ assert toolkit.searxng_host == "https://searx.example.org"
+
+
+class TestSearxNGToolkitTimeRangeValidation:
+ r"""Test cases specifically for the time_range validation functionality."""
+
+ @pytest.mark.parametrize(
+ "time_range",
+ ["day", "week", "month", "year"],
+ )
+ def test_valid_time_ranges(self, time_range):
+ r"""Test that valid time ranges are accepted."""
+ # This should not raise an exception
+ toolkit = SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range=time_range,
+ )
+ assert toolkit.time_range == time_range
+
+ @pytest.mark.parametrize(
+ "invalid_time_range",
+ [
+ "hour",
+ "daily",
+ "monthly",
+ "yearly",
+ "quarter",
+ "decade",
+ "century",
+ "",
+ "DAY", # Case sensitive
+ " day", # With whitespace
+ "day ",
+ 123, # Wrong type (will be caught by type checking)
+ ],
+ )
+ def test_invalid_time_ranges(self, invalid_time_range):
+ r"""Test that invalid time ranges raise ValueError."""
+ with pytest.raises(ValueError) as excinfo:
+ SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range=invalid_time_range,
+ )
+
+ # Verify the error message contains useful information
+ error_message = str(excinfo.value)
+ assert "Invalid time_range" in error_message
+ assert str(invalid_time_range) in error_message
+ assert "day" in error_message
+ assert "week" in error_message
+ assert "month" in error_message
+ assert "year" in error_message
+
+ def test_none_time_range(self):
+ r"""Test that None is accepted as a time_range value."""
+ # This should not raise an exception
+ toolkit = SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range=None,
+ )
+ assert toolkit.time_range is None
+
+ def test_direct_method_call(self):
+ r"""Test directly calling the _validate_time_range method."""
+ toolkit = SearxNGToolkit(searxng_host="https://searx.example.org")
+
+ # Valid time ranges should not raise exceptions
+ for valid_range in ["day", "week", "month", "year"]:
+ toolkit._validate_time_range(valid_range) # Should not raise
+
+ # Invalid time range should raise ValueError
+ with pytest.raises(ValueError):
+ toolkit._validate_time_range("invalid_range")
+
+
+class TestSearxNGToolkitTimeRangeEdgeCases:
+ r"""Test edge cases for time_range validation."""
+
+ def test_time_range_case_sensitivity(self):
+ r"""Test that time range validation is case sensitive."""
+ with pytest.raises(ValueError):
+ SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range="Day", # Capitalized
+ )
+
+ with pytest.raises(ValueError):
+ SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range="WEEK", # All caps
+ )
+
+ def test_time_range_with_whitespace(self):
+ r"""Test that time range validation rejects values with whitespace."""
+ with pytest.raises(ValueError):
+ SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range=" day", # Leading space
+ )
+
+ with pytest.raises(ValueError):
+ SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range="month ", # Trailing space
+ )
+
+ with pytest.raises(ValueError):
+ SearxNGToolkit(
+ searxng_host="https://searx.example.org",
+ time_range=" year ", # Both leading and trailing space
+ )