Skip to content

A programming framework for agentic AI

Remove duckduckgo search tools and agents (#6783)

Eric Zhuaf85eb26c206
diff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.toml
index 8ac3d5b91..5cfb7b914 100644
--- a/python/packages/autogen-ext/pyproject.toml
+++ b/python/packages/autogen-ext/pyproject.toml
@@ -155,13 +155,6 @@ canvas = [
"unidiff>=0.7.5",
]
-duckduckgo-search = [
- "httpx>=0.27.0",
- "beautifulsoup4>=4.12.0",
- "html2text>=2024.2.26",
- "autogen-agentchat==0.6.3",
-]
-
[tool.hatch.build.targets.wheel]
packages = ["src/autogen_ext"]
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/duckduckgo_search/__init__.py b/python/packages/autogen-ext/src/autogen_ext/agents/duckduckgo_search/__init__.py
deleted file mode 100644
index 7e18d619f..000000000
--- a/python/packages/autogen-ext/src/autogen_ext/agents/duckduckgo_search/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from ._duckduckgo_agent import DuckDuckGoSearchAgent
-
-__all__ = [
- "DuckDuckGoSearchAgent",
-]
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/duckduckgo_search/_duckduckgo_agent.py b/python/packages/autogen-ext/src/autogen_ext/agents/duckduckgo_search/_duckduckgo_agent.py
deleted file mode 100644
index 1dac92a23..000000000
--- a/python/packages/autogen-ext/src/autogen_ext/agents/duckduckgo_search/_duckduckgo_agent.py
+++ /dev/null
@@ -1,97 +0,0 @@
-# DuckDuckGo Search Agent created by Varad Srivastava
-from typing import Any, Optional
-
-from autogen_agentchat.agents import AssistantAgent
-from autogen_core.models import ChatCompletionClient
-from autogen_ext.tools.duckduckgo_search import DuckDuckGoSearchTool
-
-
-class DuckDuckGoSearchAgent(AssistantAgent):
- """
- A specialized AssistantAgent that uses DuckDuckGo for web searches.
-
- This agent is designed to perform web searches using DuckDuckGo and provide
- relevant information based on the search results. It can be used in group chats
- or as a standalone agent for research and information gathering tasks.
-
- The agent comes pre-configured with a DuckDuckGo search tool and a system message
- optimized for research tasks.
-
- Example:
- .. code-block:: python
-
- import asyncio
- from autogen_ext.agents.duckduckgo_search import DuckDuckGoSearchAgent
- from autogen_ext.models.openai import OpenAIChatCompletionClient
-
-
- async def main() -> None:
- # Create a model client
- model_client = OpenAIChatCompletionClient(model="gpt-4")
-
- # Create a DuckDuckGo search agent
- search_agent = DuckDuckGoSearchAgent(
- name="researcher",
- model_client=model_client,
- )
-
- # Use the agent
- result = await search_agent.run(task="What are the latest developments in AI?")
- print(result.messages[-1].content) # type: ignore
-
-
- asyncio.run(main())
- """
-
- DEFAULT_DESCRIPTION = "A research assistant that uses DuckDuckGo to find and analyze information from the web."
-
- DEFAULT_SYSTEM_MESSAGE = """You are a research assistant that uses DuckDuckGo to find accurate information.
-
-When conducting research:
-1. Break down complex queries into specific, targeted search terms
-2. Use the duckduckgo_search tool to find relevant information
-3. Analyze and synthesize information from multiple sources when possible
-4. Explain why the information is relevant and how it connects to the query
-5. Cite your sources when providing information
-6. If you're unsure about something, say so and explain why
-7. Provide clear, well-structured responses with key findings highlighted
-"""
-
- def __init__(
- self,
- name: str,
- model_client: ChatCompletionClient,
- description: Optional[str] = None,
- system_message: Optional[str] = None,
- **kwargs: Any,
- ) -> None:
- """
- Initialize a DuckDuckGo Search Agent.
-
- Args:
- name (str): The name of the agent
- model_client (ChatCompletionClient): The model client to use for generating responses
- description (Optional[str]): A description of the agent's capabilities. If not provided,
- a default description will be used.
- system_message (Optional[str]): The system message to use for the agent. If not provided,
- a default message will be used.
- **kwargs: Additional keyword arguments passed to the parent AssistantAgent
- """
- if description is None:
- description = self.DEFAULT_DESCRIPTION
-
- if system_message is None:
- system_message = self.DEFAULT_SYSTEM_MESSAGE
-
- # Create the DuckDuckGo search tool
- search_tool = DuckDuckGoSearchTool()
-
- # Initialize the parent AssistantAgent with the search tool
- super().__init__(
- name=name,
- model_client=model_client,
- description=description,
- system_message=system_message,
- tools=[search_tool],
- **kwargs,
- )
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/duckduckgo_search/__init__.py b/python/packages/autogen-ext/src/autogen_ext/tools/duckduckgo_search/__init__.py
deleted file mode 100644
index 7471bfffa..000000000
--- a/python/packages/autogen-ext/src/autogen_ext/tools/duckduckgo_search/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from ._duckduckgo_search import DuckDuckGoSearchArgs, DuckDuckGoSearchResult, DuckDuckGoSearchTool
-
-__all__ = [
- "DuckDuckGoSearchTool",
- "DuckDuckGoSearchArgs",
- "DuckDuckGoSearchResult",
-]
diff --git a/python/packages/autogen-ext/src/autogen_ext/tools/duckduckgo_search/_duckduckgo_search.py b/python/packages/autogen-ext/src/autogen_ext/tools/duckduckgo_search/_duckduckgo_search.py
deleted file mode 100644
index 92ec54a22..000000000
--- a/python/packages/autogen-ext/src/autogen_ext/tools/duckduckgo_search/_duckduckgo_search.py
+++ /dev/null
@@ -1,373 +0,0 @@
-# DuckDuckGo Search Tool created by Varad Srivastava
-from typing import Dict, List, Optional, Union
-from urllib.parse import parse_qs, quote_plus, unquote, urljoin, urlparse
-
-import html2text
-import httpx
-from autogen_core import CancellationToken
-from autogen_core.tools import BaseTool
-from bs4 import BeautifulSoup, NavigableString, Tag
-from pydantic import BaseModel, Field
-
-
-class DuckDuckGoSearchArgs(BaseModel):
- """Arguments for DuckDuckGo search."""
-
- query: str = Field(..., description="Search query string")
- num_results: int = Field(default=3, description="Number of results to return (max 10)")
- include_snippets: bool = Field(default=True, description="Include result snippets in output")
- include_content: bool = Field(default=True, description="Include full webpage content in markdown format")
- content_max_length: Optional[int] = Field(
- default=10000, description="Maximum length of webpage content (if included)"
- )
- language: str = Field(default="en", description="Language code for search results (e.g., en, es, fr)")
- region: Optional[str] = Field(default=None, description="Optional region code for search results (e.g., us, uk)")
- safe_search: bool = Field(default=True, description="Enable safe search filtering")
-
-
-class DuckDuckGoSearchResult(BaseModel):
- """Result from DuckDuckGo search."""
-
- results: List[Dict[str, str]] = Field(description="List of search results")
-
-
-class DuckDuckGoSearchTool(BaseTool[DuckDuckGoSearchArgs, DuckDuckGoSearchResult]):
- """A tool for performing DuckDuckGo web searches.
-
- This tool uses DuckDuckGo's HTML interface to perform web searches without requiring
- an API key. It can optionally fetch and convert webpage content to markdown format.
-
- Installation:
-
- .. code-block:: bash
-
- pip install "autogen-ext[duckduckgo-search]"
-
- Key Features:
-
- * No API key required
- * Supports multiple languages and regions
- * Optional webpage content fetching
- * Safe search filtering
- * Markdown conversion of webpage content
- * Configurable number of results
-
- Examples:
-
- Using the tool directly:
-
- .. code-block:: python
-
- from autogen_ext.tools.duckduckgo_search import DuckDuckGoSearchTool, DuckDuckGoSearchArgs
- from autogen_core import CancellationToken
- import asyncio
-
-
- async def main() -> None:
- # Create the search tool
- search_tool = DuckDuckGoSearchTool()
-
- # Create search arguments
- search_args = DuckDuckGoSearchArgs(
- query="What is AutoGen?", num_results=3, include_snippets=True, include_content=True
- )
-
- # Perform the search
- results = await search_tool.run(search_args, CancellationToken())
-
- # Print results
- print(f"Results as string:\\n{search_tool.return_value_as_string(results)}")
-
-
- asyncio.run(main())
-
- This will output something like:
-
- .. code-block:: text
-
- Result 1:
- Title: Microsoft AutoGen: The Next-Gen AI Framework
- URL: https://microsoft.github.io/autogen/
- Snippet: AutoGen is a framework that enables development of LLM applications...
- Content: AutoGen is a framework that enables the development of LLM applications...
-
- Basic usage with an :class:`~autogen_agentchat.agents.AssistantAgent`:
-
- .. code-block:: python
-
- import asyncio
- from autogen_ext.tools.duckduckgo_search import DuckDuckGoSearchTool
- from autogen_agentchat.agents import AssistantAgent
- from autogen_ext.models.openai import OpenAIChatCompletionClient
-
-
- async def main() -> None:
- # Create the search tool
- search_tool = DuckDuckGoSearchTool()
-
- # Create an agent with the search tool
- model_client = OpenAIChatCompletionClient(model="gpt-4")
- agent = AssistantAgent(name="search_agent", model_client=model_client, tools=[search_tool])
-
- # Run the agent with a search task
- response = await agent.run(task="Search for information about artificial intelligence")
- print(response.messages[-1].content) # type: ignore
-
-
- asyncio.run(main())
-
- .. note::
-
- The tool uses web scraping to fetch results, so it may be affected by:
- * DuckDuckGo's rate limiting
- * Changes to DuckDuckGo's HTML structure
- * Website blocking or requiring JavaScript
- * Network connectivity issues
-
- Returns:
- DuckDuckGoSearchResult: A result object.
- """
-
- def __init__(self) -> None:
- super().__init__(
- args_type=DuckDuckGoSearchArgs,
- return_type=DuckDuckGoSearchResult,
- name="duckduckgo_search",
- description="Perform DuckDuckGo searches with optional webpage content fetching. No API key required.",
- )
-
- async def run(self, args: DuckDuckGoSearchArgs, cancellation_token: CancellationToken) -> DuckDuckGoSearchResult:
- """Execute the DuckDuckGo search."""
- num_results = min(max(1, args.num_results), 10)
-
- async def fetch_page_content(url: str, max_length: Optional[int] = 50000) -> str:
- """Helper function to fetch and convert webpage content to markdown"""
- headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
-
- try:
- async with httpx.AsyncClient() as client:
- response = await client.get(url, headers=headers, timeout=10)
- response.raise_for_status()
-
- soup = BeautifulSoup(response.text, "html.parser")
-
- # Remove unwanted elements that typically contain navigation/boilerplate
- for element in soup(
- ["script", "style", "nav", "header", "footer", "aside", "menu", "form", "button", "input"]
- ):
- if hasattr(element, "decompose"): # type: ignore
- element.decompose() # type: ignore
-
- # Try to find main content areas first (in order of preference)
- main_content: Optional[Union[Tag, NavigableString]] = None # type: ignore
- content_selectors = [
- "main",
- "article",
- "[role='main']",
- ".content",
- ".post",
- ".entry",
- ".article-content",
- ".post-content",
- ".main-content",
- ".page-content",
- "#content",
- "#main",
- "#article",
- ".container .content",
- ]
-
- for selector in content_selectors:
- main_content = soup.select_one(selector) # type: ignore
- if main_content and main_content.get_text(strip=True):
- break
-
- # If no main content found, try to get the largest text block
- if not main_content or not main_content.get_text(strip=True):
- # Find all divs and get the one with most text content
- all_divs = soup.find_all("div")
- if all_divs:
- main_content = max(all_divs, key=lambda div: len(div.get_text(strip=True)))
-
- # Final fallback: use body but clean it up
- if not main_content or len(main_content.get_text(strip=True)) < 100:
- main_content = soup.find("body") or soup
-
- # Only remove clearly problematic elements for fallback
- if hasattr(main_content, "find_all"): # type: ignore
- for element in main_content.find_all(["iframe", "embed", "object", "video", "audio"]): # type: ignore
- if hasattr(element, "decompose"): # type: ignore
- element.decompose() # type: ignore
-
- # Remove elements with obvious navigation class names (less aggressive)
- for class_name in ["navigation", "navbar", "sidebar", "footer", "header"]:
- # Find elements with navigation class names using a simpler approach
- nav_elements = [] # type: ignore
- try:
- nav_elements = main_content.find_all(class_=class_name) # type: ignore
- except Exception: # type: ignore
- # If direct class search fails, try finding elements that contain the class name
- try:
- all_elements = main_content.find_all() # type: ignore
- for elem in all_elements: # type: ignore
- if hasattr(elem, "get") and elem.get("class"): # type: ignore
- elem_classes = elem.get("class") # type: ignore
- if isinstance(elem_classes, list) and any(
- class_name in str(cls).lower() # type: ignore
- for cls in elem_classes # type: ignore
- ): # type: ignore
- nav_elements.append(elem) # type: ignore
- except Exception: # type: ignore
- pass # type: ignore
-
- for element in nav_elements: # type: ignore
- if hasattr(element, "decompose"): # type: ignore
- element.decompose() # type: ignore
-
- # Convert relative URLs to absolute
- if hasattr(main_content, "find_all"): # type: ignore
- for tag in main_content.find_all(["a", "img"]): # type: ignore
- if hasattr(tag, "get"): # type: ignore
- if tag.get("href"): # type: ignore
- tag["href"] = urljoin(url, str(tag.get("href"))) # type: ignore
- if tag.get("src"): # type: ignore
- tag["src"] = urljoin(url, str(tag.get("src"))) # type: ignore
-
- h2t = html2text.HTML2Text()
- h2t.body_width = 0
- h2t.ignore_images = True # Ignore images to reduce noise
- h2t.ignore_emphasis = False
- h2t.ignore_links = True # Ignore links to reduce noise
- h2t.ignore_tables = False
-
- markdown = h2t.handle(str(main_content))
-
- # Clean up the markdown - remove excessive whitespace and empty lines
- lines: List[str] = markdown.split("\n")
- cleaned_lines: List[str] = []
- for line in lines:
- line = line.strip()
- # Keep lines that have content (not just headers or empty)
- if line and (not line.startswith("#") or len(line.split()) > 1):
- cleaned_lines.append(line)
-
- # Remove consecutive empty lines but keep some structure
- final_lines: List[str] = []
- prev_empty = False
- for line in cleaned_lines:
- if line.strip():
- final_lines.append(line)
- prev_empty = False
- elif not prev_empty and len(final_lines) > 0:
- final_lines.append("")
- prev_empty = True
-
- markdown = "\n".join(final_lines)
-
- # If we still don't have much content, return a basic text extraction
- if len(markdown.strip()) < 50:
- text_content = (
- main_content.get_text(separator="\n", strip=True)
- if hasattr(main_content, "get_text")
- else ""
- )
- if text_content:
- lines = [line.strip() for line in text_content.split("\n") if line.strip()]
- markdown = "\n".join(lines[:50]) # Take first 50 non-empty lines
- else:
- markdown = "Content could not be extracted from this page."
-
- if max_length and len(markdown) > max_length:
- markdown = markdown[:max_length] + "\n...(truncated)"
-
- return markdown.strip()
-
- except Exception as e:
- return f"Error fetching content: {str(e)}"
-
- # Build DuckDuckGo search URL
- params = {
- "q": args.query,
- "kl": args.language,
- "kad": args.region.upper() if args.region else None,
- "safesearch": "1" if args.safe_search else "0",
- }
-
- # Remove None values
- params = {k: v for k, v in params.items() if v is not None}
-
- search_url = f"https://html.duckduckgo.com/html/?{quote_plus(args.query)}"
-
- try:
- async with httpx.AsyncClient() as client:
- response = await client.get(search_url, params=params, timeout=10)
- response.raise_for_status()
-
- soup = BeautifulSoup(response.text, "html.parser")
- results: List[Dict[str, str]] = []
-
- # Find all result elements
- result_elements = soup.find_all("div", class_="result")
-
- for element in result_elements[:num_results]:
- title_element = element.find("a", class_="result__a")
- snippet_element = element.find("a", class_="result__snippet")
-
- if title_element:
- raw_link = title_element.get("href", "") or ""
-
- # Handle DuckDuckGo redirect URLs and protocol-relative URLs
- if raw_link.startswith("//duckduckgo.com/l/"):
- # Extract the actual URL from DuckDuckGo redirect
- parsed = parse_qs(urlparse(raw_link).query)
- actual_url_list = parsed.get("uddg", [""])
- if actual_url_list and actual_url_list[0]:
- link = unquote(actual_url_list[0])
- else:
- link = "https:" + raw_link
- elif raw_link.startswith("//"):
- # Protocol-relative URL
- link = "https:" + raw_link
- elif raw_link.startswith("/"):
- # Relative URL
- link = "https://duckduckgo.com" + raw_link
- else:
- # Absolute URL
- link = raw_link
-
- result: Dict[str, str] = {"title": title_element.get_text(strip=True), "link": link}
-
- if args.include_snippets and snippet_element:
- result["snippet"] = snippet_element.get_text(strip=True)
-
- if args.include_content and link.startswith(("http://", "https://")):
- result["content"] = await fetch_page_content(link, max_length=args.content_max_length)
- elif args.include_content:
- result["content"] = "Error: Invalid URL format"
-
- results.append(result)
-
- return DuckDuckGoSearchResult(results=results)
-
- except httpx.RequestError as e:
- raise ValueError(f"Failed to perform search: {str(e)}") from e
- except Exception as e:
- raise ValueError(f"Error during search: {str(e)}") from e
-
- def return_value_as_string(self, value: DuckDuckGoSearchResult) -> str:
- """Convert the search results to a human/LLM-readable string format."""
- if not value.results:
- return "No search results found."
-
- parts: List[str] = []
- for i, result in enumerate(value.results, 1):
- parts.append(f"Result {i}:")
- parts.append(f"Title: {result['title']}")
- parts.append(f"URL: {result['link']}")
- if "snippet" in result:
- parts.append(f"Snippet: {result['snippet']}")
- if "content" in result:
- parts.append(f"Content: {result['content']}")
- parts.append("") # Empty line between results
-
- return "\n".join(parts)
diff --git a/python/packages/autogen-ext/tests/tools/duckduckgo_search/__init__.py b/python/packages/autogen-ext/tests/tools/duckduckgo_search/__init__.py
deleted file mode 100644
index 4f6ab69a9..000000000
--- a/python/packages/autogen-ext/tests/tools/duckduckgo_search/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# Tests for web search tools
diff --git a/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_agent.py b/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_agent.py
deleted file mode 100644
index d2b64cdad..000000000
--- a/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_agent.py
+++ /dev/null
@@ -1,116 +0,0 @@
-import pytest
-from autogen_core.models import ModelInfo
-from autogen_ext.agents.duckduckgo_search._duckduckgo_agent import DuckDuckGoSearchAgent
-from autogen_ext.models.replay import ReplayChatCompletionClient
-from autogen_ext.tools.duckduckgo_search import DuckDuckGoSearchTool
-
-
-class TestDuckDuckGoSearchAgent:
- """Test suite for DuckDuckGoSearchAgent."""
-
- @pytest.fixture
- def mock_model_client(self) -> ReplayChatCompletionClient:
- """Create a replay model client for testing."""
- model_info = ModelInfo(
- vision=False,
- function_calling=True,
- json_output=False,
- family="test",
- structured_output=False,
- )
- return ReplayChatCompletionClient(chat_completions=["test response"], model_info=model_info)
-
- @pytest.fixture
- def search_agent(self, mock_model_client: ReplayChatCompletionClient) -> DuckDuckGoSearchAgent:
- """Create a DuckDuckGoSearchAgent instance for testing."""
- return DuckDuckGoSearchAgent(name="test_agent", model_client=mock_model_client)
-
- def test_agent_initialization_with_defaults(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test that the agent initializes correctly with default values."""
- agent = DuckDuckGoSearchAgent(name="test_agent", model_client=mock_model_client)
-
- assert agent.name == "test_agent"
- assert "research assistant" in agent.description.lower()
- assert "duckduckgo" in agent.description.lower()
-
- # Check that the agent has the DuckDuckGo search tool
- assert len(agent._tools) == 1 # type: ignore
- assert isinstance(agent._tools[0], DuckDuckGoSearchTool) # type: ignore
- assert agent._tools[0].name == "duckduckgo_search" # type: ignore
-
- def test_agent_initialization_with_custom_values(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test that the agent initializes correctly with custom values."""
- custom_description = "Custom research agent"
- custom_system_message = "You are a custom search agent."
-
- agent = DuckDuckGoSearchAgent(
- name="custom_agent",
- model_client=mock_model_client,
- description=custom_description,
- system_message=custom_system_message,
- )
-
- assert agent.name == "custom_agent"
- assert agent.description == custom_description
- # Note: We can't directly access system_message, but we can verify it was passed
-
- def test_agent_has_search_tool(self, search_agent: DuckDuckGoSearchAgent) -> None:
- """Test that the agent has the DuckDuckGo search tool configured."""
- tools = search_agent._tools # type: ignore
- assert len(tools) == 1
-
- search_tool = tools[0]
- assert isinstance(search_tool, DuckDuckGoSearchTool)
- assert search_tool.name == "duckduckgo_search"
-
- def test_agent_system_message_content(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test that the default system message contains expected content."""
- agent = DuckDuckGoSearchAgent(name="test_agent", model_client=mock_model_client)
-
- # The system message should be in the agent's system messages
- system_messages = agent._system_messages # type: ignore
- assert len(system_messages) == 1
-
- system_content = system_messages[0].content
- assert "research assistant" in system_content.lower()
- assert "duckduckgo" in system_content.lower()
- assert "search" in system_content.lower()
-
- def test_agent_inherits_from_assistant_agent(self, search_agent: DuckDuckGoSearchAgent) -> None:
- """Test that DuckDuckGoSearchAgent properly inherits from AssistantAgent."""
- from autogen_agentchat.agents import AssistantAgent
-
- assert isinstance(search_agent, AssistantAgent)
-
- def test_agent_model_client_assignment(
- self, search_agent: DuckDuckGoSearchAgent, mock_model_client: ReplayChatCompletionClient
- ) -> None:
- """Test that the model client is properly assigned."""
- assert search_agent._model_client == mock_model_client # type: ignore
-
- def test_agent_with_additional_kwargs(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test that additional kwargs are passed to the parent class."""
- agent = DuckDuckGoSearchAgent(
- name="test_agent", model_client=mock_model_client, model_client_stream=True, reflect_on_tool_use=True
- )
-
- assert agent._model_client_stream is True # type: ignore
- assert agent._reflect_on_tool_use is True # type: ignore
-
- def test_agent_description_default(self, search_agent: DuckDuckGoSearchAgent) -> None:
- """Test the default description content."""
- description = search_agent.description
- expected_keywords = ["research", "assistant", "duckduckgo", "web", "information"]
-
- for keyword in expected_keywords:
- assert keyword.lower() in description.lower()
-
- def test_multiple_agents_have_separate_tools(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test that multiple agents have separate tool instances."""
- agent1 = DuckDuckGoSearchAgent(name="agent1", model_client=mock_model_client)
- agent2 = DuckDuckGoSearchAgent(name="agent2", model_client=mock_model_client)
-
- # Each agent should have its own tool instance
- assert agent1._tools[0] is not agent2._tools[0] # type: ignore
- assert isinstance(agent1._tools[0], DuckDuckGoSearchTool) # type: ignore
- assert isinstance(agent2._tools[0], DuckDuckGoSearchTool) # type: ignore
diff --git a/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_integration.py b/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_integration.py
deleted file mode 100644
index 4d27d1755..000000000
--- a/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_integration.py
+++ /dev/null
@@ -1,203 +0,0 @@
-"""
-Integration tests for DuckDuckGo Search Agent.
-
-These tests demonstrate the full functionality of the DuckDuckGo search agent
-and can be used to verify that the implementation works as expected.
-
-Note: These tests make actual network requests to DuckDuckGo and may be slower
-or fail if there are network issues. They are marked as integration tests
-and can be skipped in CI environments.
-"""
-
-from typing import Any, Dict, List
-
-import pytest
-from autogen_core import CancellationToken, FunctionCall
-from autogen_core.models import CreateResult, ModelInfo, RequestUsage
-from autogen_ext.agents.duckduckgo_search._duckduckgo_agent import DuckDuckGoSearchAgent
-from autogen_ext.models.replay import ReplayChatCompletionClient
-from autogen_ext.tools.duckduckgo_search import DuckDuckGoSearchArgs, DuckDuckGoSearchTool
-
-
-class TestDuckDuckGoIntegration:
- """Integration tests for DuckDuckGo functionality."""
-
- @pytest.fixture
- def mock_model_client(self) -> ReplayChatCompletionClient:
- """Create a replay model client for testing."""
- model_info = ModelInfo(
- vision=False,
- function_calling=True,
- json_output=False,
- family="test",
- structured_output=False,
- )
-
- # Create responses that simulate tool calling behavior
- tool_call_response = CreateResult(
- content=[
- FunctionCall(
- id="call_123",
- name="duckduckgo_search",
- arguments='{"query": "Python programming", "num_results": 2, "include_content": false}',
- )
- ],
- usage=RequestUsage(prompt_tokens=0, completion_tokens=0),
- cached=False,
- finish_reason="function_calls",
- )
-
- final_response = CreateResult(
- content="Based on my search results, I found information about Python programming. Python is a popular programming language known for its simplicity and versatility.",
- usage=RequestUsage(prompt_tokens=0, completion_tokens=0),
- cached=False,
- finish_reason="stop",
- )
-
- return ReplayChatCompletionClient(chat_completions=[tool_call_response, final_response], model_info=model_info)
-
- @pytest.mark.asyncio
- async def test_search_tool_real_request(self) -> None:
- """Test the search tool with a real request to DuckDuckGo."""
- search_tool = DuckDuckGoSearchTool()
-
- # Test with a simple query
- args = DuckDuckGoSearchArgs(
- query="Python programming language",
- num_results=2,
- include_content=False, # Skip content fetching for faster test
- include_snippets=True,
- )
-
- result = await search_tool.run(args, CancellationToken())
-
- # Verify we got results
- assert len(result.results) > 0
- assert len(result.results) <= 2
-
- # Check that each result has the expected structure
- for search_result in result.results:
- assert "title" in search_result
- assert "link" in search_result
- assert search_result["title"] # Should not be empty
- assert search_result["link"].startswith("http") # Should be a valid URL
-
- if args.include_snippets:
- assert "snippet" in search_result
-
- @pytest.mark.asyncio
- async def test_agent_with_mock_model(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test the agent with a mock model client that simulates tool usage."""
- agent = DuckDuckGoSearchAgent(name="test_researcher", model_client=mock_model_client)
-
- # This test just verifies the agent is set up correctly with mock model
- assert agent.name == "test_researcher"
- assert len(agent._tools) == 1 # type: ignore
- assert agent._tools[0].name == "duckduckgo_search" # type: ignore
-
- @pytest.mark.asyncio
- async def test_search_with_content_fetching(self) -> None:
- """Test search with content fetching enabled (slower test)."""
- search_tool = DuckDuckGoSearchTool()
-
- args = DuckDuckGoSearchArgs(
- query="OpenAI",
- num_results=1,
- include_content=True,
- content_max_length=500, # Limit content length for faster test
- )
-
- result = await search_tool.run(args, CancellationToken())
-
- # Verify we got results with content
- assert len(result.results) > 0
- first_result = result.results[0]
-
- assert "title" in first_result
- assert "link" in first_result
- assert "content" in first_result
- assert first_result["content"] # Should not be empty
-
- # Content should be markdown-formatted
- content = first_result["content"]
- assert len(content) <= 500 + 20 # Allow some margin for truncation message
-
- @pytest.mark.asyncio
- async def test_search_error_handling(self) -> None:
- """Test that the search tool handles errors gracefully."""
- search_tool = DuckDuckGoSearchTool()
-
- # Test with an invalid/problematic query
- args = DuckDuckGoSearchArgs(
- query="", # Empty query might cause issues
- num_results=1,
- include_content=False,
- )
-
- # This should either return empty results or handle the error gracefully
- try:
- result = await search_tool.run(args, CancellationToken())
- # If it succeeds, verify the result structure
- assert hasattr(result, "results")
- assert isinstance(result.results, list)
- except ValueError as e:
- # If it fails, it should be a ValueError with a descriptive message
- assert "search" in str(e).lower() or "error" in str(e).lower()
-
- def test_agent_configuration(self, mock_model_client: ReplayChatCompletionClient) -> None:
- """Test that the agent is configured correctly."""
- agent = DuckDuckGoSearchAgent(name="config_test", model_client=mock_model_client)
-
- # Verify basic configuration
- assert agent.name == "config_test"
- assert "duckduckgo" in agent.description.lower()
-
- # Verify tool configuration
- assert len(agent._tools) == 1 # type: ignore
- search_tool = agent._tools[0] # type: ignore
- assert isinstance(search_tool, DuckDuckGoSearchTool)
- assert search_tool.name == "duckduckgo_search"
-
- # Verify system message contains expected content
- system_messages = agent._system_messages # type: ignore
- assert len(system_messages) == 1
- system_content = system_messages[0].content.lower()
- assert "research" in system_content
- assert "duckduckgo" in system_content
- assert "search" in system_content
-
- @pytest.mark.asyncio
- async def test_search_different_parameters(self) -> None:
- """Test search with different parameter combinations."""
- search_tool = DuckDuckGoSearchTool()
-
- # Test with different languages and regions
- test_cases: List[Dict[str, Any]] = [
- {"query": "machine learning", "language": "en", "region": "us"},
- {"query": "artificial intelligence", "num_results": 1, "safe_search": True},
- {"query": "programming", "include_snippets": False, "include_content": False},
- ]
-
- for case in test_cases:
- args = DuckDuckGoSearchArgs(**case)
- result = await search_tool.run(args, CancellationToken())
-
- # Basic validation
- assert hasattr(result, "results")
- assert isinstance(result.results, list)
-
- # If we got results, verify structure
- if result.results:
- first_result = result.results[0]
- assert "title" in first_result
- assert "link" in first_result
-
- # Check snippets based on parameter
- include_snippets = case.get("include_snippets", True)
- if include_snippets:
- # Snippets might not always be available
- pass
- else:
- # When snippets are disabled, they shouldn't be included
- assert "snippet" not in first_result or not first_result["snippet"]
diff --git a/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_search.py b/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_search.py
deleted file mode 100644
index d0a428e79..000000000
--- a/python/packages/autogen-ext/tests/tools/duckduckgo_search/test_duckduckgo_search.py
+++ /dev/null
@@ -1,734 +0,0 @@
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import httpx
-import pytest
-from autogen_core import CancellationToken
-from autogen_ext.tools.duckduckgo_search import (
- DuckDuckGoSearchArgs,
- DuckDuckGoSearchResult,
- DuckDuckGoSearchTool,
-)
-
-
-class TestDuckDuckGoSearchTool:
- """Test suite for DuckDuckGoSearchTool."""
-
- @pytest.fixture
- def search_tool(self) -> DuckDuckGoSearchTool:
- """Create a DuckDuckGoSearchTool instance for testing."""
- return DuckDuckGoSearchTool()
-
- @pytest.fixture
- def search_args(self) -> DuckDuckGoSearchArgs:
- """Create default search arguments for testing."""
- return DuckDuckGoSearchArgs(
- query="test query",
- num_results=3,
- include_snippets=True,
- include_content=False, # Disable content fetching for faster tests
- language="en",
- safe_search=True,
- )
-
- @pytest.fixture
- def mock_html_response(self) -> str:
- """Mock HTML response from DuckDuckGo."""
- return """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="https://example.com/1">Test Result 1</a>
- <a class="result__snippet">This is a test snippet 1</a>
- </div>
- <div class="result">
- <a class="result__a" href="https://example.com/2">Test Result 2</a>
- <a class="result__snippet">This is a test snippet 2</a>
- </div>
- <div class="result">
- <a class="result__a" href="https://example.com/3">Test Result 3</a>
- <a class="result__snippet">This is a test snippet 3</a>
- </div>
- </body>
- </html>
- """
-
- def test_search_args_validation(self) -> None:
- """Test that search arguments are validated correctly."""
- # Valid args
- args = DuckDuckGoSearchArgs(query="test")
- assert args.query == "test"
- assert args.num_results == 3 # default
- assert args.include_snippets is True # default
-
- # Test with custom values
- args = DuckDuckGoSearchArgs(
- query="custom query", num_results=5, include_snippets=False, language="es", region="us"
- )
- assert args.query == "custom query"
- assert args.num_results == 5
- assert args.include_snippets is False
- assert args.language == "es"
- assert args.region == "us"
-
- @pytest.mark.asyncio
- async def test_successful_search(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs, mock_html_response: str
- ) -> None:
- """Test a successful search operation."""
- # Mock the HTTP response
- mock_response = MagicMock()
- mock_response.text = mock_html_response
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert isinstance(result, DuckDuckGoSearchResult)
- assert len(result.results) == 3
-
- # Check first result
- first_result = result.results[0]
- assert first_result["title"] == "Test Result 1"
- assert first_result["link"] == "https://example.com/1"
- assert first_result["snippet"] == "This is a test snippet 1"
-
- @pytest.mark.asyncio
- async def test_search_with_content_fetching(
- self, search_tool: DuckDuckGoSearchTool, mock_html_response: str
- ) -> None:
- """Test search with content fetching enabled."""
- search_args = DuckDuckGoSearchArgs(
- query="test query", num_results=1, include_content=True, content_max_length=1000
- )
-
- # Mock the search response
- mock_search_response = MagicMock()
- mock_search_response.text = mock_html_response
- mock_search_response.raise_for_status = MagicMock()
-
- # Mock the content response
- mock_content_response = MagicMock()
- mock_content_response.text = "<html><body><h1>Test Content</h1><p>This is test content.</p></body></html>"
- mock_content_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.side_effect = [mock_search_response, mock_content_response]
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- assert "content" in result.results[0]
- assert "Test Content" in result.results[0]["content"]
-
- @pytest.mark.asyncio
- async def test_search_without_snippets(self, search_tool: DuckDuckGoSearchTool, mock_html_response: str) -> None:
- """Test search with snippets disabled."""
- search_args = DuckDuckGoSearchArgs(query="test query", include_snippets=False, include_content=False)
-
- mock_response = MagicMock()
- mock_response.text = mock_html_response
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- # Snippets should not be included
- for search_result in result.results:
- assert "snippet" not in search_result or search_result["snippet"] == ""
-
- @pytest.mark.asyncio
- async def test_num_results_limit(self, search_tool: DuckDuckGoSearchTool, mock_html_response: str) -> None:
- """Test that num_results is properly limited."""
- # Test with more than 10 results requested
- search_args = DuckDuckGoSearchArgs(
- query="test query",
- num_results=2, # Should be limited to 2
- include_content=False,
- )
-
- mock_response = MagicMock()
- mock_response.text = mock_html_response
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- # Should return only 2 results (from mock HTML)
- assert len(result.results) == 2
-
- @pytest.mark.asyncio
- async def test_http_error_handling(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of HTTP errors."""
- mock_client = AsyncMock()
- mock_client.get.side_effect = Exception("Network error")
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- with pytest.raises(ValueError, match="Error during search"):
- await search_tool.run(search_args, CancellationToken())
-
- @pytest.mark.asyncio
- async def test_content_fetch_error_handling(
- self, search_tool: DuckDuckGoSearchTool, mock_html_response: str
- ) -> None:
- """Test handling of content fetching errors."""
- search_args = DuckDuckGoSearchArgs(query="test query", num_results=1, include_content=True)
-
- # Mock successful search but failed content fetch
- mock_search_response = MagicMock()
- mock_search_response.text = mock_html_response
- mock_search_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- # First call succeeds (search), second call fails (content fetch)
- mock_client.get.side_effect = [mock_search_response, Exception("Content fetch error")]
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- # Should still return results, but with error message in content
- assert len(result.results) == 1
- assert "Error fetching content" in result.results[0]["content"]
-
- @pytest.mark.asyncio
- async def test_empty_search_results(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of empty search results."""
- empty_html = "<html><body></body></html>"
-
- mock_response = MagicMock()
- mock_response.text = empty_html
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert isinstance(result, DuckDuckGoSearchResult)
- assert len(result.results) == 0
-
- def test_search_args_defaults(self) -> None:
- """Test that search arguments have correct defaults."""
- args = DuckDuckGoSearchArgs(query="test")
-
- assert args.num_results == 3
- assert args.include_snippets is True
- assert args.include_content is True
- assert args.content_max_length == 10000
- assert args.language == "en"
- assert args.region is None
- assert args.safe_search is True
-
- def test_search_args_edge_cases(self) -> None:
- """Test edge cases for search arguments."""
- # Test empty query handling in validation
- args = DuckDuckGoSearchArgs(query="")
- assert args.query == ""
-
- # Test maximum num_results
- args = DuckDuckGoSearchArgs(query="test", num_results=50)
- assert args.num_results == 50
-
- # Test zero num_results
- args = DuckDuckGoSearchArgs(query="test", num_results=0)
- assert args.num_results == 0
-
- # Test negative num_results
- args = DuckDuckGoSearchArgs(query="test", num_results=-5)
- assert args.num_results == -5
-
- # Test very long content_max_length
- args = DuckDuckGoSearchArgs(query="test", content_max_length=1000000)
- assert args.content_max_length == 1000000
-
- # Test None content_max_length
- args = DuckDuckGoSearchArgs(query="test", content_max_length=None)
- assert args.content_max_length is None
-
- @pytest.mark.asyncio
- async def test_search_url_building(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test that search URLs are built correctly with different parameters."""
- search_args = DuckDuckGoSearchArgs(
- query="test query with spaces", language="es", region="mx", safe_search=False
- )
-
- mock_response = MagicMock()
- mock_response.text = "<html><body></body></html>"
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- await search_tool.run(search_args, CancellationToken())
-
- # Verify the client.get was called with correct parameters
- mock_client.get.assert_called_once()
- call_args = mock_client.get.call_args
-
- # Check URL contains query
- assert "test%2Bquery%2Bwith%2Bspaces" in call_args[0][0] or "test+query+with+spaces" in call_args[0][0]
-
- # Check parameters
- params = call_args[1]["params"]
- assert params["kl"] == "es"
- assert params["kad"] == "MX"
- assert params["safesearch"] == "0"
-
- @pytest.mark.asyncio
- async def test_duckduckgo_redirect_url_handling(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of DuckDuckGo redirect URLs."""
- redirect_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A//example.com/test%3Fq%3Dvalue">Test Result</a>
- <a class="result__snippet">Test snippet</a>
- </div>
- </body>
- </html>
- """
-
- mock_response = MagicMock()
- mock_response.text = redirect_html
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- assert result.results[0]["link"] == "https://example.com/test?q=value"
-
- @pytest.mark.asyncio
- async def test_protocol_relative_url_handling(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of protocol-relative URLs."""
- protocol_relative_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="//example.com/test">Test Result</a>
- <a class="result__snippet">Test snippet</a>
- </div>
- </body>
- </html>
- """
-
- mock_response = MagicMock()
- mock_response.text = protocol_relative_html
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- assert result.results[0]["link"] == "https://example.com/test"
-
- @pytest.mark.asyncio
- async def test_relative_url_handling(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of relative URLs."""
- relative_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="/relative/path">Test Result</a>
- <a class="result__snippet">Test snippet</a>
- </div>
- </body>
- </html>
- """
-
- mock_response = MagicMock()
- mock_response.text = relative_html
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- assert result.results[0]["link"] == "https://duckduckgo.com/relative/path"
-
- @pytest.mark.asyncio
- async def test_malformed_html_handling(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of malformed HTML."""
- malformed_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a">Missing href attribute</a>
- <a class="result__snippet">Test snippet</a>
- </div>
- <div class="result">
- <a class="result__a" href="https://example.com">Valid link without title</a>
- </div>
- </body>
- </html>
- """
-
- mock_response = MagicMock()
- mock_response.text = malformed_html
- mock_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- # Should handle malformed HTML gracefully
- assert len(result.results) == 2
-
- # First result should have empty link due to missing href
- assert result.results[0]["link"] == ""
- assert result.results[0]["title"] == "Missing href attribute"
-
- # Second result should be valid
- assert result.results[1]["link"] == "https://example.com"
- assert result.results[1]["title"] == "Valid link without title"
-
- @pytest.mark.asyncio
- async def test_content_fetching_with_various_html_structures(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test content fetching with different HTML structures."""
- search_args = DuckDuckGoSearchArgs(query="test", num_results=1, include_content=True)
-
- search_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="https://example.com">Test Result</a>
- <a class="result__snippet">Test snippet</a>
- </div>
- </body>
- </html>
- """
-
- # Test content with main tag
- content_with_main = """
- <html>
- <body>
- <header>Header content</header>
- <nav>Navigation</nav>
- <main>
- <h1>Main Content Title</h1>
- <p>This is the main content that should be extracted.</p>
- </main>
- <footer>Footer content</footer>
- </body>
- </html>
- """
-
- mock_search_response = MagicMock()
- mock_search_response.text = search_html
- mock_search_response.raise_for_status = MagicMock()
-
- mock_content_response = MagicMock()
- mock_content_response.text = content_with_main
- mock_content_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.side_effect = [mock_search_response, mock_content_response]
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- content = result.results[0]["content"]
- assert "Main Content Title" in content
- assert "main content that should be extracted" in content
- # Should not contain header, nav, or footer content
- assert "Header content" not in content
- assert "Navigation" not in content
- assert "Footer content" not in content
-
- @pytest.mark.asyncio
- async def test_content_fetching_fallback_to_largest_div(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test content fetching fallback to largest div when no main content found."""
- search_args = DuckDuckGoSearchArgs(query="test", num_results=1, include_content=True)
-
- search_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="https://example.com">Test Result</a>
- </div>
- </body>
- </html>
- """
-
- # Content with no main tag, should fallback to largest div
- content_no_main = """
- <html>
- <body>
- <div>Small content</div>
- <div>
- <h1>Large Content Section</h1>
- <p>This is a much larger content section that should be selected as the main content when no main tag is present. It contains multiple paragraphs and more text.</p>
- <p>Another paragraph with more content to make this div larger.</p>
- </div>
- <div>Another small div</div>
- </body>
- </html>
- """
-
- mock_search_response = MagicMock()
- mock_search_response.text = search_html
- mock_search_response.raise_for_status = MagicMock()
-
- mock_content_response = MagicMock()
- mock_content_response.text = content_no_main
- mock_content_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.side_effect = [mock_search_response, mock_content_response]
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- content = result.results[0]["content"]
- assert "Large Content Section" in content
- assert "larger content section" in content
- assert "Small content" not in content
-
- @pytest.mark.asyncio
- async def test_content_fetching_timeout_handling(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test handling of timeout errors during content fetching."""
- search_args = DuckDuckGoSearchArgs(query="test", num_results=1, include_content=True)
-
- search_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="https://example.com">Test Result</a>
- </div>
- </body>
- </html>
- """
-
- mock_search_response = MagicMock()
- mock_search_response.text = search_html
- mock_search_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- # First call succeeds (search), second call times out (content fetch)
- mock_client.get.side_effect = [mock_search_response, httpx.TimeoutException("Timeout")]
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- assert "Error fetching content: Timeout" in result.results[0]["content"]
-
- @pytest.mark.asyncio
- async def test_content_length_truncation(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test that content is properly truncated when it exceeds max length."""
- search_args = DuckDuckGoSearchArgs(query="test", num_results=1, include_content=True, content_max_length=100)
-
- search_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="https://example.com">Test Result</a>
- </div>
- </body>
- </html>
- """
-
- # Content that's longer than max_length
- long_content = (
- """
- <html>
- <body>
- <main>
- <p>"""
- + "This is a very long paragraph that should be truncated. " * 20
- + """</p>
- </main>
- </body>
- </html>
- """
- )
-
- mock_search_response = MagicMock()
- mock_search_response.text = search_html
- mock_search_response.raise_for_status = MagicMock()
-
- mock_content_response = MagicMock()
- mock_content_response.text = long_content
- mock_content_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.side_effect = [mock_search_response, mock_content_response]
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- content = result.results[0]["content"]
- assert len(content) <= 115 # 100 + "...(truncated)"
- assert content.endswith("...(truncated)")
-
- @pytest.mark.asyncio
- async def test_invalid_url_content_handling(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test handling of invalid URLs when fetching content."""
- search_args = DuckDuckGoSearchArgs(query="test", num_results=1, include_content=True)
-
- search_html = """
- <html>
- <body>
- <div class="result">
- <a class="result__a" href="invalid-url">Test Result</a>
- </div>
- </body>
- </html>
- """
-
- mock_search_response = MagicMock()
- mock_search_response.text = search_html
- mock_search_response.raise_for_status = MagicMock()
-
- mock_client = AsyncMock()
- mock_client.get.return_value = mock_search_response
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- result = await search_tool.run(search_args, CancellationToken())
-
- assert len(result.results) == 1
- assert result.results[0]["content"] == "Error: Invalid URL format"
-
- @pytest.mark.asyncio
- async def test_httpx_request_error_handling(
- self, search_tool: DuckDuckGoSearchTool, search_args: DuckDuckGoSearchArgs
- ) -> None:
- """Test handling of httpx RequestError."""
- mock_client = AsyncMock()
- mock_client.get.side_effect = httpx.RequestError("Connection failed")
-
- with patch("httpx.AsyncClient") as mock_async_client:
- mock_async_client.return_value.__aenter__.return_value = mock_client
-
- with pytest.raises(ValueError, match="Failed to perform search: Connection failed"):
- await search_tool.run(search_args, CancellationToken())
-
- def test_return_value_as_string_empty_results(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test return_value_as_string with empty results."""
- empty_result = DuckDuckGoSearchResult(results=[])
- string_output = search_tool.return_value_as_string(empty_result)
- assert string_output == "No search results found."
-
- def test_return_value_as_string_with_all_fields(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test return_value_as_string with all possible fields."""
- result = DuckDuckGoSearchResult(
- results=[
- {
- "title": "Test Title 1",
- "link": "https://example.com/1",
- "snippet": "Test snippet 1",
- "content": "Test content 1",
- },
- {
- "title": "Test Title 2",
- "link": "https://example.com/2",
- "snippet": "Test snippet 2",
- "content": "Test content 2",
- },
- ]
- )
-
- string_output = search_tool.return_value_as_string(result)
-
- assert "Result 1:" in string_output
- assert "Title: Test Title 1" in string_output
- assert "URL: https://example.com/1" in string_output
- assert "Snippet: Test snippet 1" in string_output
- assert "Content: Test content 1" in string_output
-
- assert "Result 2:" in string_output
- assert "Title: Test Title 2" in string_output
- assert "URL: https://example.com/2" in string_output
- assert "Snippet: Test snippet 2" in string_output
- assert "Content: Test content 2" in string_output
-
- def test_return_value_as_string_missing_fields(self, search_tool: DuckDuckGoSearchTool) -> None:
- """Test return_value_as_string with missing optional fields."""
- result = DuckDuckGoSearchResult(
- results=[
- {
- "title": "Test Title",
- "link": "https://example.com",
- # No snippet or content
- }
- ]
- )
-
- string_output = search_tool.return_value_as_string(result)
-
- assert "Result 1:" in string_output
- assert "Title: Test Title" in string_output
- assert "URL: https://example.com" in string_output
- assert "Snippet:" not in string_output
- assert "Content:" not in string_output
diff --git a/python/uv.lock b/python/uv.lock
index 17f7c434c..3afd5fefe 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -609,12 +609,6 @@ docker-jupyter-executor = [
{ name = "requests" },
{ name = "websockets" },
]
-duckduckgo-search = [
- { name = "autogen-agentchat" },
- { name = "beautifulsoup4" },
- { name = "html2text" },
- { name = "httpx" },
-]
file-surfer = [
{ name = "autogen-agentchat" },
{ name = "magika" },
@@ -742,7 +736,6 @@ requires-dist = [
{ name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.48" },
{ name = "asyncio-atexit", marker = "extra == 'docker'", specifier = ">=1.0.1" },
{ name = "asyncio-atexit", marker = "extra == 'docker-jupyter-executor'", specifier = ">=1.0.1" },
- { name = "autogen-agentchat", marker = "extra == 'duckduckgo-search'", editable = "packages/autogen-agentchat" },
{ name = "autogen-agentchat", marker = "extra == 'file-surfer'", editable = "packages/autogen-agentchat" },
{ name = "autogen-agentchat", marker = "extra == 'magentic-one'", editable = "packages/autogen-agentchat" },
{ name = "autogen-agentchat", marker = "extra == 'video-surfer'", editable = "packages/autogen-agentchat" },
@@ -753,7 +746,6 @@ requires-dist = [
{ name = "azure-core", marker = "extra == 'azure'" },
{ name = "azure-identity", marker = "extra == 'azure'" },
{ name = "azure-search-documents", marker = "extra == 'azure'", specifier = ">=11.4.0" },
- { name = "beautifulsoup4", marker = "extra == 'duckduckgo-search'", specifier = ">=4.12.0" },
{ name = "chromadb", marker = "extra == 'chromadb'", specifier = ">=1.0.0" },
{ name = "chromadb", marker = "extra == 'mem0-local'", specifier = ">=1.0.0" },
{ name = "chromadb", marker = "extra == 'task-centric-memory'", specifier = ">=1.0.0" },
@@ -764,8 +756,6 @@ requires-dist = [
{ name = "google-genai", marker = "extra == 'gemini'", specifier = ">=1.0.0" },
{ name = "graphrag", marker = "extra == 'graphrag'", specifier = ">=1.0.1" },
{ name = "grpcio", marker = "extra == 'grpc'", specifier = "~=1.70.0" },
- { name = "html2text", marker = "extra == 'duckduckgo-search'", specifier = ">=2024.2.26" },
- { name = "httpx", marker = "extra == 'duckduckgo-search'", specifier = ">=0.27.0" },
{ name = "httpx", marker = "extra == 'http-tool'", specifier = ">=0.27.0" },
{ name = "ipykernel", marker = "extra == 'jupyter-executor'", specifier = ">=6.29.5" },
{ name = "json-schema-to-pydantic", marker = "extra == 'http-tool'", specifier = ">=0.2.0" },
@@ -809,7 +799,7 @@ requires-dist = [
{ name = "unidiff", marker = "extra == 'canvas'", specifier = ">=0.7.5" },
{ name = "websockets", marker = "extra == 'docker-jupyter-executor'", specifier = ">=15.0.1" },
]
-provides-extras = ["anthropic", "langchain", "azure", "docker", "ollama", "openai", "file-surfer", "llama-cpp", "graphrag", "chromadb", "mem0", "mem0-local", "web-surfer", "magentic-one", "video-surfer", "diskcache", "redis", "grpc", "jupyter-executor", "docker-jupyter-executor", "task-centric-memory", "semantic-kernel-core", "gemini", "semantic-kernel-google", "semantic-kernel-hugging-face", "semantic-kernel-mistralai", "semantic-kernel-ollama", "semantic-kernel-onnx", "semantic-kernel-anthropic", "semantic-kernel-pandas", "semantic-kernel-aws", "semantic-kernel-dapr", "http-tool", "semantic-kernel-all", "rich", "mcp", "canvas", "duckduckgo-search"]
+provides-extras = ["anthropic", "langchain", "azure", "docker", "ollama", "openai", "file-surfer", "llama-cpp", "graphrag", "chromadb", "mem0", "mem0-local", "web-surfer", "magentic-one", "video-surfer", "diskcache", "redis", "grpc", "jupyter-executor", "docker-jupyter-executor", "task-centric-memory", "semantic-kernel-core", "gemini", "semantic-kernel-google", "semantic-kernel-hugging-face", "semantic-kernel-mistralai", "semantic-kernel-ollama", "semantic-kernel-onnx", "semantic-kernel-anthropic", "semantic-kernel-pandas", "semantic-kernel-aws", "semantic-kernel-dapr", "http-tool", "semantic-kernel-all", "rich", "mcp", "canvas"]
[package.metadata.requires-dev]
dev = [