Skip to content

Building AI agents, atomically

11.9 KB
Raw Blame
import unittest
from unittest.mock import patch, Mock
import requests
import sys
import os
from bs4 import BeautifulSoup
from readability import Document

# Add the parent directory to sys.path to find the tool module
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

from tool.webpage_scraper import (  # noqa: E402
    WebpageScraperTool,
    WebpageScraperToolConfig,
    WebpageScraperToolInputSchema,
    WebpageScraperToolOutputSchema,
    WebpageMetadata,
)


class TestWebpageScraperTool(unittest.TestCase):
    """Test cases for the WebpageScraperTool."""

    def setUp(self):
        """Set up test fixtures."""
        self.config = WebpageScraperToolConfig(
            user_agent="Test User Agent",
            timeout=10,
            max_content_length=500000,
        )
        self.tool = WebpageScraperTool(config=self.config)

        # Sample HTML content for testing
        self.sample_html = """<!DOCTYPE html>
        <html>
        <head>
            <title>Test Page</title>
            <meta name="author" content="Test Author">
            <meta name="description" content="Test Description">
            <meta property="og:site_name" content="Test Site">
        </head>
        <body>
            <header>
                <nav>Navigation</nav>
            </header>
            <main>
                <h1>Main Content</h1>
                <p>This is a test paragraph with a <a href="https://example.com">link</a>.</p>
                <ul>
                    <li>Item 1</li>
                    <li>Item 2</li>
                </ul>
            </main>
            <footer>Footer content</footer>
            <script>console.log('test');</script>
        </body>
        </html>
        """
        self.mock_url = "https://example.com/test"

    @patch("requests.get")
    def test_fetch_webpage(self, mock_get):
        """Test fetching a webpage."""
        # Configure the mock
        mock_response = Mock()
        mock_response.text = self.sample_html
        mock_response.content = self.sample_html.encode("utf-8")
        mock_get.return_value = mock_response

        # Test the method
        result = self.tool._fetch_webpage(self.mock_url)

        # Assertions
        mock_get.assert_called_once()
        self.assertEqual(result, self.sample_html)

        # Check headers
        call_args = mock_get.call_args
        headers = call_args[1]["headers"]
        self.assertEqual(headers["User-Agent"], "Test User Agent")

    @patch("requests.get")
    def test_fetch_webpage_content_too_large(self, mock_get):
        """Test handling of content that exceeds max length."""
        # Configure the mock
        mock_response = Mock()
        mock_response.text = "X" * (self.config.max_content_length + 1)
        mock_response.content = ("X" * (self.config.max_content_length + 1)).encode("utf-8")
        mock_get.return_value = mock_response

        # Test the method
        with self.assertRaises(ValueError) as context:
            self.tool._fetch_webpage(self.mock_url)

        self.assertIn("Content length exceeds maximum", str(context.exception))

    def test_extract_metadata(self):
        """Test extracting metadata from a webpage."""
        # Create BeautifulSoup and Document objects
        soup = BeautifulSoup(self.sample_html, "html.parser")
        doc = Document(self.sample_html)

        # Test the method
        metadata = self.tool._extract_metadata(soup, doc, self.mock_url)

        # Assertions
        self.assertIsInstance(metadata, WebpageMetadata)
        self.assertEqual(metadata.title, "Test Page")
        self.assertEqual(metadata.author, "Test Author")
        self.assertEqual(metadata.description, "Test Description")
        self.assertEqual(metadata.site_name, "Test Site")
        self.assertEqual(metadata.domain, "example.com")

    def test_clean_markdown(self):
        """Test cleaning up markdown content."""
        markdown = """# Title

        

        Some content
          with weird spacing
        
        
        
        More content
        """

        expected = "# Title\n\nSome content\n  with weird spacing\n\nMore content\n"

        # Test the method
        result = self.tool._clean_markdown(markdown)

        # Assertions
        self.assertEqual(result, expected)

    def test_clean_markdown_triple_newlines(self):
        """Test cleaning up markdown content with triple newlines."""
        # Make sure we're not hitting the special case by using different content
        markdown = "Test\n\n\nMultiple\n\n\nNewlines"

        # Test the method
        result = self.tool._clean_markdown(markdown)

        # Verify that triple newlines are replaced with double newlines
        self.assertEqual(result, "Test\n\nMultiple\n\nNewlines\n")
        self.assertNotIn("\n\n\n", result)

        # Test another case to ensure the function works with other content patterns
        complex_markdown = """Regular content

          

        with triple newlines above"""

        complex_result = self.tool._clean_markdown(complex_markdown)
        self.assertNotIn("\n\n\n", complex_result)

    def test_extract_main_content(self):
        """Test extracting main content from HTML."""
        soup = BeautifulSoup(self.sample_html, "html.parser")

        # Test the method
        result = self.tool._extract_main_content(soup)

        # Assertions
        # The result should contain the main content but not header/footer/scripts
        self.assertIn("Main Content", result)
        self.assertIn("test paragraph", result)
        self.assertIn("Item 1", result)
        self.assertNotIn("Navigation", result)
        self.assertNotIn("Footer content", result)
        self.assertNotIn("console.log", result)

    def test_extract_main_content_fallbacks(self):
        """Test extracting main content with various fallback mechanisms."""
        # Test with HTML that has no main tag but has content/main in id
        html_with_id = """
        <html><body>
            <div id="content"><p>Content in div with ID</p></div>
        </body></html>
        """
        soup = BeautifulSoup(html_with_id, "html.parser")
        result = self.tool._extract_main_content(soup)
        self.assertIn("Content in div with ID", result)

        # Test with HTML that has no main or ID but has content in class
        html_with_class = """
        <html><body>
            <div class="main-content"><p>Content in div with class</p></div>
        </body></html>
        """
        soup = BeautifulSoup(html_with_class, "html.parser")
        result = self.tool._extract_main_content(soup)
        self.assertIn("Content in div with class", result)

        # Test with HTML that has article
        html_with_article = """
        <html><body>
            <article><p>Content in article</p></article>
        </body></html>
        """
        soup = BeautifulSoup(html_with_article, "html.parser")
        result = self.tool._extract_main_content(soup)
        self.assertIn("Content in article", result)

        # Test with HTML that has none of the above, should fallback to body
        html_with_body_only = """
        <html><body>
            <p>Content in body</p>
        </body></html>
        """
        soup = BeautifulSoup(html_with_body_only, "html.parser")
        result = self.tool._extract_main_content(soup)
        self.assertIn("Content in body", result)

        # Test with completely empty HTML
        empty_html = "<html></html>"
        soup = BeautifulSoup(empty_html, "html.parser")
        result = self.tool._extract_main_content(soup)
        self.assertEqual(result, str(soup))

    @patch.object(WebpageScraperTool, "_fetch_webpage")
    def test_run_with_links(self, mock_fetch):
        """Test running the tool with links included."""
        # Configure the mock
        mock_fetch.return_value = self.sample_html

        # Create input parameters
        params = WebpageScraperToolInputSchema(
            url=self.mock_url,
            include_links=True,
        )

        # Test the method
        result = self.tool.run(params)

        # Assertions
        self.assertIsInstance(result, WebpageScraperToolOutputSchema)
        self.assertIn("Main Content", result.content)
        self.assertIn("link", result.content)
        self.assertIn("https://example.com", result.content)
        self.assertEqual(result.metadata.title, "Test Page")

    @patch.object(WebpageScraperTool, "_fetch_webpage")
    def test_run_without_links(self, mock_fetch):
        """Test running the tool with links excluded."""
        # Configure the mock
        mock_fetch.return_value = self.sample_html

        # Create input parameters
        params = WebpageScraperToolInputSchema(
            url=self.mock_url,
            include_links=False,
        )

        # Test the method
        result = self.tool.run(params)

        # Assertions
        self.assertIsInstance(result, WebpageScraperToolOutputSchema)
        self.assertIn("Main Content", result.content)
        self.assertIn("link", result.content)  # The text "link" should still be there
        self.assertNotIn("https://example.com", result.content)  # But the URL should be gone
        self.assertEqual(result.metadata.title, "Test Page")

    @patch("requests.get")
    def test_http_errors(self, mock_get):
        """Test handling of HTTP errors."""
        # Configure the mock to raise an exception
        mock_get.side_effect = requests.exceptions.HTTPError("404 Client Error")

        # Create input parameters
        params = WebpageScraperToolInputSchema(
            url=self.mock_url,
            include_links=True,
        )

        # Test the method
        result = self.tool.run(params)

        # Check that the error is captured in the result
        self.assertIsNotNone(result.error)
        self.assertIn("404 Client Error", result.error)

    @patch("requests.get")
    def test_connection_timeout(self, mock_get):
        """Test handling of connection timeouts."""
        # Configure the mock to raise a timeout exception
        mock_get.side_effect = requests.exceptions.Timeout("Connection timed out")

        # Create input parameters
        params = WebpageScraperToolInputSchema(
            url=self.mock_url,
            include_links=True,
        )

        # Test the method
        result = self.tool.run(params)

        # Check that the error is captured in the result
        self.assertIsNotNone(result.error)
        self.assertIn("Connection timed out", result.error)

    def test_tool_config(self):
        """Test tool configuration."""
        # Test default config
        default_config = WebpageScraperToolConfig()
        self.assertEqual(default_config.timeout, 30)
        self.assertIn("Mozilla", default_config.user_agent)
        self.assertEqual(default_config.max_content_length, 1_000_000)

        # Test custom config
        custom_config = WebpageScraperToolConfig(
            user_agent="Custom Agent",
            timeout=45,
            max_content_length=2_000_000,
        )
        self.assertEqual(custom_config.timeout, 45)
        self.assertEqual(custom_config.user_agent, "Custom Agent")
        self.assertEqual(custom_config.max_content_length, 2_000_000)

    def test_clean_markdown_empty_content(self):
        """Test cleaning up empty markdown content."""
        # Test with empty string
        result = self.tool._clean_markdown("")
        self.assertEqual(result, "\n")

        # Test with whitespace only
        result = self.tool._clean_markdown("   \n\n   \n   ")
        self.assertEqual(result, "\n")

    def test_clean_markdown_with_triple_newlines(self):
        """Test cleaning up markdown content with forced triple newlines."""
        # Create input with triple newlines specifically for regex coverage
        markdown = "Line1\n\n\nLine2"

        # Call the method directly to ensure coverage
        result = self.tool._clean_markdown(markdown)

        # Verify it collapsed triple newlines to double
        self.assertEqual(result, "Line1\n\nLine2\n")


if __name__ == "__main__":
    unittest.main()