Skip to content

Building AI agents, atomically

feat(WebpageScraperTool): Add separate error field to output schema #105, added tests for webpage_scraper

jon fox42801a4434c5
diff --git a/atomic-examples/deep-research/deep_research/tools/webpage_scraper.py b/atomic-examples/deep-research/deep_research/tools/webpage_scraper.py
index f9bafa2..cdbc68d 100644
--- a/atomic-examples/deep-research/deep_research/tools/webpage_scraper.py
+++ b/atomic-examples/deep-research/deep_research/tools/webpage_scraper.py
@@ -48,6 +48,7 @@ class WebpageScraperToolOutputSchema(BaseIOSchema):
content: str = Field(..., description="The scraped content in markdown format.")
metadata: WebpageMetadata = Field(..., description="Metadata about the scraped webpage.")
+ error: Optional[str] = Field(None, description="Error message if the scraping failed.")
#################
@@ -214,38 +215,47 @@ class WebpageScraperTool(BaseTool):
Returns:
WebpageScraperToolOutputSchema: The output containing the markdown content and metadata.
"""
- # Fetch webpage content
- html_content = self._fetch_webpage(str(params.url))
- # Parse HTML with BeautifulSoup
- soup = BeautifulSoup(html_content, "html.parser")
+ try:
+ # Fetch webpage content
+ html_content = self._fetch_webpage(str(params.url))
- # Extract main content using custom extraction
- main_content = self._extract_main_content(soup)
+ # Parse HTML with BeautifulSoup
+ soup = BeautifulSoup(html_content, "html.parser")
- # Convert to markdown
- markdown_options = {
- "strip": ["script", "style"],
- "heading_style": "ATX",
- "bullets": "-",
- "wrap": True,
- }
+ # Extract main content using custom extraction
+ main_content = self._extract_main_content(soup)
- if not params.include_links:
- markdown_options["strip"].append("a")
+ # Convert to markdown
+ markdown_options = {
+ "strip": ["script", "style"],
+ "heading_style": "ATX",
+ "bullets": "-",
+ "wrap": True,
+ }
- markdown_content = markdownify(main_content, **markdown_options)
+ if not params.include_links:
+ markdown_options["strip"].append("a")
- # Clean up the markdown
- markdown_content = self._clean_markdown(markdown_content)
+ markdown_content = markdownify(main_content, **markdown_options)
- # Extract metadata
- metadata = self._extract_metadata(soup, Document(html_content), str(params.url))
+ # Clean up the markdown
+ markdown_content = self._clean_markdown(markdown_content)
- return WebpageScraperToolOutputSchema(
- content=markdown_content,
- metadata=metadata,
- )
+ # Extract metadata
+ metadata = self._extract_metadata(soup, Document(html_content), str(params.url))
+
+ return WebpageScraperToolOutputSchema(
+ content=markdown_content,
+ metadata=metadata,
+ )
+ except Exception as e:
+ # Create empty/minimal metadata with at least the domain
+ domain = urlparse(str(params.url)).netloc
+ minimal_metadata = WebpageMetadata(title="Error retrieving page", domain=domain)
+
+ # Return with error message in the error field
+ return WebpageScraperToolOutputSchema(content="", metadata=minimal_metadata, error=str(e))
#################
@@ -266,11 +276,16 @@ if __name__ == "__main__":
)
)
- console.print(Panel.fit("Metadata", style="bold green"))
- console.print(result.metadata.model_dump_json(indent=2))
+ # Check if there was an error during scraping, otherwise print the results
+ if result.error:
+ console.print(Panel.fit("Error", style="bold red"))
+ console.print(f"[red]{result.error}[/red]")
+ else:
+ console.print(Panel.fit("Metadata", style="bold green"))
+ console.print(result.metadata.model_dump_json(indent=2))
- console.print(Panel.fit("Content Preview (first 500 chars)", style="bold green"))
- console.print(result.content)
+ console.print(Panel.fit("Content Preview (first 500 chars)", style="bold green"))
+ console.print(result.content[:500] + ("..." if len(result.content) > 500 else ""))
except Exception as e:
console.print(f"[red]Error:[/red] {str(e)}")
diff --git a/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py b/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py
index e69de29..6834b86 100644
--- a/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py
+++ b/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py
@@ -0,0 +1,191 @@
+import os
+import sys
+import pytest
+from unittest.mock import patch, MagicMock
+from urllib.parse import urlparse
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from tool.webpage_scraper import ( # noqa: E402
+ WebpageScraperTool,
+ WebpageScraperToolInputSchema,
+ WebpageScraperToolOutputSchema,
+ WebpageScraperToolConfig,
+ WebpageMetadata,
+)
+
+
+def mock_requests_get():
+ with patch("tool.webpage_scraper.requests.get") as mock_get:
+ # Create mock response
+ mock_response = MagicMock()
+ mock_response.text = """
+ <html>
+ <head>
+ <title>Test Page</title>
+ <meta name="author" content="Test Author">
+ <meta name="description" content="Test Description">
+ <meta property="og:site_name" content="Test Site">
+ </head>
+ <body>
+ <main>
+ <h1>Test Heading</h1>
+ <p>Test paragraph with <a href="https://example.com">link</a>.</p>
+ </main>
+ </body>
+ </html>
+ """
+ mock_response.content = mock_response.text.encode("utf-8")
+ mock_response.status_code = 200
+ mock_response.raise_for_status = MagicMock()
+
+ # Configure the mock
+ mock_get.return_value = mock_response
+ yield mock_get
+
+
+def test_webpage_scraper_tool_basic(mock_requests_get):
+ # Initialize the tool
+ scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
+ input_schema = WebpageScraperToolInputSchema(url="https://example.com")
+
+ # Run the tool
+ result = scraper_tool.run(input_schema)
+
+ # Assertions
+ assert isinstance(result, WebpageScraperToolOutputSchema)
+ assert "Test Heading" in result.content
+ assert "Test paragraph" in result.content
+ assert "link" in result.content
+ assert result.metadata.title == "Test Page"
+ assert result.metadata.author == "Test Author"
+ assert result.metadata.description == "Test Description"
+ assert result.metadata.site_name == "Test Site"
+ assert result.metadata.domain == "example.com"
+ assert result.error is None
+
+
+def test_webpage_scraper_tool_without_links(mock_requests_get):
+ # Initialize the tool
+ scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
+ input_schema = WebpageScraperToolInputSchema(url="https://example.com", include_links=False)
+
+ # Run the tool
+ result = scraper_tool.run(input_schema)
+
+ # Assertions
+ assert isinstance(result, WebpageScraperToolOutputSchema)
+ assert "Test paragraph with link" in result.content
+ assert "https://example.com" not in result.content # Link URL should not be included
+
+
+def test_webpage_scraper_tool_http_error(mock_requests_get):
+ # Configure mock to raise an exception
+ mock_requests_get.return_value.raise_for_status.side_effect = Exception("404 Client Error")
+
+ # Initialize the tool
+ scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
+ input_schema = WebpageScraperToolInputSchema(url="https://example.com/not-found")
+
+ # Run the tool
+ result = scraper_tool.run(input_schema)
+
+ # Assertions
+ assert isinstance(result, WebpageScraperToolOutputSchema)
+ assert result.content == "" # Content should be empty
+ assert result.metadata.title == "Error retrieving page"
+ assert result.metadata.domain == "example.com"
+ assert "404 Client Error" in result.error
+
+
+def test_webpage_scraper_tool_content_too_large(mock_requests_get):
+ # Configure mock content to exceed max length
+ max_length = 1_000_000
+ mock_requests_get.return_value.content = b"a" * (max_length + 1)
+
+ # Initialize the tool
+ scraper_tool = WebpageScraperTool(WebpageScraperToolConfig(max_content_length=max_length))
+ input_schema = WebpageScraperToolInputSchema(url="https://example.com/large-page")
+
+ # Run the tool
+ result = scraper_tool.run(input_schema)
+
+ # Assertions
+ assert isinstance(result, WebpageScraperToolOutputSchema)
+ assert "exceeds maximum" in result.error
+
+
+def test_webpage_scraper_tool_extract_metadata():
+ # Initialize the tool
+ scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
+
+ # Create a minimal soup object with metadata
+ soup = MagicMock()
+
+ # Create individual mock tags with get methods
+ author_tag = MagicMock()
+ author_tag.get.return_value = "Author Name"
+
+ description_tag = MagicMock()
+ description_tag.get.return_value = "Page Description"
+
+ site_name_tag = MagicMock()
+ site_name_tag.get.return_value = "Site Name"
+
+ # Configure find method to return the right mock based on arguments
+ def mock_find(tag, attrs=None):
+ if tag == "meta" and attrs == {"name": "author"}:
+ return author_tag
+ elif tag == "meta" and attrs == {"name": "description"}:
+ return description_tag
+ elif tag == "meta" and attrs == {"property": "og:site_name"}:
+ return site_name_tag
+ return None
+
+ soup.find.side_effect = mock_find
+
+ doc = MagicMock()
+ doc.title.return_value = "Page Title"
+
+ # Call the method directly
+ metadata = scraper_tool._extract_metadata(soup, doc, "https://example.org/page")
+
+ # Assertions
+ assert metadata.title == "Page Title"
+ assert metadata.author == "Author Name"
+ assert metadata.description == "Page Description"
+ assert metadata.site_name == "Site Name"
+ assert metadata.domain == "example.org"
+
+
+def test_webpage_scraper_tool_clean_markdown():
+ # Initialize the tool
+ scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
+
+ # Input markdown with excess whitespace
+ dirty_markdown = """
+ # Title
+
+
+
+ Paragraph with trailing spaces
+
+
+ * List item 1
+ * List item 2
+
+
+ """
+
+ # Clean the markdown
+ cleaned = scraper_tool._clean_markdown(dirty_markdown)
+
+ # Assertions
+ assert cleaned.count("\n\n\n") == 0 # No triple newlines
+ assert "spaces \n" not in cleaned # No trailing spaces
+ assert cleaned.endswith("\n") # Ends with newline
+
+
+if __name__ == "__main__":
+ pytest.main([__file__])
\ No newline at end of file
diff --git a/atomic-forge/tools/webpage_scraper/tool/webpage_scraper.py b/atomic-forge/tools/webpage_scraper/tool/webpage_scraper.py
index 5513c7b..4afd01d 100644
--- a/atomic-forge/tools/webpage_scraper/tool/webpage_scraper.py
+++ b/atomic-forge/tools/webpage_scraper/tool/webpage_scraper.py
@@ -48,6 +48,7 @@ class WebpageScraperToolOutputSchema(BaseIOSchema):
content: str = Field(..., description="The scraped content in markdown format.")
metadata: WebpageMetadata = Field(..., description="Metadata about the scraped webpage.")
+ error: Optional[str] = Field(None, description="Error message if the scraping failed.")
#################
@@ -214,38 +215,46 @@ class WebpageScraperTool(BaseTool):
Returns:
WebpageScraperToolOutputSchema: The output containing the markdown content and metadata.
"""
- # Fetch webpage content
- html_content = self._fetch_webpage(str(params.url))
+ try:
+ # Fetch webpage content
+ html_content = self._fetch_webpage(str(params.url))
- # Parse HTML with BeautifulSoup
- soup = BeautifulSoup(html_content, "html.parser")
+ # Parse HTML with BeautifulSoup
+ soup = BeautifulSoup(html_content, "html.parser")
- # Extract main content using custom extraction
- main_content = self._extract_main_content(soup)
+ # Extract main content using custom extraction
+ main_content = self._extract_main_content(soup)
- # Convert to markdown
- markdown_options = {
- "strip": ["script", "style"],
- "heading_style": "ATX",
- "bullets": "-",
- "wrap": True,
- }
+ # Convert to markdown
+ markdown_options = {
+ "strip": ["script", "style"],
+ "heading_style": "ATX",
+ "bullets": "-",
+ "wrap": True,
+ }
- if not params.include_links:
- markdown_options["strip"].append("a")
+ if not params.include_links:
+ markdown_options["strip"].append("a")
- markdown_content = markdownify(main_content, **markdown_options)
+ markdown_content = markdownify(main_content, **markdown_options)
- # Clean up the markdown
- markdown_content = self._clean_markdown(markdown_content)
+ # Clean up the markdown
+ markdown_content = self._clean_markdown(markdown_content)
- # Extract metadata
- metadata = self._extract_metadata(soup, Document(html_content), str(params.url))
+ # Extract metadata
+ metadata = self._extract_metadata(soup, Document(html_content), str(params.url))
- return WebpageScraperToolOutputSchema(
- content=markdown_content,
- metadata=metadata,
- )
+ return WebpageScraperToolOutputSchema(
+ content=markdown_content,
+ metadata=metadata,
+ )
+ except Exception as e:
+ # Create empty/minimal metadata with at least the domain
+ domain = urlparse(str(params.url)).netloc
+ minimal_metadata = WebpageMetadata(title="Error retrieving page", domain=domain)
+
+ # Return with error message in the error field
+ return WebpageScraperToolOutputSchema(content="", metadata=minimal_metadata, error=str(e))
#################
@@ -266,11 +275,16 @@ if __name__ == "__main__":
)
)
- console.print(Panel.fit("Metadata", style="bold green"))
- console.print(result.metadata.model_dump_json(indent=2))
+ # Check if there was an error during scraping
+ if result.error:
+ console.print(Panel.fit("Error", style="bold red"))
+ console.print(f"[red]{result.error}[/red]")
+ else:
+ console.print(Panel.fit("Metadata", style="bold green"))
+ console.print(result.metadata.model_dump_json(indent=2))
- console.print(Panel.fit("Content Preview (first 500 chars)", style="bold green"))
- console.print(result.content)
+ console.print(Panel.fit("Content Preview (first 500 chars)", style="bold green"))
+ console.print(result.content[:500] + ("..." if len(result.content) > 500 else ""))
except Exception as e:
console.print(f"[red]Error:[/red] {str(e)}")