Skip to content

Building AI agents, atomically

fixing flake and black formatting on file

jon fox9e809700d9e8
diff --git a/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py b/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py
index 6834b86..bfc829a 100644
--- a/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py
+++ b/atomic-forge/tools/webpage_scraper/tests/test_webpage_scraper.py
@@ -2,7 +2,6 @@ import os
import sys
import pytest
from unittest.mock import patch, MagicMock
-from urllib.parse import urlparse
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
@@ -11,7 +10,6 @@ from tool.webpage_scraper import ( # noqa: E402
WebpageScraperToolInputSchema,
WebpageScraperToolOutputSchema,
WebpageScraperToolConfig,
- WebpageMetadata,
)
@@ -39,7 +37,7 @@ def mock_requests_get():
mock_response.content = mock_response.text.encode("utf-8")
mock_response.status_code = 200
mock_response.raise_for_status = MagicMock()
-
+
# Configure the mock
mock_get.return_value = mock_response
yield mock_get
@@ -49,10 +47,10 @@ def test_webpage_scraper_tool_basic(mock_requests_get):
# Initialize the tool
scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
input_schema = WebpageScraperToolInputSchema(url="https://example.com")
-
+
# Run the tool
result = scraper_tool.run(input_schema)
-
+
# Assertions
assert isinstance(result, WebpageScraperToolOutputSchema)
assert "Test Heading" in result.content
@@ -70,10 +68,10 @@ def test_webpage_scraper_tool_without_links(mock_requests_get):
# Initialize the tool
scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
input_schema = WebpageScraperToolInputSchema(url="https://example.com", include_links=False)
-
+
# Run the tool
result = scraper_tool.run(input_schema)
-
+
# Assertions
assert isinstance(result, WebpageScraperToolOutputSchema)
assert "Test paragraph with link" in result.content
@@ -83,14 +81,14 @@ def test_webpage_scraper_tool_without_links(mock_requests_get):
def test_webpage_scraper_tool_http_error(mock_requests_get):
# Configure mock to raise an exception
mock_requests_get.return_value.raise_for_status.side_effect = Exception("404 Client Error")
-
+
# Initialize the tool
scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
input_schema = WebpageScraperToolInputSchema(url="https://example.com/not-found")
-
+
# Run the tool
result = scraper_tool.run(input_schema)
-
+
# Assertions
assert isinstance(result, WebpageScraperToolOutputSchema)
assert result.content == "" # Content should be empty
@@ -103,14 +101,14 @@ def test_webpage_scraper_tool_content_too_large(mock_requests_get):
# Configure mock content to exceed max length
max_length = 1_000_000
mock_requests_get.return_value.content = b"a" * (max_length + 1)
-
+
# Initialize the tool
scraper_tool = WebpageScraperTool(WebpageScraperToolConfig(max_content_length=max_length))
input_schema = WebpageScraperToolInputSchema(url="https://example.com/large-page")
-
+
# Run the tool
result = scraper_tool.run(input_schema)
-
+
# Assertions
assert isinstance(result, WebpageScraperToolOutputSchema)
assert "exceeds maximum" in result.error
@@ -119,20 +117,20 @@ def test_webpage_scraper_tool_content_too_large(mock_requests_get):
def test_webpage_scraper_tool_extract_metadata():
# Initialize the tool
scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
-
+
# Create a minimal soup object with metadata
soup = MagicMock()
-
+
# Create individual mock tags with get methods
author_tag = MagicMock()
author_tag.get.return_value = "Author Name"
-
+
description_tag = MagicMock()
description_tag.get.return_value = "Page Description"
-
+
site_name_tag = MagicMock()
site_name_tag.get.return_value = "Site Name"
-
+
# Configure find method to return the right mock based on arguments
def mock_find(tag, attrs=None):
if tag == "meta" and attrs == {"name": "author"}:
@@ -142,15 +140,15 @@ def test_webpage_scraper_tool_extract_metadata():
elif tag == "meta" and attrs == {"property": "og:site_name"}:
return site_name_tag
return None
-
+
soup.find.side_effect = mock_find
-
+
doc = MagicMock()
doc.title.return_value = "Page Title"
-
+
# Call the method directly
metadata = scraper_tool._extract_metadata(soup, doc, "https://example.org/page")
-
+
# Assertions
assert metadata.title == "Page Title"
assert metadata.author == "Author Name"
@@ -162,25 +160,24 @@ def test_webpage_scraper_tool_extract_metadata():
def test_webpage_scraper_tool_clean_markdown():
# Initialize the tool
scraper_tool = WebpageScraperTool(WebpageScraperToolConfig())
-
+
# Input markdown with excess whitespace
dirty_markdown = """
# Title
-
-
-
- Paragraph with trailing spaces
-
-
- * List item 1
+
+
+
+ Paragraph with trailing spaces
+
+ * List item 1
* List item 2
-
-
+
+
"""
-
+
# Clean the markdown
cleaned = scraper_tool._clean_markdown(dirty_markdown)
-
+
# Assertions
assert cleaned.count("\n\n\n") == 0 # No triple newlines
assert "spaces \n" not in cleaned # No trailing spaces
@@ -188,4 +185,4 @@ def test_webpage_scraper_tool_clean_markdown():
if __name__ == "__main__":
- pytest.main([__file__])
\ No newline at end of file
+ pytest.main([__file__])