Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): enable strict typing for Firecrawl extractor tests (#43599)

Eddy ZHANGa212b1b0b1c2
diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
index cce929d3780..fb558002d47 100644
--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
@@ -19,16 +19,16 @@ def _response(status_code: int, json_data: Mapping[str, Any] | None = None, text
response = MagicMock()
response.status_code = status_code
response.text = text
- response.json.return_value = json_data if json_data is not None else {}
+ response.json.return_value = json_data if json_data is not None else dict[str, object]()
return response
class TestFirecrawlApp:
- def test_init_requires_api_key_for_default_base_url(self):
+ def test_init_requires_api_key_for_default_base_url(self) -> None:
with pytest.raises(ValueError, match="No API key provided"):
FirecrawlApp(api_key=None, base_url="https://api.firecrawl.dev")
- def test_prepare_headers_and_build_url(self):
+ def test_prepare_headers_and_build_url(self) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev/")
assert app._prepare_headers() == {
@@ -37,7 +37,7 @@ class TestFirecrawlApp:
}
assert app._build_url("/v2/crawl") == "https://custom.firecrawl.dev/v2/crawl"
- def test_requests_use_bounded_timeout(self, mocker: MockerFixture):
+ def test_requests_use_bounded_timeout(self, mocker: MockerFixture) -> None:
"""Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))
@@ -49,7 +49,7 @@ class TestFirecrawlApp:
assert mock_post.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT
assert mock_get.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT
- def test_scrape_url_success(self, mocker: MockerFixture):
+ def test_scrape_url_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch(
"httpx.post",
@@ -77,7 +77,7 @@ class TestFirecrawlApp:
"markdown": "body",
}
- def test_scrape_url_handles_known_error_status(self, mocker: MockerFixture):
+ def test_scrape_url_handles_known_error_status(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("boom"))
mocker.patch("httpx.post", return_value=_response(429, {"error": "limit"}))
@@ -87,20 +87,20 @@ class TestFirecrawlApp:
mock_handle.assert_called_once()
- def test_scrape_url_unknown_status_raises(self, mocker: MockerFixture):
+ def test_scrape_url_unknown_status_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(404, text="Not Found"))
with pytest.raises(Exception, match="Failed to scrape URL. Status code: 404"):
app.scrape_url("https://example.com")
- def test_crawl_url_success(self, mocker: MockerFixture):
+ def test_crawl_url_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))
assert app.crawl_url("https://example.com") == "job-1"
- def test_crawl_url_non_200_uses_error_handler(self, mocker: MockerFixture):
+ def test_crawl_url_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl failed"))
mocker.patch("httpx.post", return_value=_response(500, {"error": "server"}))
@@ -110,13 +110,13 @@ class TestFirecrawlApp:
mock_handle.assert_called_once()
- def test_map_success(self, mocker: MockerFixture):
+ def test_map_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(200, {"success": True, "links": ["a", "b"]}))
assert app.map("https://example.com") == {"success": True, "links": ["a", "b"]}
- def test_map_known_error(self, mocker: MockerFixture):
+ def test_map_known_error(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("map error"))
mocker.patch("httpx.post", return_value=_response(409, {"error": "conflict"}))
@@ -125,14 +125,14 @@ class TestFirecrawlApp:
app.map("https://example.com")
mock_handle.assert_called_once()
- def test_map_unknown_error_raises(self, mocker: MockerFixture):
+ def test_map_unknown_error_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(418, text="teapot"))
with pytest.raises(Exception, match="Failed to start map job. Status code: 418"):
app.map("https://example.com")
- def test_check_crawl_status_completed_with_data(self, mocker: MockerFixture):
+ def test_check_crawl_status_completed_with_data(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
payload = {
"status": "completed",
@@ -171,23 +171,23 @@ class TestFirecrawlApp:
assert len(save_calls) == 1
assert save_calls[0][0] == "website_files/job-42.txt"
- def test_check_crawl_status_completed_with_zero_total_raises(self, mocker: MockerFixture):
+ def test_check_crawl_status_completed_with_zero_total_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": 0, "data": []}))
with pytest.raises(Exception, match="No page found"):
app.check_crawl_status("job-1")
- def test_check_crawl_status_completed_with_null_total_raises(self, mocker: MockerFixture):
+ def test_check_crawl_status_completed_with_null_total_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": None, "data": []}))
with pytest.raises(Exception, match="No page found"):
app.check_crawl_status("job-1")
- def test_check_crawl_status_non_completed(self, mocker: MockerFixture):
+ def test_check_crawl_status_non_completed(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- payload = {"status": "processing", "total": 5, "completed": 1, "data": []}
+ payload = {"status": "processing", "total": 5, "completed": 1, "data": list[object]()}
mocker.patch("httpx.get", return_value=_response(200, payload))
assert app.check_crawl_status("job-1") == {
@@ -197,7 +197,7 @@ class TestFirecrawlApp:
"data": [],
}
- def test_check_crawl_status_non_200_uses_error_handler(self, mocker: MockerFixture):
+ def test_check_crawl_status_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl error"))
mocker.patch("httpx.get", return_value=_response(500, {"error": "server"}))
@@ -206,7 +206,7 @@ class TestFirecrawlApp:
app.check_crawl_status("job-1")
mock_handle.assert_called_once()
- def test_check_crawl_status_save_failure_raises(self, mocker: MockerFixture):
+ def test_check_crawl_status_save_failure_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
payload = {
"status": "completed",
@@ -224,7 +224,7 @@ class TestFirecrawlApp:
with pytest.raises(Exception, match="Error saving crawl data"):
app.check_crawl_status("job-err")
- def test_check_crawl_status_follows_pagination(self, mocker: MockerFixture):
+ def test_check_crawl_status_follows_pagination(self, mocker: MockerFixture) -> None:
"""When status is completed and next is present, follow pagination to collect all pages."""
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
page1 = {
@@ -259,7 +259,7 @@ class TestFirecrawlApp:
assert len(result["data"]) == 3
assert [d["title"] for d in result["data"]] == ["p1", "p2", "p3"]
- def test_check_crawl_status_pagination_error_raises(self, mocker: MockerFixture):
+ def test_check_crawl_status_pagination_error_raises(self, mocker: MockerFixture) -> None:
"""An error while fetching a paginated page raises an exception; no partial data is returned."""
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
page1 = {
@@ -274,7 +274,7 @@ class TestFirecrawlApp:
with pytest.raises(Exception, match="fetch next crawl page"):
app.check_crawl_status("job-99")
- def test_check_crawl_status_pagination_capped_at_total(self, mocker: MockerFixture):
+ def test_check_crawl_status_pagination_capped_at_total(self, mocker: MockerFixture) -> None:
"""Pagination stops once pages_processed reaches total, even if next is present."""
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
# total=1: only the first page should be processed; next must not be followed
@@ -295,7 +295,7 @@ class TestFirecrawlApp:
assert len(result["data"]) == 1
mock_get.assert_called_once() # initial fetch only; next URL is not followed due to cap
- def test_extract_common_fields_and_status_formatter(self):
+ def test_extract_common_fields_and_status_formatter(self) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
fields = app._extract_common_fields(
@@ -306,7 +306,7 @@ class TestFirecrawlApp:
status = app._format_crawl_status_response("completed", {"total": 1, "completed": 1}, [fields])
assert status == {"status": "completed", "total": 1, "current": 1, "data": [fields]}
- def test_post_and_get_request_retry_logic(self, mocker: MockerFixture):
+ def test_post_and_get_request_retry_logic(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
sleep_mock = mocker.patch.object(firecrawl_module.time, "sleep")
@@ -324,7 +324,7 @@ class TestFirecrawlApp:
assert sleep_mock.call_count == 2
- def test_post_and_get_request_return_last_502(self, mocker: MockerFixture):
+ def test_post_and_get_request_return_last_502(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
sleep_mock = mocker.patch.object(firecrawl_module.time, "sleep")
@@ -338,7 +338,7 @@ class TestFirecrawlApp:
assert sleep_mock.call_count == 4
- def test_handle_error_with_json_and_plain_text(self):
+ def test_handle_error_with_json_and_plain_text(self) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
json_error = _response(400, {"message": "bad request"})
@@ -353,18 +353,18 @@ class TestFirecrawlApp:
with pytest.raises(Exception, match="plain error"):
app._handle_error(non_json, "run task")
- def test_search_success(self, mocker: MockerFixture):
+ def test_search_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(200, {"success": True, "data": [{"url": "x"}]}))
assert app.search("python")["success"] is True
- def test_search_warning_failure(self, mocker: MockerFixture):
+ def test_search_warning_failure(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(200, {"success": False, "warning": "bad search"}))
with pytest.raises(Exception, match="bad search"):
app.search("python")
- def test_search_known_http_error(self, mocker: MockerFixture):
+ def test_search_known_http_error(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("search error"))
mocker.patch("httpx.post", return_value=_response(408, {"error": "timeout"}))
@@ -372,7 +372,7 @@ class TestFirecrawlApp:
app.search("python")
mock_handle.assert_called_once()
- def test_search_unknown_http_error(self, mocker: MockerFixture):
+ def test_search_unknown_http_error(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch("httpx.post", return_value=_response(418, text="teapot"))
with pytest.raises(Exception, match="Failed to perform search. Status code: 418"):
@@ -380,7 +380,7 @@ class TestFirecrawlApp:
class TestFirecrawlWebExtractor:
- def test_extract_crawl_mode_returns_document(self, mocker: MockerFixture):
+ def test_extract_crawl_mode_returns_document(self, mocker: MockerFixture) -> None:
mocker.patch(
"core.rag.extractor.firecrawl.firecrawl_web_extractor.WebsiteService.get_crawl_url_data",
return_value={
@@ -404,7 +404,7 @@ class TestFirecrawlWebExtractor:
assert docs[0].page_content == "crawl content"
assert docs[0].metadata["source_url"] == "https://example.com"
- def test_extract_crawl_mode_with_missing_data_returns_empty(self, mocker: MockerFixture):
+ def test_extract_crawl_mode_with_missing_data_returns_empty(self, mocker: MockerFixture) -> None:
mocker.patch(
"core.rag.extractor.firecrawl.firecrawl_web_extractor.WebsiteService.get_crawl_url_data",
return_value=None,
@@ -419,7 +419,7 @@ class TestFirecrawlWebExtractor:
)
assert extractor.extract() == []
- def test_extract_scrape_mode_returns_document(self, mocker: MockerFixture):
+ def test_extract_scrape_mode_returns_document(self, mocker: MockerFixture) -> None:
mock_scrape = mocker.patch(
"core.rag.extractor.firecrawl.firecrawl_web_extractor.WebsiteService.get_scrape_url_data",
return_value={
@@ -444,7 +444,7 @@ class TestFirecrawlWebExtractor:
assert docs[0].page_content == "scrape content"
mock_scrape.assert_called_once_with("firecrawl", "https://example.com", "tenant-1", False)
- def test_extract_unknown_mode_returns_empty(self):
+ def test_extract_unknown_mode_returns_empty(self) -> None:
extractor = FirecrawlWebExtractor(
"https://example.com",
"job-1",
diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml
index e102df6b2a2..c6aa4fff8e0 100644
--- a/api/tests/unit_tests/pyrefly.toml
+++ b/api/tests/unit_tests/pyrefly.toml
@@ -448,7 +448,6 @@ project-excludes = [
"core/rag/embedding/test_embedding_base.py",
"core/rag/embedding/test_embedding_service.py",
"core/rag/extractor/blob/test_blob.py",
- "core/rag/extractor/firecrawl/test_firecrawl.py",
"core/rag/extractor/test_csv_extractor.py",
"core/rag/extractor/test_excel_extractor.py",
"core/rag/extractor/test_extract_processor.py",