Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): enable strict typing for Firecrawl extractor tests (#43599)
Eddy ZHANGa212b1b0b1c2
diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.pyindex cce929d3780..fb558002d47 100644--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py@@ -19,16 +19,16 @@ def _response(status_code: int, json_data: Mapping[str, Any] | None = None, textresponse = MagicMock()response.status_code = status_coderesponse.text = text- response.json.return_value = json_data if json_data is not None else {}+ response.json.return_value = json_data if json_data is not None else dict[str, object]()return responseclass TestFirecrawlApp:- def test_init_requires_api_key_for_default_base_url(self):+ def test_init_requires_api_key_for_default_base_url(self) -> None:with pytest.raises(ValueError, match="No API key provided"):FirecrawlApp(api_key=None, base_url="https://api.firecrawl.dev")- def test_prepare_headers_and_build_url(self):+ def test_prepare_headers_and_build_url(self) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev/")assert app._prepare_headers() == {@@ -37,7 +37,7 @@ class TestFirecrawlApp:}assert app._build_url("/v2/crawl") == "https://custom.firecrawl.dev/v2/crawl"- def test_requests_use_bounded_timeout(self, mocker: MockerFixture):+ def test_requests_use_bounded_timeout(self, mocker: MockerFixture) -> None:"""Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))@@ -49,7 +49,7 @@ class TestFirecrawlApp:assert mock_post.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUTassert mock_get.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT- def test_scrape_url_success(self, mocker: MockerFixture):+ def test_scrape_url_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post",@@ -77,7 +77,7 @@ class TestFirecrawlApp:"markdown": "body",}- def test_scrape_url_handles_known_error_status(self, mocker: MockerFixture):+ def test_scrape_url_handles_known_error_status(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("boom"))mocker.patch("httpx.post", return_value=_response(429, {"error": "limit"}))@@ -87,20 +87,20 @@ class TestFirecrawlApp:mock_handle.assert_called_once()- def test_scrape_url_unknown_status_raises(self, mocker: MockerFixture):+ def test_scrape_url_unknown_status_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(404, text="Not Found"))with pytest.raises(Exception, match="Failed to scrape URL. Status code: 404"):app.scrape_url("https://example.com")- def test_crawl_url_success(self, mocker: MockerFixture):+ def test_crawl_url_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))assert app.crawl_url("https://example.com") == "job-1"- def test_crawl_url_non_200_uses_error_handler(self, mocker: MockerFixture):+ def test_crawl_url_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl failed"))mocker.patch("httpx.post", return_value=_response(500, {"error": "server"}))@@ -110,13 +110,13 @@ class TestFirecrawlApp:mock_handle.assert_called_once()- def test_map_success(self, mocker: MockerFixture):+ def test_map_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(200, {"success": True, "links": ["a", "b"]}))assert app.map("https://example.com") == {"success": True, "links": ["a", "b"]}- def test_map_known_error(self, mocker: MockerFixture):+ def test_map_known_error(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("map error"))mocker.patch("httpx.post", return_value=_response(409, {"error": "conflict"}))@@ -125,14 +125,14 @@ class TestFirecrawlApp:app.map("https://example.com")mock_handle.assert_called_once()- def test_map_unknown_error_raises(self, mocker: MockerFixture):+ def test_map_unknown_error_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(418, text="teapot"))with pytest.raises(Exception, match="Failed to start map job. Status code: 418"):app.map("https://example.com")- def test_check_crawl_status_completed_with_data(self, mocker: MockerFixture):+ def test_check_crawl_status_completed_with_data(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")payload = {"status": "completed",@@ -171,23 +171,23 @@ class TestFirecrawlApp:assert len(save_calls) == 1assert save_calls[0][0] == "website_files/job-42.txt"- def test_check_crawl_status_completed_with_zero_total_raises(self, mocker: MockerFixture):+ def test_check_crawl_status_completed_with_zero_total_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": 0, "data": []}))with pytest.raises(Exception, match="No page found"):app.check_crawl_status("job-1")- def test_check_crawl_status_completed_with_null_total_raises(self, mocker: MockerFixture):+ def test_check_crawl_status_completed_with_null_total_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": None, "data": []}))with pytest.raises(Exception, match="No page found"):app.check_crawl_status("job-1")- def test_check_crawl_status_non_completed(self, mocker: MockerFixture):+ def test_check_crawl_status_non_completed(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- payload = {"status": "processing", "total": 5, "completed": 1, "data": []}+ payload = {"status": "processing", "total": 5, "completed": 1, "data": list[object]()}mocker.patch("httpx.get", return_value=_response(200, payload))assert app.check_crawl_status("job-1") == {@@ -197,7 +197,7 @@ class TestFirecrawlApp:"data": [],}- def test_check_crawl_status_non_200_uses_error_handler(self, mocker: MockerFixture):+ def test_check_crawl_status_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl error"))mocker.patch("httpx.get", return_value=_response(500, {"error": "server"}))@@ -206,7 +206,7 @@ class TestFirecrawlApp:app.check_crawl_status("job-1")mock_handle.assert_called_once()- def test_check_crawl_status_save_failure_raises(self, mocker: MockerFixture):+ def test_check_crawl_status_save_failure_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")payload = {"status": "completed",@@ -224,7 +224,7 @@ class TestFirecrawlApp:with pytest.raises(Exception, match="Error saving crawl data"):app.check_crawl_status("job-err")- def test_check_crawl_status_follows_pagination(self, mocker: MockerFixture):+ def test_check_crawl_status_follows_pagination(self, mocker: MockerFixture) -> None:"""When status is completed and next is present, follow pagination to collect all pages."""app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")page1 = {@@ -259,7 +259,7 @@ class TestFirecrawlApp:assert len(result["data"]) == 3assert [d["title"] for d in result["data"]] == ["p1", "p2", "p3"]- def test_check_crawl_status_pagination_error_raises(self, mocker: MockerFixture):+ def test_check_crawl_status_pagination_error_raises(self, mocker: MockerFixture) -> None:"""An error while fetching a paginated page raises an exception; no partial data is returned."""app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")page1 = {@@ -274,7 +274,7 @@ class TestFirecrawlApp:with pytest.raises(Exception, match="fetch next crawl page"):app.check_crawl_status("job-99")- def test_check_crawl_status_pagination_capped_at_total(self, mocker: MockerFixture):+ def test_check_crawl_status_pagination_capped_at_total(self, mocker: MockerFixture) -> None:"""Pagination stops once pages_processed reaches total, even if next is present."""app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")# total=1: only the first page should be processed; next must not be followed@@ -295,7 +295,7 @@ class TestFirecrawlApp:assert len(result["data"]) == 1mock_get.assert_called_once() # initial fetch only; next URL is not followed due to cap- def test_extract_common_fields_and_status_formatter(self):+ def test_extract_common_fields_and_status_formatter(self) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")fields = app._extract_common_fields(@@ -306,7 +306,7 @@ class TestFirecrawlApp:status = app._format_crawl_status_response("completed", {"total": 1, "completed": 1}, [fields])assert status == {"status": "completed", "total": 1, "current": 1, "data": [fields]}- def test_post_and_get_request_retry_logic(self, mocker: MockerFixture):+ def test_post_and_get_request_retry_logic(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")sleep_mock = mocker.patch.object(firecrawl_module.time, "sleep")@@ -324,7 +324,7 @@ class TestFirecrawlApp:assert sleep_mock.call_count == 2- def test_post_and_get_request_return_last_502(self, mocker: MockerFixture):+ def test_post_and_get_request_return_last_502(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")sleep_mock = mocker.patch.object(firecrawl_module.time, "sleep")@@ -338,7 +338,7 @@ class TestFirecrawlApp:assert sleep_mock.call_count == 4- def test_handle_error_with_json_and_plain_text(self):+ def test_handle_error_with_json_and_plain_text(self) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")json_error = _response(400, {"message": "bad request"})@@ -353,18 +353,18 @@ class TestFirecrawlApp:with pytest.raises(Exception, match="plain error"):app._handle_error(non_json, "run task")- def test_search_success(self, mocker: MockerFixture):+ def test_search_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(200, {"success": True, "data": [{"url": "x"}]}))assert app.search("python")["success"] is True- def test_search_warning_failure(self, mocker: MockerFixture):+ def test_search_warning_failure(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(200, {"success": False, "warning": "bad search"}))with pytest.raises(Exception, match="bad search"):app.search("python")- def test_search_known_http_error(self, mocker: MockerFixture):+ def test_search_known_http_error(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("search error"))mocker.patch("httpx.post", return_value=_response(408, {"error": "timeout"}))@@ -372,7 +372,7 @@ class TestFirecrawlApp:app.search("python")mock_handle.assert_called_once()- def test_search_unknown_http_error(self, mocker: MockerFixture):+ def test_search_unknown_http_error(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch("httpx.post", return_value=_response(418, text="teapot"))with pytest.raises(Exception, match="Failed to perform search. Status code: 418"):@@ -380,7 +380,7 @@ class TestFirecrawlApp:class TestFirecrawlWebExtractor:- def test_extract_crawl_mode_returns_document(self, mocker: MockerFixture):+ def test_extract_crawl_mode_returns_document(self, mocker: MockerFixture) -> None:mocker.patch("core.rag.extractor.firecrawl.firecrawl_web_extractor.WebsiteService.get_crawl_url_data",return_value={@@ -404,7 +404,7 @@ class TestFirecrawlWebExtractor:assert docs[0].page_content == "crawl content"assert docs[0].metadata["source_url"] == "https://example.com"- def test_extract_crawl_mode_with_missing_data_returns_empty(self, mocker: MockerFixture):+ def test_extract_crawl_mode_with_missing_data_returns_empty(self, mocker: MockerFixture) -> None:mocker.patch("core.rag.extractor.firecrawl.firecrawl_web_extractor.WebsiteService.get_crawl_url_data",return_value=None,@@ -419,7 +419,7 @@ class TestFirecrawlWebExtractor:)assert extractor.extract() == []- def test_extract_scrape_mode_returns_document(self, mocker: MockerFixture):+ def test_extract_scrape_mode_returns_document(self, mocker: MockerFixture) -> None:mock_scrape = mocker.patch("core.rag.extractor.firecrawl.firecrawl_web_extractor.WebsiteService.get_scrape_url_data",return_value={@@ -444,7 +444,7 @@ class TestFirecrawlWebExtractor:assert docs[0].page_content == "scrape content"mock_scrape.assert_called_once_with("firecrawl", "https://example.com", "tenant-1", False)- def test_extract_unknown_mode_returns_empty(self):+ def test_extract_unknown_mode_returns_empty(self) -> None:extractor = FirecrawlWebExtractor("https://example.com","job-1",diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.tomlindex e102df6b2a2..c6aa4fff8e0 100644--- a/api/tests/unit_tests/pyrefly.toml+++ b/api/tests/unit_tests/pyrefly.toml@@ -448,7 +448,6 @@ project-excludes = ["core/rag/embedding/test_embedding_base.py","core/rag/embedding/test_embedding_service.py","core/rag/extractor/blob/test_blob.py",- "core/rag/extractor/firecrawl/test_firecrawl.py","core/rag/extractor/test_csv_extractor.py","core/rag/extractor/test_excel_extractor.py","core/rag/extractor/test_extract_processor.py",