Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix: maintenance update (#43898)
-LAN-c523a7d52187
diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.pyindex 77edffa77cd..0fcfcb4bde9 100644--- a/api/core/rag/extractor/firecrawl/firecrawl_app.py+++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py@@ -4,6 +4,7 @@ from typing import Any, NotRequired, TypedDict, castimport httpx+from core.helper import ssrf_proxyfrom extensions.ext_storage import storage# Bounded connect/read timeout so a slow or hanging Firecrawl endpoint cannot@@ -180,7 +181,7 @@ class FirecrawlApp:def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response:response: httpx.Response | None = Nonefor attempt in range(retries):- response = httpx.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT)+ response = ssrf_proxy.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT, max_retries=0)if response.status_code == 502:time.sleep(backoff_factor * (2**attempt))else:@@ -191,7 +192,7 @@ class FirecrawlApp:def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response:response: httpx.Response | None = Nonefor attempt in range(retries):- response = httpx.get(url, headers=headers, timeout=_REQUEST_TIMEOUT)+ response = ssrf_proxy.get(url, headers=headers, timeout=_REQUEST_TIMEOUT, max_retries=0)if response.status_code == 502:time.sleep(backoff_factor * (2**attempt))else:diff --git a/api/core/rag/extractor/watercrawl/client.py b/api/core/rag/extractor/watercrawl/client.pyindex bb93730dd4c..eb301b7aba4 100644--- a/api/core/rag/extractor/watercrawl/client.py+++ b/api/core/rag/extractor/watercrawl/client.py@@ -6,6 +6,7 @@ from urllib.parse import urljoinimport httpxfrom httpx import Response+from core.helper import ssrf_proxyfrom core.rag.extractor.watercrawl.exceptions import (WaterCrawlAuthenticationError,WaterCrawlBadRequestError,@@ -46,19 +47,13 @@ class BaseAPIClient:def __init__(self, api_key, base_url):self.api_key = api_keyself.base_url = base_url- self.session = self.init_session()-- def init_session(self):- headers = {+ self.headers = {"X-API-Key": self.api_key,"Content-Type": "application/json","Accept": "application/json","User-Agent": "WaterCrawl-Plugin","Accept-Language": "en-US",}- # Regular requests use WATERCRAWL_REQUEST_TIMEOUT; the long-lived- # crawl-status stream overrides it with _STREAM_TIMEOUT in _request.- return httpx.Client(headers=headers, timeout=WATERCRAWL_REQUEST_TIMEOUT)def _request(self,@@ -70,11 +65,19 @@ class BaseAPIClient:) -> Response:stream = kwargs.pop("stream", False)url = urljoin(self.base_url, endpoint)- if stream:- request = self.session.build_request(method, url, params=query_params, json=data, timeout=_STREAM_TIMEOUT)- return self.session.send(request, stream=True, **kwargs)-- return self.session.request(method, url, params=query_params, json=data, **kwargs)+ headers = httpx.Headers(self.headers)+ headers.update(kwargs.pop("headers", {}))+ kwargs.setdefault("timeout", _STREAM_TIMEOUT if stream else WATERCRAWL_REQUEST_TIMEOUT)+ return ssrf_proxy.make_request(+ method,+ url,+ headers=dict(headers),+ params=query_params,+ json=data,+ max_retries=0,+ stream_response=stream,+ **kwargs,+ )def _get(self, endpoint: str, query_params: dict[str, Any] | None = None, **kwargs):return self._request("GET", endpoint, query_params=query_params, **kwargs)@@ -232,7 +235,7 @@ class WaterCrawlAPIClient(BaseAPIClient):return event_data["data"]def download_result(self, result_object: dict[str, Any]):- response = httpx.get(result_object["result"], timeout=WATERCRAWL_REQUEST_TIMEOUT)+ response = ssrf_proxy.get(result_object["result"], timeout=WATERCRAWL_REQUEST_TIMEOUT, max_retries=0)try:response.raise_for_status()result_object["result"] = response.json()diff --git a/api/services/data_source/auth/firecrawl/firecrawl.py b/api/services/data_source/auth/firecrawl/firecrawl.pyindex fb0b8d8e9aa..d393c067284 100644--- a/api/services/data_source/auth/firecrawl/firecrawl.py+++ b/api/services/data_source/auth/firecrawl/firecrawl.py@@ -4,6 +4,7 @@ from typing import Neverimport httpx+from core.helper import ssrf_proxyfrom services.auth.errors import (DataSourceApiKeyAuthCredentialValidationError,DataSourceApiKeyAuthProviderUnavailableError,@@ -49,7 +50,7 @@ class FirecrawlAuth:return f"{self.base_url.rstrip('/')}/{path.lstrip('/')}"def _post_request(self, url, data, headers):- return httpx.post(url, headers=headers, json=data, timeout=_CREDENTIAL_TIMEOUT)+ return ssrf_proxy.post(url, headers=headers, json=data, timeout=_CREDENTIAL_TIMEOUT, max_retries=0)def _handle_error(self, response) -> Never:if (diff --git a/api/services/data_source/auth/watercrawl/watercrawl.py b/api/services/data_source/auth/watercrawl/watercrawl.pyindex fe7a9e85f6b..b93b7c17eb3 100644--- a/api/services/data_source/auth/watercrawl/watercrawl.py+++ b/api/services/data_source/auth/watercrawl/watercrawl.py@@ -5,6 +5,7 @@ from urllib.parse import urljoinimport httpx+from core.helper import ssrf_proxyfrom services.auth.errors import (DataSourceApiKeyAuthCredentialValidationError,DataSourceApiKeyAuthProviderUnavailableError,@@ -42,7 +43,7 @@ class WatercrawlAuth:return {"Content-Type": "application/json", "X-API-KEY": self.api_key}def _get_request(self, url, headers):- return httpx.get(url, headers=headers, timeout=_CREDENTIAL_TIMEOUT)+ return ssrf_proxy.get(url, headers=headers, timeout=_CREDENTIAL_TIMEOUT, max_retries=0)def _handle_error(self, response) -> Never:if (diff --git a/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py b/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.pyindex 12ac010819f..83f1301e80c 100644--- a/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py+++ b/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py@@ -67,7 +67,7 @@ class TestAuthIntegration:encryptor=TenantApiKeyAuthCredentialEncryptor(),)- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")@patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token")def test_end_to_end_auth_flow(self,@@ -97,7 +97,7 @@ class TestAuthIntegration:assert bindings[0].provider == _FIRECRAWL_PROVIDER@patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token")- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")@patch("services.data_source.auth.jina.jina._http_client.post")def test_multi_tenant_isolation(self,@@ -132,7 +132,7 @@ class TestAuthIntegration:assert len(result2) == 1assert result2[0].provider == _JINA_PROVIDER- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")@patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token", return_value="encrypted_key")def test_concurrent_creation_safety(self,@@ -168,7 +168,7 @@ class TestAuthIntegration:assert len(results) == 5assert len(exceptions) == 0- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")def test_network_failure_recovery(self,mock_http,diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.pyindex fb558002d47..9d4d106c982 100644--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py@@ -40,8 +40,8 @@ class TestFirecrawlApp:def test_requests_use_bounded_timeout(self, mocker: MockerFixture) -> None:"""Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))- mock_get = mocker.patch("httpx.get", return_value=_response(200, {"status": "completed"}))+ mock_post = mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(200, {"id": "job-1"}))+ mock_get = mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, {"status": "completed"}))app._post_request("https://custom.firecrawl.dev/v1/crawl", {}, app._prepare_headers())app._get_request("https://custom.firecrawl.dev/v1/crawl/job-1", app._prepare_headers())@@ -52,7 +52,7 @@ class TestFirecrawlApp:def test_scrape_url_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch(- "httpx.post",+ "core.helper.ssrf_proxy.post",return_value=_response(200,{@@ -80,7 +80,7 @@ class TestFirecrawlApp:def test_scrape_url_handles_known_error_status(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("boom"))- mocker.patch("httpx.post", return_value=_response(429, {"error": "limit"}))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(429, {"error": "limit"}))with pytest.raises(Exception, match="boom"):app.scrape_url("https://example.com")@@ -89,21 +89,21 @@ class TestFirecrawlApp:def test_scrape_url_unknown_status_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(404, text="Not Found"))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(404, text="Not Found"))with pytest.raises(Exception, match="Failed to scrape URL. Status code: 404"):app.scrape_url("https://example.com")def test_crawl_url_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(200, {"id": "job-1"}))assert app.crawl_url("https://example.com") == "job-1"def test_crawl_url_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl failed"))- mocker.patch("httpx.post", return_value=_response(500, {"error": "server"}))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(500, {"error": "server"}))with pytest.raises(Exception, match="crawl failed"):app.crawl_url("https://example.com")@@ -112,14 +112,14 @@ class TestFirecrawlApp:def test_map_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(200, {"success": True, "links": ["a", "b"]}))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(200, {"success": True, "links": ["a", "b"]}))assert app.map("https://example.com") == {"success": True, "links": ["a", "b"]}def test_map_known_error(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("map error"))- mocker.patch("httpx.post", return_value=_response(409, {"error": "conflict"}))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(409, {"error": "conflict"}))with pytest.raises(Exception, match="map error"):app.map("https://example.com")@@ -127,7 +127,7 @@ class TestFirecrawlApp:def test_map_unknown_error_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(418, text="teapot"))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(418, text="teapot"))with pytest.raises(Exception, match="Failed to start map job. Status code: 418"):app.map("https://example.com")@@ -150,7 +150,7 @@ class TestFirecrawlApp:{"metadata": {"title": "skip"}},],}- mocker.patch("httpx.get", return_value=_response(200, payload))+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, payload))save_calls: list[tuple[str, bytes]] = []delete_calls: list[str] = []@@ -173,14 +173,19 @@ class TestFirecrawlApp:def test_check_crawl_status_completed_with_zero_total_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": 0, "data": []}))+ mocker.patch(+ "core.helper.ssrf_proxy.get", return_value=_response(200, {"status": "completed", "total": 0, "data": []})+ )with pytest.raises(Exception, match="No page found"):app.check_crawl_status("job-1")def test_check_crawl_status_completed_with_null_total_raises(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": None, "data": []}))+ mocker.patch(+ "core.helper.ssrf_proxy.get",+ return_value=_response(200, {"status": "completed", "total": None, "data": []}),+ )with pytest.raises(Exception, match="No page found"):app.check_crawl_status("job-1")@@ -188,7 +193,7 @@ class TestFirecrawlApp:def test_check_crawl_status_non_completed(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")payload = {"status": "processing", "total": 5, "completed": 1, "data": list[object]()}- mocker.patch("httpx.get", return_value=_response(200, payload))+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, payload))assert app.check_crawl_status("job-1") == {"status": "processing",@@ -200,7 +205,7 @@ class TestFirecrawlApp:def test_check_crawl_status_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl error"))- mocker.patch("httpx.get", return_value=_response(500, {"error": "server"}))+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(500, {"error": "server"}))with pytest.raises(Exception, match="crawl error"):app.check_crawl_status("job-1")@@ -214,7 +219,7 @@ class TestFirecrawlApp:"completed": 1,"data": [{"metadata": {"title": "a", "sourceURL": "https://a"}, "markdown": "m-a"}],}- mocker.patch("httpx.get", return_value=_response(200, payload))+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, payload))mock_storage = MagicMock()mock_storage.exists.return_value = False@@ -247,7 +252,10 @@ class TestFirecrawlApp:"completed": 3,"data": [{"metadata": {"title": "p3", "description": "", "sourceURL": "https://p3"}, "markdown": "m3"}],}- mocker.patch("httpx.get", side_effect=[_response(200, page1), _response(200, page2), _response(200, page3)])+ mocker.patch(+ "core.helper.ssrf_proxy.get",+ side_effect=[_response(200, page1), _response(200, page2), _response(200, page3)],+ )mock_storage = MagicMock()mock_storage.exists.return_value = Falsemocker.patch.object(firecrawl_module, "storage", mock_storage)@@ -269,7 +277,9 @@ class TestFirecrawlApp:"next": "https://custom.firecrawl.dev/v2/crawl/job-99?skip=1","data": [{"metadata": {"title": "p1", "description": "", "sourceURL": "https://p1"}, "markdown": "m1"}],}- mocker.patch("httpx.get", side_effect=[_response(200, page1), _response(500, {"error": "server error"})])+ mocker.patch(+ "core.helper.ssrf_proxy.get", side_effect=[_response(200, page1), _response(500, {"error": "server error"})]+ )with pytest.raises(Exception, match="fetch next crawl page"):app.check_crawl_status("job-99")@@ -285,7 +295,7 @@ class TestFirecrawlApp:"next": "https://custom.firecrawl.dev/v2/crawl/job-cap?skip=1","data": [{"metadata": {"title": "p1", "description": "", "sourceURL": "https://p1"}, "markdown": "m1"}],}- mock_get = mocker.patch("httpx.get", return_value=_response(200, page1))+ mock_get = mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, page1))mock_storage = MagicMock()mock_storage.exists.return_value = Falsemocker.patch.object(firecrawl_module, "storage", mock_storage)@@ -314,11 +324,11 @@ class TestFirecrawlApp:resp_502_b = _response(502)resp_200 = _response(200)- mocker.patch("httpx.post", side_effect=[resp_502_a, resp_200])+ mocker.patch("core.helper.ssrf_proxy.post", side_effect=[resp_502_a, resp_200])post_result = app._post_request("u", {"x": 1}, {"h": 1}, retries=3, backoff_factor=0.5)assert post_result is resp_200- mocker.patch("httpx.get", side_effect=[resp_502_b, _response(200)])+ mocker.patch("core.helper.ssrf_proxy.get", side_effect=[resp_502_b, _response(200)])get_result = app._get_request("u", {"h": 1}, retries=3, backoff_factor=0.25)assert get_result.status_code == 200@@ -329,11 +339,11 @@ class TestFirecrawlApp:sleep_mock = mocker.patch.object(firecrawl_module.time, "sleep")last_post = _response(502)- mocker.patch("httpx.post", side_effect=[_response(502), last_post])+ mocker.patch("core.helper.ssrf_proxy.post", side_effect=[_response(502), last_post])assert app._post_request("u", {}, {}, retries=2).status_code == 502last_get = _response(502)- mocker.patch("httpx.get", side_effect=[_response(502), last_get])+ mocker.patch("core.helper.ssrf_proxy.get", side_effect=[_response(502), last_get])assert app._get_request("u", {}, retries=2).status_code == 502assert sleep_mock.call_count == 4@@ -355,26 +365,30 @@ class TestFirecrawlApp:def test_search_success(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(200, {"success": True, "data": [{"url": "x"}]}))+ mocker.patch(+ "core.helper.ssrf_proxy.post", return_value=_response(200, {"success": True, "data": [{"url": "x"}]})+ )assert app.search("python")["success"] is Truedef test_search_warning_failure(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(200, {"success": False, "warning": "bad search"}))+ mocker.patch(+ "core.helper.ssrf_proxy.post", return_value=_response(200, {"success": False, "warning": "bad search"})+ )with pytest.raises(Exception, match="bad search"):app.search("python")def test_search_known_http_error(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("search error"))- mocker.patch("httpx.post", return_value=_response(408, {"error": "timeout"}))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(408, {"error": "timeout"}))with pytest.raises(Exception, match="search error"):app.search("python")mock_handle.assert_called_once()def test_search_unknown_http_error(self, mocker: MockerFixture) -> None:app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")- mocker.patch("httpx.post", return_value=_response(418, text="teapot"))+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(418, text="teapot"))with pytest.raises(Exception, match="Failed to perform search. Status code: 418"):app.search("python")diff --git a/api/tests/unit_tests/core/rag/extractor/test_crawler_ssrf.py b/api/tests/unit_tests/core/rag/extractor/test_crawler_ssrf.pynew file mode 100644index 00000000000..81e8ea05b2d--- /dev/null+++ b/api/tests/unit_tests/core/rag/extractor/test_crawler_ssrf.py@@ -0,0 +1,158 @@+from collections.abc import Callable, Iterator+from contextlib import ExitStack+from typing import Never++import httpx+import pytest++from core.helper import ssrf_proxy+from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp+from core.rag.extractor.watercrawl.client import WaterCrawlAPIClient+from core.tools.errors import ToolSSRFError+from services.data_source.auth.firecrawl.firecrawl import FirecrawlAuth+from services.data_source.auth.watercrawl.watercrawl import WatercrawlAuth+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentials+++def proxy_transport(+ monkeypatch: pytest.MonkeyPatch,+) -> Iterator[Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]]]:+ with ExitStack() as stack:++ def install(handler: Callable[[httpx.Request], httpx.Response]) -> list[httpx.Request]:+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ return handler(request)++ client = stack.enter_context(httpx.Client(transport=httpx.MockTransport(respond)))+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _verify: client)++ def reject_direct_request(*_args: object, **_kwargs: object) -> Never:+ pytest.fail("Crawler request bypassed the shared SSRF client")++ monkeypatch.setattr(httpx, "get", reject_direct_request)+ monkeypatch.setattr(httpx, "post", reject_direct_request)+ monkeypatch.setattr(httpx, "Client", reject_direct_request)+ return requests++ yield install+++ "operation",+ [+ "firecrawl_post",+ "firecrawl_get",+ "watercrawl_post",+ "watercrawl_get",+ "watercrawl_stream",+ "watercrawl_download",+ ],+)+def test_crawler_requests_honor_proxy_rejection(+ operation: str,+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],+) -> None:+ rejected = httpx.Response(403, headers={"Server": "squid"})+ requests = proxy_transport(lambda _request: rejected)+ firecrawl = FirecrawlApp(api_key="test-key", base_url="http://127.0.0.1")+ watercrawl = WaterCrawlAPIClient(api_key="test-key", base_url="http://127.0.0.1")+ operations: dict[str, Callable[[], object]] = {+ "firecrawl_post": lambda: firecrawl.scrape_url("https://example.com"),+ "firecrawl_get": lambda: firecrawl.check_crawl_status("job"),+ "watercrawl_post": lambda: watercrawl.create_crawl_request(url="https://example.com"),+ "watercrawl_get": lambda: watercrawl.get_crawl_request_results("job"),+ "watercrawl_stream": lambda: list(watercrawl.monitor_crawl_request("job")),+ "watercrawl_download": lambda: watercrawl.download_result({"result": "http://127.0.0.1/result.json"}),+ }++ with pytest.raises(ToolSSRFError, match="blocked by SSRF protection"):+ operations[operation]()++ assert len(requests) == 1+ assert requests[0].url.host == "127.0.0.1"+ assert rejected.is_closed+++def test_firecrawl_pagination_honors_proxy_rejection(+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],+) -> None:+ def respond(request: httpx.Request) -> httpx.Response:+ if request.url.host == "crawler.example.com":+ return httpx.Response(+ 200, json={"status": "completed", "total": 2, "data": [], "next": "http://127.0.0.1/next"}+ )+ return httpx.Response(403, headers={"Via": "squid"})++ requests = proxy_transport(respond)+ firecrawl = FirecrawlApp(api_key="test-key", base_url="https://crawler.example.com")++ with pytest.raises(ToolSSRFError, match="blocked by SSRF protection"):+ firecrawl.check_crawl_status("job")++ assert [str(request.url) for request in requests] == [+ "https://crawler.example.com/v2/crawl/job",+ "http://127.0.0.1/next",+ ]++[email protected]("provider", ["firecrawl", "watercrawl"])+def test_credential_validation_honors_proxy_rejection(+ provider: str,+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],+) -> None:+ requests = proxy_transport(lambda _request: httpx.Response(403, headers={"Server": "squid"}))+ credentials = DataSourceApiKeyAuthCredentials(+ "bearer" if provider == "firecrawl" else "x-api-key", "test-key", {"base_url": "http://127.0.0.1"}+ )+ auth = FirecrawlAuth(credentials) if provider == "firecrawl" else WatercrawlAuth(credentials)++ with pytest.raises(ToolSSRFError, match="blocked by SSRF protection"):+ auth.validate_credentials()++ assert len(requests) == 1+ assert requests[0].extensions["timeout"]["read"] == 10.0+++def test_watercrawl_stream_is_lazy_and_closed_after_events(+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],+) -> None:+ response = httpx.Response(+ 200,+ headers={"Content-Type": "text/event-stream"},+ stream=httpx.ByteStream(b'data: {"type": "result", "data": {"markdown": "page"}}\n\n'),+ )+ requests = proxy_transport(lambda _request: response)+ client = WaterCrawlAPIClient(api_key="test-key")+ events = client.monitor_crawl_request("job")++ assert requests == []+ assert next(events) == {"type": "result", "data": {"markdown": "page"}}+ events.close()+ assert response.is_closed+ assert requests[0].extensions["timeout"]["read"] is None++[email protected]("status_code", [200, 500])+def test_watercrawl_download_closes_response(+ status_code: int,+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],+) -> None:+ response = httpx.Response(status_code, json={"markdown": "page"})+ requests = proxy_transport(lambda _request: response)+ client = WaterCrawlAPIClient(api_key="test-key")++ if status_code == 200:+ assert client.download_result({"result": "https://results.example.com/page.json"}) == {+ "result": {"markdown": "page"}+ }+ else:+ with pytest.raises(httpx.HTTPStatusError):+ client.download_result({"result": "https://results.example.com/page.json"})++ assert len(requests) == 1+ assert "X-API-Key" not in requests[0].headers+ assert response.is_closeddiff --git a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.pyindex 0b13ebc84c5..304d56bb8cd 100644--- a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py@@ -62,72 +62,30 @@ class TestWaterCrawlExceptions:class TestBaseAPIClient:- def test_init_session_builds_expected_headers(self, monkeypatch: pytest.MonkeyPatch):- captured = {}-- def fake_client(**kwargs):- captured.update(kwargs)- return "session"-- monkeypatch.setattr(client_module.httpx, "Client", fake_client)-- client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")-- assert client.session == "session"- assert captured["headers"]["X-API-Key"] == "k"- assert captured["headers"]["User-Agent"] == "WaterCrawl-Plugin"- assert captured["timeout"] is not None- assert captured["timeout"].connect is not None- assert captured["timeout"].read is not None-- def test_init_session_uses_bounded_default_timeout(self):- # Regression: the session was built with timeout=None, which disables- # httpx's timeouts entirely, so a stalled WaterCrawl endpoint would hang- # the calling worker forever. Regular requests must keep a bounded timeout.- session = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev").session-- assert session._timeout.connect is not None- assert session._timeout.read is not None-- def test_request_stream_and_non_stream_paths(self, monkeypatch: pytest.MonkeyPatch):- class FakeSession:- def __init__(self):- self.request_calls = []- self.build_calls = []- self.send_calls = []-- def request(self, method, url, params=None, json=None, **kwargs):- self.request_calls.append((method, url, params, json, kwargs))- return "non-stream-response"-- def build_request(self, method, url, params=None, json=None, timeout=None):- req = (method, url, params, json, timeout)- self.build_calls.append(req)- return req-- def send(self, request, stream=False, **kwargs):- self.send_calls.append((request, stream, kwargs))- return "stream-response"-- fake_session = FakeSession()- monkeypatch.setattr(BaseAPIClient, "init_session", lambda self: fake_session)-- client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")-- assert client._request("GET", "/v1/items", query_params={"a": 1}) == "non-stream-response"- assert fake_session.request_calls[0][1] == "https://watercrawl.dev/v1/items"-- assert client._request("GET", "/v1/items", stream=True) == "stream-response"- assert fake_session.build_calls- assert fake_session.send_calls[0][1] is True- # the streaming request keeps an unbounded read (the SSE status stream can- # stay open for the whole crawl) while still capping the connection- stream_timeout = fake_session.build_calls[0][4]- assert stream_timeout.read is None- assert stream_timeout.connect is not None+ @pytest.mark.parametrize("stream", [False, True])+ def test_request_preserves_headers_and_timeouts(self, monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ return httpx.Response(200, stream=httpx.ByteStream(b"response"))++ with httpx.Client(transport=httpx.MockTransport(respond)) as proxy_client:+ monkeypatch.setattr(client_module.ssrf_proxy, "_get_ssrf_client", lambda _verify: proxy_client)+ client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")+ response = client._request("GET", "/v1/items", query_params={"a": 1}, stream=stream)++ assert response.is_stream_consumed is not stream+ assert requests[0].url == "https://watercrawl.dev/v1/items?a=1"+ assert requests[0].headers["X-API-Key"] == "k"+ assert requests[0].headers["User-Agent"] == "WaterCrawl-Plugin"+ timeout = requests[0].extensions["timeout"]+ assert timeout["connect"] == (10.0 if stream else 5.0)+ assert timeout["read"] == (None if stream else 30.0)+ response.close()+ assert response.is_closeddef test_http_method_helpers_delegate_to_request(self, monkeypatch: pytest.MonkeyPatch):- monkeypatch.setattr(BaseAPIClient, "init_session", lambda self: MagicMock())client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")calls = []@@ -299,7 +257,7 @@ class TestWaterCrawlAPIClient:captured.update(kwargs)return response- monkeypatch.setattr(client_module.httpx, "get", fake_get)+ monkeypatch.setattr(client_module.ssrf_proxy, "get", fake_get)result = client.download_result({"result": "https://example.com/result.json"})diff --git a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py b/api/tests/unit_tests/services/auth/test_firecrawl_auth.pyindex d606cc7db97..d77ab1195c7 100644--- a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py+++ b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py@@ -63,7 +63,7 @@ class TestFirecrawlAuth:with pytest.raises(InvalidDataSourceApiKeyAuthCredentialsError, match="No API key provided"):FirecrawlAuth(_credentials(api_key=""))- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_validate_valid_credentials_successfully(self, mock_post, auth_instance: FirecrawlAuth):"""Test successful credential validation"""mock_response = httpx.Response(200)@@ -84,6 +84,7 @@ class TestFirecrawlAuth:headers={"Content-Type": "application/json", "Authorization": "Bearer test_api_key_123"},json=expected_data,timeout=httpx.Timeout(10.0),+ max_retries=0,)@pytest.mark.parametrize(@@ -93,7 +94,7 @@ class TestFirecrawlAuth:(409, "Conflict error"),],)- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_handle_http_errors(self, mock_post, status_code, error_message, auth_instance: FirecrawlAuth):"""Test handling of various HTTP error codes"""mock_response = httpx.Response(status_code, json={"error": error_message})@@ -104,7 +105,7 @@ class TestFirecrawlAuth:assert str(exc_info.value) == f"Failed to authorize. Status code: {status_code}. Error: {error_message}"@pytest.mark.parametrize("status_code", [429, 500, 502, 503])- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_map_upstream_failure_to_provider_unavailable(self,mock_post,@@ -130,7 +131,7 @@ class TestFirecrawlAuth:(401, "Not JSON", True, "Failed to authorize. Status code: 401. Error: Not JSON"),],)- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_handle_unexpected_errors(self,mock_post,@@ -160,7 +161,7 @@ class TestFirecrawlAuth:(httpx.ConnectTimeout, "Connection timeout"),],)- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_handle_network_errors(self, mock_post, exception_type, exception_message, auth_instance: FirecrawlAuth):@@ -184,7 +185,7 @@ class TestFirecrawlAuth:FirecrawlAuth(_credentials(auth_type="basic", api_key="super_secret_key_12345"))assert "super_secret_key_12345" not in str(exc_info.value)- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_use_custom_base_url_in_validation(self, mock_post):"""Test that custom base URL is used in validation and normalized"""mock_response = httpx.Response(200)@@ -198,7 +199,7 @@ class TestFirecrawlAuth:assert result is Trueassert mock_post.call_args[0][0] == "https://custom.firecrawl.dev/v1/crawl"- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)def test_should_handle_timeout_with_retry_suggestion(self, mock_post, auth_instance: FirecrawlAuth):"""Test that timeout errors are handled gracefully with appropriate error message"""mock_post.side_effect = httpx.TimeoutException("The request timed out after 30 seconds")diff --git a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py b/api/tests/unit_tests/services/auth/test_watercrawl_auth.pyindex 1ae28d9a403..16661593f60 100644--- a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py+++ b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py@@ -65,7 +65,7 @@ class TestWatercrawlAuth:with pytest.raises(InvalidDataSourceApiKeyAuthCredentialsError, match="No API key provided"):WatercrawlAuth(_credentials(api_key=""))- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_validate_valid_credentials_successfully(self, mock_get: MagicMock, auth_instance: WatercrawlAuth) -> None:@@ -79,6 +79,7 @@ class TestWatercrawlAuth:"https://app.watercrawl.dev/api/v1/core/crawl-requests/",headers={"Content-Type": "application/json", "X-API-KEY": "test_api_key_123"},timeout=httpx.Timeout(10.0),+ max_retries=0,)@pytest.mark.parametrize(@@ -88,7 +89,7 @@ class TestWatercrawlAuth:(409, "Conflict error"),],)- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_handle_http_errors(self, mock_get: MagicMock, status_code: int, error_message: str, auth_instance: WatercrawlAuth) -> None:@@ -100,7 +101,7 @@ class TestWatercrawlAuth:assert str(exc_info.value) == f"Failed to authorize. Status code: {status_code}. Error: {error_message}"@pytest.mark.parametrize("status_code", [429, 500, 502, 503])- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_map_upstream_failure_to_provider_unavailable(self,mock_get: MagicMock,@@ -115,7 +116,7 @@ class TestWatercrawlAuth:assert exc_info.value.provider == "watercrawl"assert exc_info.value.status_code == status_code- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_handle_http_error_with_non_json_text_response(self, mock_get: MagicMock, auth_instance: WatercrawlAuth) -> None:@@ -134,7 +135,7 @@ class TestWatercrawlAuth:(401, "Not JSON", "Failed to authorize. Status code: 401. Error: Not JSON"),],)- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_handle_unexpected_errors(self,mock_get: MagicMock,@@ -159,7 +160,7 @@ class TestWatercrawlAuth:(httpx.ConnectTimeout, "Connection timeout"),],)- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_handle_network_errors(self,mock_get: MagicMock,@@ -187,7 +188,7 @@ class TestWatercrawlAuth:WatercrawlAuth(_credentials(auth_type="bearer", api_key="super_secret_key_12345"))assert "super_secret_key_12345" not in str(exc_info.value)- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_use_custom_base_url_in_validation(self, mock_get: MagicMock) -> None:"""Test that custom base URL is used in validation"""mock_get.return_value = httpx.Response(200)@@ -207,7 +208,7 @@ class TestWatercrawlAuth:("https://app.watercrawl.dev//", "https://app.watercrawl.dev/api/v1/core/crawl-requests/"),],)- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_use_urljoin_for_url_construction(self, mock_get: MagicMock, base_url: str, expected_url: str) -> None:@@ -221,7 +222,7 @@ class TestWatercrawlAuth:# Verify the correct URL was calledassert mock_get.call_args[0][0] == expected_url- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)def test_should_handle_timeout_with_retry_suggestion(self, mock_get: MagicMock, auth_instance: WatercrawlAuth) -> None: