Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: maintenance update (#43898)

-LAN-c523a7d52187
diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py
index 77edffa77cd..0fcfcb4bde9 100644
--- a/api/core/rag/extractor/firecrawl/firecrawl_app.py
+++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py
@@ -4,6 +4,7 @@ from typing import Any, NotRequired, TypedDict, cast
import httpx
+from core.helper import ssrf_proxy
from extensions.ext_storage import storage
# Bounded connect/read timeout so a slow or hanging Firecrawl endpoint cannot
@@ -180,7 +181,7 @@ class FirecrawlApp:
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
response: httpx.Response | None = None
for attempt in range(retries):
- response = httpx.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT)
+ response = ssrf_proxy.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT, max_retries=0)
if response.status_code == 502:
time.sleep(backoff_factor * (2**attempt))
else:
@@ -191,7 +192,7 @@ class FirecrawlApp:
def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
response: httpx.Response | None = None
for attempt in range(retries):
- response = httpx.get(url, headers=headers, timeout=_REQUEST_TIMEOUT)
+ response = ssrf_proxy.get(url, headers=headers, timeout=_REQUEST_TIMEOUT, max_retries=0)
if response.status_code == 502:
time.sleep(backoff_factor * (2**attempt))
else:
diff --git a/api/core/rag/extractor/watercrawl/client.py b/api/core/rag/extractor/watercrawl/client.py
index bb93730dd4c..eb301b7aba4 100644
--- a/api/core/rag/extractor/watercrawl/client.py
+++ b/api/core/rag/extractor/watercrawl/client.py
@@ -6,6 +6,7 @@ from urllib.parse import urljoin
import httpx
from httpx import Response
+from core.helper import ssrf_proxy
from core.rag.extractor.watercrawl.exceptions import (
WaterCrawlAuthenticationError,
WaterCrawlBadRequestError,
@@ -46,19 +47,13 @@ class BaseAPIClient:
def __init__(self, api_key, base_url):
self.api_key = api_key
self.base_url = base_url
- self.session = self.init_session()
-
- def init_session(self):
- headers = {
+ self.headers = {
"X-API-Key": self.api_key,
"Content-Type": "application/json",
"Accept": "application/json",
"User-Agent": "WaterCrawl-Plugin",
"Accept-Language": "en-US",
}
- # Regular requests use WATERCRAWL_REQUEST_TIMEOUT; the long-lived
- # crawl-status stream overrides it with _STREAM_TIMEOUT in _request.
- return httpx.Client(headers=headers, timeout=WATERCRAWL_REQUEST_TIMEOUT)
def _request(
self,
@@ -70,11 +65,19 @@ class BaseAPIClient:
) -> Response:
stream = kwargs.pop("stream", False)
url = urljoin(self.base_url, endpoint)
- if stream:
- request = self.session.build_request(method, url, params=query_params, json=data, timeout=_STREAM_TIMEOUT)
- return self.session.send(request, stream=True, **kwargs)
-
- return self.session.request(method, url, params=query_params, json=data, **kwargs)
+ headers = httpx.Headers(self.headers)
+ headers.update(kwargs.pop("headers", {}))
+ kwargs.setdefault("timeout", _STREAM_TIMEOUT if stream else WATERCRAWL_REQUEST_TIMEOUT)
+ return ssrf_proxy.make_request(
+ method,
+ url,
+ headers=dict(headers),
+ params=query_params,
+ json=data,
+ max_retries=0,
+ stream_response=stream,
+ **kwargs,
+ )
def _get(self, endpoint: str, query_params: dict[str, Any] | None = None, **kwargs):
return self._request("GET", endpoint, query_params=query_params, **kwargs)
@@ -232,7 +235,7 @@ class WaterCrawlAPIClient(BaseAPIClient):
return event_data["data"]
def download_result(self, result_object: dict[str, Any]):
- response = httpx.get(result_object["result"], timeout=WATERCRAWL_REQUEST_TIMEOUT)
+ response = ssrf_proxy.get(result_object["result"], timeout=WATERCRAWL_REQUEST_TIMEOUT, max_retries=0)
try:
response.raise_for_status()
result_object["result"] = response.json()
diff --git a/api/services/data_source/auth/firecrawl/firecrawl.py b/api/services/data_source/auth/firecrawl/firecrawl.py
index fb0b8d8e9aa..d393c067284 100644
--- a/api/services/data_source/auth/firecrawl/firecrawl.py
+++ b/api/services/data_source/auth/firecrawl/firecrawl.py
@@ -4,6 +4,7 @@ from typing import Never
import httpx
+from core.helper import ssrf_proxy
from services.auth.errors import (
DataSourceApiKeyAuthCredentialValidationError,
DataSourceApiKeyAuthProviderUnavailableError,
@@ -49,7 +50,7 @@ class FirecrawlAuth:
return f"{self.base_url.rstrip('/')}/{path.lstrip('/')}"
def _post_request(self, url, data, headers):
- return httpx.post(url, headers=headers, json=data, timeout=_CREDENTIAL_TIMEOUT)
+ return ssrf_proxy.post(url, headers=headers, json=data, timeout=_CREDENTIAL_TIMEOUT, max_retries=0)
def _handle_error(self, response) -> Never:
if (
diff --git a/api/services/data_source/auth/watercrawl/watercrawl.py b/api/services/data_source/auth/watercrawl/watercrawl.py
index fe7a9e85f6b..b93b7c17eb3 100644
--- a/api/services/data_source/auth/watercrawl/watercrawl.py
+++ b/api/services/data_source/auth/watercrawl/watercrawl.py
@@ -5,6 +5,7 @@ from urllib.parse import urljoin
import httpx
+from core.helper import ssrf_proxy
from services.auth.errors import (
DataSourceApiKeyAuthCredentialValidationError,
DataSourceApiKeyAuthProviderUnavailableError,
@@ -42,7 +43,7 @@ class WatercrawlAuth:
return {"Content-Type": "application/json", "X-API-KEY": self.api_key}
def _get_request(self, url, headers):
- return httpx.get(url, headers=headers, timeout=_CREDENTIAL_TIMEOUT)
+ return ssrf_proxy.get(url, headers=headers, timeout=_CREDENTIAL_TIMEOUT, max_retries=0)
def _handle_error(self, response) -> Never:
if (
diff --git a/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py b/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py
index 12ac010819f..83f1301e80c 100644
--- a/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py
+++ b/api/tests/test_containers_integration_tests/services/auth/test_auth_integration.py
@@ -67,7 +67,7 @@ class TestAuthIntegration:
encryptor=TenantApiKeyAuthCredentialEncryptor(),
)
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")
@patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token")
def test_end_to_end_auth_flow(
self,
@@ -97,7 +97,7 @@ class TestAuthIntegration:
assert bindings[0].provider == _FIRECRAWL_PROVIDER
@patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token")
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")
@patch("services.data_source.auth.jina.jina._http_client.post")
def test_multi_tenant_isolation(
self,
@@ -132,7 +132,7 @@ class TestAuthIntegration:
assert len(result2) == 1
assert result2[0].provider == _JINA_PROVIDER
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")
@patch("services.data_source.auth.api_key_gateways.encrypter.encrypt_token", return_value="encrypted_key")
def test_concurrent_creation_safety(
self,
@@ -168,7 +168,7 @@ class TestAuthIntegration:
assert len(results) == 5
assert len(exceptions) == 0
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post")
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post")
def test_network_failure_recovery(
self,
mock_http,
diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
index fb558002d47..9d4d106c982 100644
--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
@@ -40,8 +40,8 @@ class TestFirecrawlApp:
def test_requests_use_bounded_timeout(self, mocker: MockerFixture) -> None:
"""Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))
- mock_get = mocker.patch("httpx.get", return_value=_response(200, {"status": "completed"}))
+ mock_post = mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(200, {"id": "job-1"}))
+ mock_get = mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, {"status": "completed"}))
app._post_request("https://custom.firecrawl.dev/v1/crawl", {}, app._prepare_headers())
app._get_request("https://custom.firecrawl.dev/v1/crawl/job-1", app._prepare_headers())
@@ -52,7 +52,7 @@ class TestFirecrawlApp:
def test_scrape_url_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch(
- "httpx.post",
+ "core.helper.ssrf_proxy.post",
return_value=_response(
200,
{
@@ -80,7 +80,7 @@ class TestFirecrawlApp:
def test_scrape_url_handles_known_error_status(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("boom"))
- mocker.patch("httpx.post", return_value=_response(429, {"error": "limit"}))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(429, {"error": "limit"}))
with pytest.raises(Exception, match="boom"):
app.scrape_url("https://example.com")
@@ -89,21 +89,21 @@ class TestFirecrawlApp:
def test_scrape_url_unknown_status_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(404, text="Not Found"))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(404, text="Not Found"))
with pytest.raises(Exception, match="Failed to scrape URL. Status code: 404"):
app.scrape_url("https://example.com")
def test_crawl_url_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(200, {"id": "job-1"}))
assert app.crawl_url("https://example.com") == "job-1"
def test_crawl_url_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl failed"))
- mocker.patch("httpx.post", return_value=_response(500, {"error": "server"}))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(500, {"error": "server"}))
with pytest.raises(Exception, match="crawl failed"):
app.crawl_url("https://example.com")
@@ -112,14 +112,14 @@ class TestFirecrawlApp:
def test_map_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(200, {"success": True, "links": ["a", "b"]}))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(200, {"success": True, "links": ["a", "b"]}))
assert app.map("https://example.com") == {"success": True, "links": ["a", "b"]}
def test_map_known_error(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("map error"))
- mocker.patch("httpx.post", return_value=_response(409, {"error": "conflict"}))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(409, {"error": "conflict"}))
with pytest.raises(Exception, match="map error"):
app.map("https://example.com")
@@ -127,7 +127,7 @@ class TestFirecrawlApp:
def test_map_unknown_error_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(418, text="teapot"))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(418, text="teapot"))
with pytest.raises(Exception, match="Failed to start map job. Status code: 418"):
app.map("https://example.com")
@@ -150,7 +150,7 @@ class TestFirecrawlApp:
{"metadata": {"title": "skip"}},
],
}
- mocker.patch("httpx.get", return_value=_response(200, payload))
+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, payload))
save_calls: list[tuple[str, bytes]] = []
delete_calls: list[str] = []
@@ -173,14 +173,19 @@ class TestFirecrawlApp:
def test_check_crawl_status_completed_with_zero_total_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": 0, "data": []}))
+ mocker.patch(
+ "core.helper.ssrf_proxy.get", return_value=_response(200, {"status": "completed", "total": 0, "data": []})
+ )
with pytest.raises(Exception, match="No page found"):
app.check_crawl_status("job-1")
def test_check_crawl_status_completed_with_null_total_raises(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.get", return_value=_response(200, {"status": "completed", "total": None, "data": []}))
+ mocker.patch(
+ "core.helper.ssrf_proxy.get",
+ return_value=_response(200, {"status": "completed", "total": None, "data": []}),
+ )
with pytest.raises(Exception, match="No page found"):
app.check_crawl_status("job-1")
@@ -188,7 +193,7 @@ class TestFirecrawlApp:
def test_check_crawl_status_non_completed(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
payload = {"status": "processing", "total": 5, "completed": 1, "data": list[object]()}
- mocker.patch("httpx.get", return_value=_response(200, payload))
+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, payload))
assert app.check_crawl_status("job-1") == {
"status": "processing",
@@ -200,7 +205,7 @@ class TestFirecrawlApp:
def test_check_crawl_status_non_200_uses_error_handler(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("crawl error"))
- mocker.patch("httpx.get", return_value=_response(500, {"error": "server"}))
+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(500, {"error": "server"}))
with pytest.raises(Exception, match="crawl error"):
app.check_crawl_status("job-1")
@@ -214,7 +219,7 @@ class TestFirecrawlApp:
"completed": 1,
"data": [{"metadata": {"title": "a", "sourceURL": "https://a"}, "markdown": "m-a"}],
}
- mocker.patch("httpx.get", return_value=_response(200, payload))
+ mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, payload))
mock_storage = MagicMock()
mock_storage.exists.return_value = False
@@ -247,7 +252,10 @@ class TestFirecrawlApp:
"completed": 3,
"data": [{"metadata": {"title": "p3", "description": "", "sourceURL": "https://p3"}, "markdown": "m3"}],
}
- mocker.patch("httpx.get", side_effect=[_response(200, page1), _response(200, page2), _response(200, page3)])
+ mocker.patch(
+ "core.helper.ssrf_proxy.get",
+ side_effect=[_response(200, page1), _response(200, page2), _response(200, page3)],
+ )
mock_storage = MagicMock()
mock_storage.exists.return_value = False
mocker.patch.object(firecrawl_module, "storage", mock_storage)
@@ -269,7 +277,9 @@ class TestFirecrawlApp:
"next": "https://custom.firecrawl.dev/v2/crawl/job-99?skip=1",
"data": [{"metadata": {"title": "p1", "description": "", "sourceURL": "https://p1"}, "markdown": "m1"}],
}
- mocker.patch("httpx.get", side_effect=[_response(200, page1), _response(500, {"error": "server error"})])
+ mocker.patch(
+ "core.helper.ssrf_proxy.get", side_effect=[_response(200, page1), _response(500, {"error": "server error"})]
+ )
with pytest.raises(Exception, match="fetch next crawl page"):
app.check_crawl_status("job-99")
@@ -285,7 +295,7 @@ class TestFirecrawlApp:
"next": "https://custom.firecrawl.dev/v2/crawl/job-cap?skip=1",
"data": [{"metadata": {"title": "p1", "description": "", "sourceURL": "https://p1"}, "markdown": "m1"}],
}
- mock_get = mocker.patch("httpx.get", return_value=_response(200, page1))
+ mock_get = mocker.patch("core.helper.ssrf_proxy.get", return_value=_response(200, page1))
mock_storage = MagicMock()
mock_storage.exists.return_value = False
mocker.patch.object(firecrawl_module, "storage", mock_storage)
@@ -314,11 +324,11 @@ class TestFirecrawlApp:
resp_502_b = _response(502)
resp_200 = _response(200)
- mocker.patch("httpx.post", side_effect=[resp_502_a, resp_200])
+ mocker.patch("core.helper.ssrf_proxy.post", side_effect=[resp_502_a, resp_200])
post_result = app._post_request("u", {"x": 1}, {"h": 1}, retries=3, backoff_factor=0.5)
assert post_result is resp_200
- mocker.patch("httpx.get", side_effect=[resp_502_b, _response(200)])
+ mocker.patch("core.helper.ssrf_proxy.get", side_effect=[resp_502_b, _response(200)])
get_result = app._get_request("u", {"h": 1}, retries=3, backoff_factor=0.25)
assert get_result.status_code == 200
@@ -329,11 +339,11 @@ class TestFirecrawlApp:
sleep_mock = mocker.patch.object(firecrawl_module.time, "sleep")
last_post = _response(502)
- mocker.patch("httpx.post", side_effect=[_response(502), last_post])
+ mocker.patch("core.helper.ssrf_proxy.post", side_effect=[_response(502), last_post])
assert app._post_request("u", {}, {}, retries=2).status_code == 502
last_get = _response(502)
- mocker.patch("httpx.get", side_effect=[_response(502), last_get])
+ mocker.patch("core.helper.ssrf_proxy.get", side_effect=[_response(502), last_get])
assert app._get_request("u", {}, retries=2).status_code == 502
assert sleep_mock.call_count == 4
@@ -355,26 +365,30 @@ class TestFirecrawlApp:
def test_search_success(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(200, {"success": True, "data": [{"url": "x"}]}))
+ mocker.patch(
+ "core.helper.ssrf_proxy.post", return_value=_response(200, {"success": True, "data": [{"url": "x"}]})
+ )
assert app.search("python")["success"] is True
def test_search_warning_failure(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(200, {"success": False, "warning": "bad search"}))
+ mocker.patch(
+ "core.helper.ssrf_proxy.post", return_value=_response(200, {"success": False, "warning": "bad search"})
+ )
with pytest.raises(Exception, match="bad search"):
app.search("python")
def test_search_known_http_error(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_handle = mocker.patch.object(app, "_handle_error", side_effect=Exception("search error"))
- mocker.patch("httpx.post", return_value=_response(408, {"error": "timeout"}))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(408, {"error": "timeout"}))
with pytest.raises(Exception, match="search error"):
app.search("python")
mock_handle.assert_called_once()
def test_search_unknown_http_error(self, mocker: MockerFixture) -> None:
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
- mocker.patch("httpx.post", return_value=_response(418, text="teapot"))
+ mocker.patch("core.helper.ssrf_proxy.post", return_value=_response(418, text="teapot"))
with pytest.raises(Exception, match="Failed to perform search. Status code: 418"):
app.search("python")
diff --git a/api/tests/unit_tests/core/rag/extractor/test_crawler_ssrf.py b/api/tests/unit_tests/core/rag/extractor/test_crawler_ssrf.py
new file mode 100644
index 00000000000..81e8ea05b2d
--- /dev/null
+++ b/api/tests/unit_tests/core/rag/extractor/test_crawler_ssrf.py
@@ -0,0 +1,158 @@
+from collections.abc import Callable, Iterator
+from contextlib import ExitStack
+from typing import Never
+
+import httpx
+import pytest
+
+from core.helper import ssrf_proxy
+from core.rag.extractor.firecrawl.firecrawl_app import FirecrawlApp
+from core.rag.extractor.watercrawl.client import WaterCrawlAPIClient
+from core.tools.errors import ToolSSRFError
+from services.data_source.auth.firecrawl.firecrawl import FirecrawlAuth
+from services.data_source.auth.watercrawl.watercrawl import WatercrawlAuth
+from services.data_source.entities.api_key_auth import DataSourceApiKeyAuthCredentials
+
+
+def proxy_transport(
+ monkeypatch: pytest.MonkeyPatch,
+) -> Iterator[Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]]]:
+ with ExitStack() as stack:
+
+ def install(handler: Callable[[httpx.Request], httpx.Response]) -> list[httpx.Request]:
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return handler(request)
+
+ client = stack.enter_context(httpx.Client(transport=httpx.MockTransport(respond)))
+ monkeypatch.setattr(ssrf_proxy, "_get_ssrf_client", lambda _verify: client)
+
+ def reject_direct_request(*_args: object, **_kwargs: object) -> Never:
+ pytest.fail("Crawler request bypassed the shared SSRF client")
+
+ monkeypatch.setattr(httpx, "get", reject_direct_request)
+ monkeypatch.setattr(httpx, "post", reject_direct_request)
+ monkeypatch.setattr(httpx, "Client", reject_direct_request)
+ return requests
+
+ yield install
+
+
+ "operation",
+ [
+ "firecrawl_post",
+ "firecrawl_get",
+ "watercrawl_post",
+ "watercrawl_get",
+ "watercrawl_stream",
+ "watercrawl_download",
+ ],
+)
+def test_crawler_requests_honor_proxy_rejection(
+ operation: str,
+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],
+) -> None:
+ rejected = httpx.Response(403, headers={"Server": "squid"})
+ requests = proxy_transport(lambda _request: rejected)
+ firecrawl = FirecrawlApp(api_key="test-key", base_url="http://127.0.0.1")
+ watercrawl = WaterCrawlAPIClient(api_key="test-key", base_url="http://127.0.0.1")
+ operations: dict[str, Callable[[], object]] = {
+ "firecrawl_post": lambda: firecrawl.scrape_url("https://example.com"),
+ "firecrawl_get": lambda: firecrawl.check_crawl_status("job"),
+ "watercrawl_post": lambda: watercrawl.create_crawl_request(url="https://example.com"),
+ "watercrawl_get": lambda: watercrawl.get_crawl_request_results("job"),
+ "watercrawl_stream": lambda: list(watercrawl.monitor_crawl_request("job")),
+ "watercrawl_download": lambda: watercrawl.download_result({"result": "http://127.0.0.1/result.json"}),
+ }
+
+ with pytest.raises(ToolSSRFError, match="blocked by SSRF protection"):
+ operations[operation]()
+
+ assert len(requests) == 1
+ assert requests[0].url.host == "127.0.0.1"
+ assert rejected.is_closed
+
+
+def test_firecrawl_pagination_honors_proxy_rejection(
+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],
+) -> None:
+ def respond(request: httpx.Request) -> httpx.Response:
+ if request.url.host == "crawler.example.com":
+ return httpx.Response(
+ 200, json={"status": "completed", "total": 2, "data": [], "next": "http://127.0.0.1/next"}
+ )
+ return httpx.Response(403, headers={"Via": "squid"})
+
+ requests = proxy_transport(respond)
+ firecrawl = FirecrawlApp(api_key="test-key", base_url="https://crawler.example.com")
+
+ with pytest.raises(ToolSSRFError, match="blocked by SSRF protection"):
+ firecrawl.check_crawl_status("job")
+
+ assert [str(request.url) for request in requests] == [
+ "https://crawler.example.com/v2/crawl/job",
+ "http://127.0.0.1/next",
+ ]
+
+
[email protected]("provider", ["firecrawl", "watercrawl"])
+def test_credential_validation_honors_proxy_rejection(
+ provider: str,
+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],
+) -> None:
+ requests = proxy_transport(lambda _request: httpx.Response(403, headers={"Server": "squid"}))
+ credentials = DataSourceApiKeyAuthCredentials(
+ "bearer" if provider == "firecrawl" else "x-api-key", "test-key", {"base_url": "http://127.0.0.1"}
+ )
+ auth = FirecrawlAuth(credentials) if provider == "firecrawl" else WatercrawlAuth(credentials)
+
+ with pytest.raises(ToolSSRFError, match="blocked by SSRF protection"):
+ auth.validate_credentials()
+
+ assert len(requests) == 1
+ assert requests[0].extensions["timeout"]["read"] == 10.0
+
+
+def test_watercrawl_stream_is_lazy_and_closed_after_events(
+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],
+) -> None:
+ response = httpx.Response(
+ 200,
+ headers={"Content-Type": "text/event-stream"},
+ stream=httpx.ByteStream(b'data: {"type": "result", "data": {"markdown": "page"}}\n\n'),
+ )
+ requests = proxy_transport(lambda _request: response)
+ client = WaterCrawlAPIClient(api_key="test-key")
+ events = client.monitor_crawl_request("job")
+
+ assert requests == []
+ assert next(events) == {"type": "result", "data": {"markdown": "page"}}
+ events.close()
+ assert response.is_closed
+ assert requests[0].extensions["timeout"]["read"] is None
+
+
[email protected]("status_code", [200, 500])
+def test_watercrawl_download_closes_response(
+ status_code: int,
+ proxy_transport: Callable[[Callable[[httpx.Request], httpx.Response]], list[httpx.Request]],
+) -> None:
+ response = httpx.Response(status_code, json={"markdown": "page"})
+ requests = proxy_transport(lambda _request: response)
+ client = WaterCrawlAPIClient(api_key="test-key")
+
+ if status_code == 200:
+ assert client.download_result({"result": "https://results.example.com/page.json"}) == {
+ "result": {"markdown": "page"}
+ }
+ else:
+ with pytest.raises(httpx.HTTPStatusError):
+ client.download_result({"result": "https://results.example.com/page.json"})
+
+ assert len(requests) == 1
+ assert "X-API-Key" not in requests[0].headers
+ assert response.is_closed
diff --git a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
index 0b13ebc84c5..304d56bb8cd 100644
--- a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
+++ b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
@@ -62,72 +62,30 @@ class TestWaterCrawlExceptions:
class TestBaseAPIClient:
- def test_init_session_builds_expected_headers(self, monkeypatch: pytest.MonkeyPatch):
- captured = {}
-
- def fake_client(**kwargs):
- captured.update(kwargs)
- return "session"
-
- monkeypatch.setattr(client_module.httpx, "Client", fake_client)
-
- client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")
-
- assert client.session == "session"
- assert captured["headers"]["X-API-Key"] == "k"
- assert captured["headers"]["User-Agent"] == "WaterCrawl-Plugin"
- assert captured["timeout"] is not None
- assert captured["timeout"].connect is not None
- assert captured["timeout"].read is not None
-
- def test_init_session_uses_bounded_default_timeout(self):
- # Regression: the session was built with timeout=None, which disables
- # httpx's timeouts entirely, so a stalled WaterCrawl endpoint would hang
- # the calling worker forever. Regular requests must keep a bounded timeout.
- session = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev").session
-
- assert session._timeout.connect is not None
- assert session._timeout.read is not None
-
- def test_request_stream_and_non_stream_paths(self, monkeypatch: pytest.MonkeyPatch):
- class FakeSession:
- def __init__(self):
- self.request_calls = []
- self.build_calls = []
- self.send_calls = []
-
- def request(self, method, url, params=None, json=None, **kwargs):
- self.request_calls.append((method, url, params, json, kwargs))
- return "non-stream-response"
-
- def build_request(self, method, url, params=None, json=None, timeout=None):
- req = (method, url, params, json, timeout)
- self.build_calls.append(req)
- return req
-
- def send(self, request, stream=False, **kwargs):
- self.send_calls.append((request, stream, kwargs))
- return "stream-response"
-
- fake_session = FakeSession()
- monkeypatch.setattr(BaseAPIClient, "init_session", lambda self: fake_session)
-
- client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")
-
- assert client._request("GET", "/v1/items", query_params={"a": 1}) == "non-stream-response"
- assert fake_session.request_calls[0][1] == "https://watercrawl.dev/v1/items"
-
- assert client._request("GET", "/v1/items", stream=True) == "stream-response"
- assert fake_session.build_calls
- assert fake_session.send_calls[0][1] is True
- # the streaming request keeps an unbounded read (the SSE status stream can
- # stay open for the whole crawl) while still capping the connection
- stream_timeout = fake_session.build_calls[0][4]
- assert stream_timeout.read is None
- assert stream_timeout.connect is not None
+ @pytest.mark.parametrize("stream", [False, True])
+ def test_request_preserves_headers_and_timeouts(self, monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return httpx.Response(200, stream=httpx.ByteStream(b"response"))
+
+ with httpx.Client(transport=httpx.MockTransport(respond)) as proxy_client:
+ monkeypatch.setattr(client_module.ssrf_proxy, "_get_ssrf_client", lambda _verify: proxy_client)
+ client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")
+ response = client._request("GET", "/v1/items", query_params={"a": 1}, stream=stream)
+
+ assert response.is_stream_consumed is not stream
+ assert requests[0].url == "https://watercrawl.dev/v1/items?a=1"
+ assert requests[0].headers["X-API-Key"] == "k"
+ assert requests[0].headers["User-Agent"] == "WaterCrawl-Plugin"
+ timeout = requests[0].extensions["timeout"]
+ assert timeout["connect"] == (10.0 if stream else 5.0)
+ assert timeout["read"] == (None if stream else 30.0)
+ response.close()
+ assert response.is_closed
def test_http_method_helpers_delegate_to_request(self, monkeypatch: pytest.MonkeyPatch):
- monkeypatch.setattr(BaseAPIClient, "init_session", lambda self: MagicMock())
client = BaseAPIClient(api_key="k", base_url="https://watercrawl.dev")
calls = []
@@ -299,7 +257,7 @@ class TestWaterCrawlAPIClient:
captured.update(kwargs)
return response
- monkeypatch.setattr(client_module.httpx, "get", fake_get)
+ monkeypatch.setattr(client_module.ssrf_proxy, "get", fake_get)
result = client.download_result({"result": "https://example.com/result.json"})
diff --git a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py
index d606cc7db97..d77ab1195c7 100644
--- a/api/tests/unit_tests/services/auth/test_firecrawl_auth.py
+++ b/api/tests/unit_tests/services/auth/test_firecrawl_auth.py
@@ -63,7 +63,7 @@ class TestFirecrawlAuth:
with pytest.raises(InvalidDataSourceApiKeyAuthCredentialsError, match="No API key provided"):
FirecrawlAuth(_credentials(api_key=""))
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_validate_valid_credentials_successfully(self, mock_post, auth_instance: FirecrawlAuth):
"""Test successful credential validation"""
mock_response = httpx.Response(200)
@@ -84,6 +84,7 @@ class TestFirecrawlAuth:
headers={"Content-Type": "application/json", "Authorization": "Bearer test_api_key_123"},
json=expected_data,
timeout=httpx.Timeout(10.0),
+ max_retries=0,
)
@pytest.mark.parametrize(
@@ -93,7 +94,7 @@ class TestFirecrawlAuth:
(409, "Conflict error"),
],
)
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_handle_http_errors(self, mock_post, status_code, error_message, auth_instance: FirecrawlAuth):
"""Test handling of various HTTP error codes"""
mock_response = httpx.Response(status_code, json={"error": error_message})
@@ -104,7 +105,7 @@ class TestFirecrawlAuth:
assert str(exc_info.value) == f"Failed to authorize. Status code: {status_code}. Error: {error_message}"
@pytest.mark.parametrize("status_code", [429, 500, 502, 503])
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_map_upstream_failure_to_provider_unavailable(
self,
mock_post,
@@ -130,7 +131,7 @@ class TestFirecrawlAuth:
(401, "Not JSON", True, "Failed to authorize. Status code: 401. Error: Not JSON"),
],
)
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_handle_unexpected_errors(
self,
mock_post,
@@ -160,7 +161,7 @@ class TestFirecrawlAuth:
(httpx.ConnectTimeout, "Connection timeout"),
],
)
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_handle_network_errors(
self, mock_post, exception_type, exception_message, auth_instance: FirecrawlAuth
):
@@ -184,7 +185,7 @@ class TestFirecrawlAuth:
FirecrawlAuth(_credentials(auth_type="basic", api_key="super_secret_key_12345"))
assert "super_secret_key_12345" not in str(exc_info.value)
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_use_custom_base_url_in_validation(self, mock_post):
"""Test that custom base URL is used in validation and normalized"""
mock_response = httpx.Response(200)
@@ -198,7 +199,7 @@ class TestFirecrawlAuth:
assert result is True
assert mock_post.call_args[0][0] == "https://custom.firecrawl.dev/v1/crawl"
- @patch("services.data_source.auth.firecrawl.firecrawl.httpx.post", autospec=True)
+ @patch("services.data_source.auth.firecrawl.firecrawl.ssrf_proxy.post", autospec=True)
def test_should_handle_timeout_with_retry_suggestion(self, mock_post, auth_instance: FirecrawlAuth):
"""Test that timeout errors are handled gracefully with appropriate error message"""
mock_post.side_effect = httpx.TimeoutException("The request timed out after 30 seconds")
diff --git a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
index 1ae28d9a403..16661593f60 100644
--- a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
+++ b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
@@ -65,7 +65,7 @@ class TestWatercrawlAuth:
with pytest.raises(InvalidDataSourceApiKeyAuthCredentialsError, match="No API key provided"):
WatercrawlAuth(_credentials(api_key=""))
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_validate_valid_credentials_successfully(
self, mock_get: MagicMock, auth_instance: WatercrawlAuth
) -> None:
@@ -79,6 +79,7 @@ class TestWatercrawlAuth:
"https://app.watercrawl.dev/api/v1/core/crawl-requests/",
headers={"Content-Type": "application/json", "X-API-KEY": "test_api_key_123"},
timeout=httpx.Timeout(10.0),
+ max_retries=0,
)
@pytest.mark.parametrize(
@@ -88,7 +89,7 @@ class TestWatercrawlAuth:
(409, "Conflict error"),
],
)
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_handle_http_errors(
self, mock_get: MagicMock, status_code: int, error_message: str, auth_instance: WatercrawlAuth
) -> None:
@@ -100,7 +101,7 @@ class TestWatercrawlAuth:
assert str(exc_info.value) == f"Failed to authorize. Status code: {status_code}. Error: {error_message}"
@pytest.mark.parametrize("status_code", [429, 500, 502, 503])
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_map_upstream_failure_to_provider_unavailable(
self,
mock_get: MagicMock,
@@ -115,7 +116,7 @@ class TestWatercrawlAuth:
assert exc_info.value.provider == "watercrawl"
assert exc_info.value.status_code == status_code
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_handle_http_error_with_non_json_text_response(
self, mock_get: MagicMock, auth_instance: WatercrawlAuth
) -> None:
@@ -134,7 +135,7 @@ class TestWatercrawlAuth:
(401, "Not JSON", "Failed to authorize. Status code: 401. Error: Not JSON"),
],
)
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_handle_unexpected_errors(
self,
mock_get: MagicMock,
@@ -159,7 +160,7 @@ class TestWatercrawlAuth:
(httpx.ConnectTimeout, "Connection timeout"),
],
)
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_handle_network_errors(
self,
mock_get: MagicMock,
@@ -187,7 +188,7 @@ class TestWatercrawlAuth:
WatercrawlAuth(_credentials(auth_type="bearer", api_key="super_secret_key_12345"))
assert "super_secret_key_12345" not in str(exc_info.value)
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_use_custom_base_url_in_validation(self, mock_get: MagicMock) -> None:
"""Test that custom base URL is used in validation"""
mock_get.return_value = httpx.Response(200)
@@ -207,7 +208,7 @@ class TestWatercrawlAuth:
("https://app.watercrawl.dev//", "https://app.watercrawl.dev/api/v1/core/crawl-requests/"),
],
)
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_use_urljoin_for_url_construction(
self, mock_get: MagicMock, base_url: str, expected_url: str
) -> None:
@@ -221,7 +222,7 @@ class TestWatercrawlAuth:
# Verify the correct URL was called
assert mock_get.call_args[0][0] == expected_url
- @patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
+ @patch("services.data_source.auth.watercrawl.watercrawl.ssrf_proxy.get", autospec=True)
def test_should_handle_timeout_with_retry_suggestion(
self, mock_get: MagicMock, auth_instance: WatercrawlAuth
) -> None: