Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): add bounded timeout to Firecrawl extractor requests (#39831)

Chester074811106dc8
diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py
index 556158cf00a..77edffa77cd 100644
--- a/api/core/rag/extractor/firecrawl/firecrawl_app.py
+++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py
@@ -6,6 +6,10 @@ import httpx
from extensions.ext_storage import storage
+# Bounded connect/read timeout so a slow or hanging Firecrawl endpoint cannot
+# block extraction indefinitely (mirrors the WaterCrawl extractor client).
+_REQUEST_TIMEOUT = httpx.Timeout(30.0, connect=5.0)
+
class FirecrawlDocumentData(TypedDict):
title: str | None
@@ -176,7 +180,7 @@ class FirecrawlApp:
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
response: httpx.Response | None = None
for attempt in range(retries):
- response = httpx.post(url, headers=headers, json=data)
+ response = httpx.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT)
if response.status_code == 502:
time.sleep(backoff_factor * (2**attempt))
else:
@@ -187,7 +191,7 @@ class FirecrawlApp:
def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
response: httpx.Response | None = None
for attempt in range(retries):
- response = httpx.get(url, headers=headers)
+ response = httpx.get(url, headers=headers, timeout=_REQUEST_TIMEOUT)
if response.status_code == 502:
time.sleep(backoff_factor * (2**attempt))
else:
diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
index db49221583f..3bf1441c0af 100644
--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
@@ -35,6 +35,18 @@ class TestFirecrawlApp:
}
assert app._build_url("/v2/crawl") == "https://custom.firecrawl.dev/v2/crawl"
+ def test_requests_use_bounded_timeout(self, mocker: MockerFixture):
+ """Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""
+ app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
+ mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))
+ mock_get = mocker.patch("httpx.get", return_value=_response(200, {"status": "completed"}))
+
+ app._post_request("https://custom.firecrawl.dev/v1/crawl", {}, app._prepare_headers())
+ app._get_request("https://custom.firecrawl.dev/v1/crawl/job-1", app._prepare_headers())
+
+ assert mock_post.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT
+ assert mock_get.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT
+
def test_scrape_url_success(self, mocker: MockerFixture):
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch(