Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(rag): add bounded timeout to Firecrawl extractor requests (#39831)
Chester074811106dc8
diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.pyindex 556158cf00a..77edffa77cd 100644--- a/api/core/rag/extractor/firecrawl/firecrawl_app.py+++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py@@ -6,6 +6,10 @@ import httpxfrom extensions.ext_storage import storage+# Bounded connect/read timeout so a slow or hanging Firecrawl endpoint cannot+# block extraction indefinitely (mirrors the WaterCrawl extractor client).+_REQUEST_TIMEOUT = httpx.Timeout(30.0, connect=5.0)+class FirecrawlDocumentData(TypedDict):title: str | None@@ -176,7 +180,7 @@ class FirecrawlApp:def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response:response: httpx.Response | None = Nonefor attempt in range(retries):- response = httpx.post(url, headers=headers, json=data)+ response = httpx.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT)if response.status_code == 502:time.sleep(backoff_factor * (2**attempt))else:@@ -187,7 +191,7 @@ class FirecrawlApp:def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response:response: httpx.Response | None = Nonefor attempt in range(retries):- response = httpx.get(url, headers=headers)+ response = httpx.get(url, headers=headers, timeout=_REQUEST_TIMEOUT)if response.status_code == 502:time.sleep(backoff_factor * (2**attempt))else:diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.pyindex db49221583f..3bf1441c0af 100644--- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py@@ -35,6 +35,18 @@ class TestFirecrawlApp:}assert app._build_url("/v2/crawl") == "https://custom.firecrawl.dev/v2/crawl"+ def test_requests_use_bounded_timeout(self, mocker: MockerFixture):+ """Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""+ app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")+ mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))+ mock_get = mocker.patch("httpx.get", return_value=_response(200, {"status": "completed"}))++ app._post_request("https://custom.firecrawl.dev/v1/crawl", {}, app._prepare_headers())+ app._get_request("https://custom.firecrawl.dev/v1/crawl/job-1", app._prepare_headers())++ assert mock_post.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT+ assert mock_get.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT+def test_scrape_url_success(self, mocker: MockerFixture):app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")mocker.patch(