Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): bound JinaReader and adaptive-crawl HTTP timeouts (#39859) (#39860)
Harsh Kashyap02a9c51a3e9b
diff --git a/api/services/website_service.py b/api/services/website_service.pyindex ea584088bbc..b833c146216 100644--- a/api/services/website_service.py+++ b/api/services/website_service.py@@ -17,13 +17,22 @@ from extensions.ext_storage import storagefrom services.datasource_provider_service import DatasourceProviderService# Reuse pooled HTTP clients to avoid creating new connections per request and ease testing.+# Both clients carry a bounded read/connect timeout so a stalled Jina or+# adaptive-crawl endpoint fails fast instead of pinning a worker. The values+# match the floor used in the WaterCrawl PR (#37512). See #39859._jina_http_client: httpx.Client = get_pooled_http_client("website:jinareader",- lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)),+ lambda: httpx.Client(+ timeout=httpx.Timeout(30.0, connect=5.0),+ limits=httpx.Limits(max_keepalive_connections=50, max_connections=100),+ ),)_adaptive_http_client: httpx.Client = get_pooled_http_client("website:adaptivecrawl",- lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)),+ lambda: httpx.Client(+ timeout=httpx.Timeout(30.0, connect=5.0),+ limits=httpx.Limits(max_keepalive_connections=50, max_connections=100),+ ),)diff --git a/api/tests/unit_tests/services/test_website_service.py b/api/tests/unit_tests/services/test_website_service.pyindex 2024aec13a3..571e8e99903 100644--- a/api/tests/unit_tests/services/test_website_service.py+++ b/api/tests/unit_tests/services/test_website_service.py@@ -724,3 +724,23 @@ def test_scrape_with_watercrawl_calls_provider(monkeypatch: pytest.MonkeyPatch))assert result == {"markdown": "m"}provider_instance.scrape_url.assert_called_once_with("u")+++def test_pooled_clients_carry_bounded_timeouts() -> None:+ """Regression for #39859: the Jina and adaptive-crawl pooled clients+ must carry a read/connect timeout so a stalled endpoint fails fast+ instead of pinning a worker. Same shape as the WaterCrawl hardening+ that landed in PR #37512.+ """+ jina = website_service_module._jina_http_client+ adaptive = website_service_module._adaptive_http_client++ # Read and connect bounds are set.+ assert jina.timeout is not None+ assert adaptive.timeout is not None++ # The values match the documented floor (read 30.0 / connect 5.0).+ assert jina.timeout.read == 30.0+ assert jina.timeout.connect == 5.0+ assert adaptive.timeout.read == 30.0+ assert adaptive.timeout.connect == 5.0