Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): bound JinaReader and adaptive-crawl HTTP timeouts (#39859) (#39860)

Harsh Kashyap02a9c51a3e9b
diff --git a/api/services/website_service.py b/api/services/website_service.py
index ea584088bbc..b833c146216 100644
--- a/api/services/website_service.py
+++ b/api/services/website_service.py
@@ -17,13 +17,22 @@ from extensions.ext_storage import storage
from services.datasource_provider_service import DatasourceProviderService
# Reuse pooled HTTP clients to avoid creating new connections per request and ease testing.
+# Both clients carry a bounded read/connect timeout so a stalled Jina or
+# adaptive-crawl endpoint fails fast instead of pinning a worker. The values
+# match the floor used in the WaterCrawl PR (#37512). See #39859.
_jina_http_client: httpx.Client = get_pooled_http_client(
"website:jinareader",
- lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)),
+ lambda: httpx.Client(
+ timeout=httpx.Timeout(30.0, connect=5.0),
+ limits=httpx.Limits(max_keepalive_connections=50, max_connections=100),
+ ),
)
_adaptive_http_client: httpx.Client = get_pooled_http_client(
"website:adaptivecrawl",
- lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)),
+ lambda: httpx.Client(
+ timeout=httpx.Timeout(30.0, connect=5.0),
+ limits=httpx.Limits(max_keepalive_connections=50, max_connections=100),
+ ),
)
diff --git a/api/tests/unit_tests/services/test_website_service.py b/api/tests/unit_tests/services/test_website_service.py
index 2024aec13a3..571e8e99903 100644
--- a/api/tests/unit_tests/services/test_website_service.py
+++ b/api/tests/unit_tests/services/test_website_service.py
@@ -724,3 +724,23 @@ def test_scrape_with_watercrawl_calls_provider(monkeypatch: pytest.MonkeyPatch)
)
assert result == {"markdown": "m"}
provider_instance.scrape_url.assert_called_once_with("u")
+
+
+def test_pooled_clients_carry_bounded_timeouts() -> None:
+ """Regression for #39859: the Jina and adaptive-crawl pooled clients
+ must carry a read/connect timeout so a stalled endpoint fails fast
+ instead of pinning a worker. Same shape as the WaterCrawl hardening
+ that landed in PR #37512.
+ """
+ jina = website_service_module._jina_http_client
+ adaptive = website_service_module._adaptive_http_client
+
+ # Read and connect bounds are set.
+ assert jina.timeout is not None
+ assert adaptive.timeout is not None
+
+ # The values match the documented floor (read 30.0 / connect 5.0).
+ assert jina.timeout.read == 30.0
+ assert jina.timeout.connect == 5.0
+ assert adaptive.timeout.read == 30.0
+ assert adaptive.timeout.connect == 5.0