Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(watercrawl): handle non-json error responses (#37514)

落尘f8754286ec07
diff --git a/api/core/rag/extractor/watercrawl/client.py b/api/core/rag/extractor/watercrawl/client.py
index 51dd26b1ad0..b069977de9b 100644
--- a/api/core/rag/extractor/watercrawl/client.py
+++ b/api/core/rag/extractor/watercrawl/client.py
@@ -131,7 +131,10 @@ class WaterCrawlAPIClient(BaseAPIClient):
content_type = response.headers.get("Content-Type", "")
media_type = content_type.split(";", 1)[0].strip().lower()
if media_type == "application/json":
- return response.json() or {}
+ try:
+ return response.json() or {}
+ except ValueError as exc:
+ raise ValueError("Invalid JSON response from WaterCrawl") from exc
if media_type == "application/octet-stream":
return response.content
diff --git a/api/core/rag/extractor/watercrawl/exceptions.py b/api/core/rag/extractor/watercrawl/exceptions.py
index fc457697a22..e5b38f8acfe 100644
--- a/api/core/rag/extractor/watercrawl/exceptions.py
+++ b/api/core/rag/extractor/watercrawl/exceptions.py
@@ -1,5 +1,15 @@
+"""WaterCrawl domain exceptions.
+
+These exceptions are constructed from upstream HTTP responses, which may be
+JSON API errors or plain text/HTML proxy errors. Keep the exception type stable
+even when the body is not JSON so callers can handle WaterCrawl failures by
+domain type instead of low-level parser errors.
+"""
+
import json
-from typing import override
+from typing import Any, override
+
+from httpx import Response
class WaterCrawlError(Exception):
@@ -7,11 +17,16 @@ class WaterCrawlError(Exception):
class WaterCrawlBadRequestError(WaterCrawlError):
- def __init__(self, response):
+ def __init__(self, response: Response):
self.status_code = response.status_code
self.response = response
- data = response.json()
- self.message = data.get("message", "Unknown error occurred")
+ try:
+ data: Any = response.json()
+ except ValueError:
+ data = {}
+ if not isinstance(data, dict):
+ data = {}
+ self.message = data.get("message") or response.text or "Unknown error occurred"
self.errors = data.get("errors", {})
super().__init__(self.message)
diff --git a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
index 673f1e41c5d..bfb3754a7fd 100644
--- a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
+++ b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
@@ -176,6 +176,22 @@ class TestWaterCrawlAPIClient:
with pytest.raises(expected_exception):
client.process_response(_response(status, {"message": "bad", "errors": {"url": ["x"]}}))
+ @pytest.mark.parametrize(
+ ("status", "expected_exception"),
+ [
+ (401, WaterCrawlAuthenticationError),
+ (403, WaterCrawlPermissionError),
+ (422, WaterCrawlBadRequestError),
+ ],
+ )
+ def test_process_response_error_statuses_with_non_json_body(self, status: int, expected_exception: type[Exception]):
+ client = WaterCrawlAPIClient(api_key="k")
+ response = _response(status, text="<html>upstream error</html>")
+ response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0)
+
+ with pytest.raises(expected_exception):
+ client.process_response(response)
+
def test_process_response_204_returns_none(self):
client = WaterCrawlAPIClient(api_key="k")
assert client.process_response(_response(204, None)) is None
@@ -185,6 +201,14 @@ class TestWaterCrawlAPIClient:
assert client.process_response(_response(200, {"ok": True})) == {"ok": True}
assert client.process_response(_response(200, None)) == {}
+ def test_process_response_json_payload_with_invalid_body_raises_clear_error(self):
+ client = WaterCrawlAPIClient(api_key="k")
+ response = _response(200, text="<html>upstream error</html>")
+ response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0)
+
+ with pytest.raises(ValueError, match="Invalid JSON response from WaterCrawl"):
+ client.process_response(response)
+
def test_process_response_accepts_json_content_type_parameters(self):
client = WaterCrawlAPIClient(api_key="k")