Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(watercrawl): handle non-json error responses (#37514)
落尘f8754286ec07
diff --git a/api/core/rag/extractor/watercrawl/client.py b/api/core/rag/extractor/watercrawl/client.pyindex 51dd26b1ad0..b069977de9b 100644--- a/api/core/rag/extractor/watercrawl/client.py+++ b/api/core/rag/extractor/watercrawl/client.py@@ -131,7 +131,10 @@ class WaterCrawlAPIClient(BaseAPIClient):content_type = response.headers.get("Content-Type", "")media_type = content_type.split(";", 1)[0].strip().lower()if media_type == "application/json":- return response.json() or {}+ try:+ return response.json() or {}+ except ValueError as exc:+ raise ValueError("Invalid JSON response from WaterCrawl") from excif media_type == "application/octet-stream":return response.contentdiff --git a/api/core/rag/extractor/watercrawl/exceptions.py b/api/core/rag/extractor/watercrawl/exceptions.pyindex fc457697a22..e5b38f8acfe 100644--- a/api/core/rag/extractor/watercrawl/exceptions.py+++ b/api/core/rag/extractor/watercrawl/exceptions.py@@ -1,5 +1,15 @@+"""WaterCrawl domain exceptions.++These exceptions are constructed from upstream HTTP responses, which may be+JSON API errors or plain text/HTML proxy errors. Keep the exception type stable+even when the body is not JSON so callers can handle WaterCrawl failures by+domain type instead of low-level parser errors.+"""+import json-from typing import override+from typing import Any, override++from httpx import Responseclass WaterCrawlError(Exception):@@ -7,11 +17,16 @@ class WaterCrawlError(Exception):class WaterCrawlBadRequestError(WaterCrawlError):- def __init__(self, response):+ def __init__(self, response: Response):self.status_code = response.status_codeself.response = response- data = response.json()- self.message = data.get("message", "Unknown error occurred")+ try:+ data: Any = response.json()+ except ValueError:+ data = {}+ if not isinstance(data, dict):+ data = {}+ self.message = data.get("message") or response.text or "Unknown error occurred"self.errors = data.get("errors", {})super().__init__(self.message)diff --git a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.pyindex 673f1e41c5d..bfb3754a7fd 100644--- a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py+++ b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py@@ -176,6 +176,22 @@ class TestWaterCrawlAPIClient:with pytest.raises(expected_exception):client.process_response(_response(status, {"message": "bad", "errors": {"url": ["x"]}}))+ @pytest.mark.parametrize(+ ("status", "expected_exception"),+ [+ (401, WaterCrawlAuthenticationError),+ (403, WaterCrawlPermissionError),+ (422, WaterCrawlBadRequestError),+ ],+ )+ def test_process_response_error_statuses_with_non_json_body(self, status: int, expected_exception: type[Exception]):+ client = WaterCrawlAPIClient(api_key="k")+ response = _response(status, text="<html>upstream error</html>")+ response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0)++ with pytest.raises(expected_exception):+ client.process_response(response)+def test_process_response_204_returns_none(self):client = WaterCrawlAPIClient(api_key="k")assert client.process_response(_response(204, None)) is None@@ -185,6 +201,14 @@ class TestWaterCrawlAPIClient:assert client.process_response(_response(200, {"ok": True})) == {"ok": True}assert client.process_response(_response(200, None)) == {}+ def test_process_response_json_payload_with_invalid_body_raises_clear_error(self):+ client = WaterCrawlAPIClient(api_key="k")+ response = _response(200, text="<html>upstream error</html>")+ response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0)++ with pytest.raises(ValueError, match="Invalid JSON response from WaterCrawl"):+ client.process_response(response)+def test_process_response_accepts_json_content_type_parameters(self):client = WaterCrawlAPIClient(api_key="k")