Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use real Watercrawl and Jina responses (#43381)

Asuka Minato0753e6480b09
diff --git a/api/tests/unit_tests/services/auth/test_jina_auth.py b/api/tests/unit_tests/services/auth/test_jina_auth.py
index 507edc466b0..5f43c55036a 100644
--- a/api/tests/unit_tests/services/auth/test_jina_auth.py
+++ b/api/tests/unit_tests/services/auth/test_jina_auth.py
@@ -1,4 +1,3 @@
-import json
from unittest.mock import MagicMock, patch
import httpx
@@ -44,9 +43,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_validate_valid_credentials_successfully(self, mock_post: MagicMock):
"""Test successful credential validation"""
- mock_response = MagicMock()
- mock_response.status_code = 200
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(200)
credentials = _credentials()
auth = JinaAuth(credentials)
@@ -62,10 +59,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_handle_http_402_error(self, mock_post: MagicMock):
"""Test handling of 402 Payment Required error"""
- mock_response = MagicMock()
- mock_response.status_code = 402
- mock_response.json.return_value = {"error": "Payment required"}
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(402, json={"error": "Payment required"})
credentials = _credentials()
auth = JinaAuth(credentials)
@@ -77,11 +71,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_handle_http_error_with_non_json_text_response(self, mock_post):
"""Test handling of known HTTP errors with non-JSON text response."""
- mock_response = MagicMock()
- mock_response.status_code = 402
- mock_response.text = "Payment required"
- mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0)
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(402, text="Payment required")
credentials = _credentials()
auth = JinaAuth(credentials)
@@ -93,10 +83,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_handle_http_409_error(self, mock_post):
"""Test handling of 409 Conflict error"""
- mock_response = MagicMock()
- mock_response.status_code = 409
- mock_response.json.return_value = {"error": "Conflict error"}
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(409, json={"error": "Conflict error"})
credentials = _credentials()
auth = JinaAuth(credentials)
@@ -112,8 +99,7 @@ class TestJinaAuth:
mock_post: MagicMock,
status_code: int,
):
- mock_response = MagicMock(status_code=status_code)
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(status_code)
auth = JinaAuth(_credentials())
@@ -126,11 +112,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_handle_unexpected_error_with_text_response(self, mock_post: MagicMock):
"""Test handling of unexpected errors with text response"""
- mock_response = MagicMock()
- mock_response.status_code = 403
- mock_response.text = '{"error": "Forbidden"}'
- mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0)
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(403, json={"error": "Forbidden"})
credentials = _credentials()
auth = JinaAuth(credentials)
@@ -142,11 +124,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_handle_unexpected_error_with_non_json_text_response(self, mock_post):
"""Test handling of unexpected errors with non-JSON text response."""
- mock_response = MagicMock()
- mock_response.status_code = 403
- mock_response.text = "Forbidden"
- mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0)
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(403, text="Forbidden")
credentials = _credentials()
auth = JinaAuth(credentials)
@@ -158,11 +136,7 @@ class TestJinaAuth:
@patch("services.data_source.auth.jina.jina._http_client.post", autospec=True)
def test_should_handle_unexpected_error_without_text(self, mock_post):
"""Test handling of unexpected errors without text response"""
- mock_response = MagicMock()
- mock_response.status_code = 404
- mock_response.text = ""
- mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0)
- mock_post.return_value = mock_response
+ mock_post.return_value = httpx.Response(404)
credentials = _credentials()
auth = JinaAuth(credentials)
diff --git a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
index e54df0dee9e..a9205c88931 100644
--- a/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
+++ b/api/tests/unit_tests/services/auth/test_watercrawl_auth.py
@@ -1,4 +1,3 @@
-import json
from unittest.mock import MagicMock, patch
import httpx
@@ -67,9 +66,7 @@ class TestWatercrawlAuth:
@patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
def test_should_validate_valid_credentials_successfully(self, mock_get, auth_instance):
"""Test successful credential validation"""
- mock_response = MagicMock()
- mock_response.status_code = 200
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(200)
result = auth_instance.validate_credentials()
@@ -90,10 +87,7 @@ class TestWatercrawlAuth:
@patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
def test_should_handle_http_errors(self, mock_get, status_code, error_message, auth_instance):
"""Test handling of various HTTP error codes"""
- mock_response = MagicMock()
- mock_response.status_code = status_code
- mock_response.json.return_value = {"error": error_message}
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(status_code, json={"error": error_message})
with pytest.raises(DataSourceApiKeyAuthCredentialValidationError) as exc_info:
auth_instance.validate_credentials()
@@ -107,8 +101,7 @@ class TestWatercrawlAuth:
status_code: int,
auth_instance: WatercrawlAuth,
):
- mock_response = MagicMock(status_code=status_code)
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(status_code)
with pytest.raises(DataSourceApiKeyAuthProviderUnavailableError) as exc_info:
auth_instance.validate_credentials()
@@ -119,35 +112,26 @@ class TestWatercrawlAuth:
@patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
def test_should_handle_http_error_with_non_json_text_response(self, mock_get, auth_instance):
"""Test handling of known HTTP errors with non-JSON text response."""
- mock_response = MagicMock()
- mock_response.status_code = 402
- mock_response.text = "Payment required"
- mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0)
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(402, text="Payment required")
with pytest.raises(DataSourceApiKeyAuthCredentialValidationError) as exc_info:
auth_instance.validate_credentials()
assert str(exc_info.value) == "Failed to authorize. Status code: 402. Error: Payment required"
@pytest.mark.parametrize(
- ("status_code", "response_text", "has_json_error", "expected_error_contains"),
+ ("status_code", "response_text", "expected_error_contains"),
[
- (403, '{"error": "Forbidden"}', True, "Failed to authorize. Status code: 403. Error: Forbidden"),
- (404, "", True, "Unexpected error occurred while trying to authorize. Status code: 404"),
- (401, "Not JSON", True, "Failed to authorize. Status code: 401. Error: Not JSON"),
+ (403, '{"error": "Forbidden"}', "Failed to authorize. Status code: 403. Error: Forbidden"),
+ (404, "", "Unexpected error occurred while trying to authorize. Status code: 404"),
+ (401, "Not JSON", "Failed to authorize. Status code: 401. Error: Not JSON"),
],
)
@patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
def test_should_handle_unexpected_errors(
- self, mock_get, status_code, response_text, has_json_error, expected_error_contains, auth_instance
+ self, mock_get, status_code, response_text, expected_error_contains, auth_instance
):
"""Test handling of unexpected errors with various response formats"""
- mock_response = MagicMock()
- mock_response.status_code = status_code
- mock_response.text = response_text
- if has_json_error:
- mock_response.json.side_effect = json.JSONDecodeError("Not JSON", "", 0)
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(status_code, text=response_text)
with pytest.raises(DataSourceApiKeyAuthCredentialValidationError) as exc_info:
auth_instance.validate_credentials()
@@ -187,9 +171,7 @@ class TestWatercrawlAuth:
@patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
def test_should_use_custom_base_url_in_validation(self, mock_get):
"""Test that custom base URL is used in validation"""
- mock_response = MagicMock()
- mock_response.status_code = 200
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(200)
credentials = _credentials(base_url="https://custom.watercrawl.dev")
auth = WatercrawlAuth(credentials)
@@ -209,9 +191,7 @@ class TestWatercrawlAuth:
@patch("services.data_source.auth.watercrawl.watercrawl.httpx.get", autospec=True)
def test_should_use_urljoin_for_url_construction(self, mock_get, base_url, expected_url):
"""Test that urljoin is used correctly for URL construction with various base URLs"""
- mock_response = MagicMock()
- mock_response.status_code = 200
- mock_get.return_value = mock_response
+ mock_get.return_value = httpx.Response(200)
credentials = _credentials(base_url=base_url)
auth = WatercrawlAuth(credentials)