Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: reject 4xx/5xx responses in download_with_size_limit instead of returning the error page (#44333)

PerryLinkdca4024e5777
diff --git a/api/core/helper/download.py b/api/core/helper/download.py
index e99be256a01..3143077422a 100644
--- a/api/core/helper/download.py
+++ b/api/core/helper/download.py
@@ -3,10 +3,25 @@ from typing import Any
def download_with_size_limit(url: str, max_download_size: int, **kwargs: Any) -> bytes:
from core.file import remote_fetcher
+ from core.helper.ssrf_proxy import MaxRetriesExceededError
- response = remote_fetcher.make_request("GET", url, follow_redirects=True, **kwargs)
+ try:
+ response = remote_fetcher.make_request("GET", url, follow_redirects=True, **kwargs)
+ except MaxRetriesExceededError as e:
+ if e.status_code is None:
+ # No response was ever received: keep the transport error as is
+ # instead of inventing a status code.
+ raise
+ # Retries ran out on an explicit error response. Report it through the
+ # same message as the non-retried path so the controller mapping shows
+ # the upstream status.
+ raise ValueError(f"Failed to download file: HTTP {e.status_code}") from e
if response.status_code == 404:
raise ValueError("file not found")
+ if response.status_code >= 400:
+ # The body of a 4xx/5xx response is an error page, not the requested
+ # file. Returning it would hand callers an error document as content.
+ raise ValueError(f"Failed to download file: HTTP {response.status_code}")
total_size = 0
chunks = []
diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py
index 3af7c9d8a85..5cafeef18e0 100644
--- a/api/core/helper/ssrf_proxy.py
+++ b/api/core/helper/ssrf_proxy.py
@@ -48,9 +48,16 @@ _SSRF_CLIENT_LIMITS = httpx.Limits(
class MaxRetriesExceededError(ValueError):
- """Raised when the maximum number of retries is exceeded."""
+ """Raised when the maximum number of retries is exceeded.
- pass
+ 'status_code' is the status of the last response that was received, so a
+ caller can report what the upstream actually answered. It stays None when
+ every attempt failed before a response arrived.
+ """
+
+ def __init__(self, message: str, status_code: int | None = None) -> None:
+ super().__init__(message)
+ self.status_code = status_code
class ResponseLimitError(ValueError):
@@ -272,6 +279,7 @@ def make_request(
send_kwargs["follow_redirects"] = kwargs.pop("follow_redirects")
retries = 0
+ last_status: int | None = None
last_request_error: httpx.RequestError | None = None
while retries <= max_retries:
try:
@@ -289,6 +297,8 @@ def make_request(
response = client.send(request, **send_kwargs)
last_request_error = None
+ last_status = response.status_code
+
# Check for SSRF protection by Squid proxy
if response.status_code in (401, 403):
# Only a response that Squid generated itself proves that the
@@ -340,7 +350,10 @@ def make_request(
retries += 1
if retries <= max_retries:
time.sleep(BACKOFF_FACTOR * (2 ** (retries - 1)))
- error = MaxRetriesExceededError(f"Reached maximum retries ({max_retries}) for URL {url}")
+ error = MaxRetriesExceededError(
+ f"Reached maximum retries ({max_retries}) for URL {url}",
+ status_code=last_status,
+ )
if last_request_error is not None:
raise error from last_request_error
raise error
diff --git a/api/core/plugin/plugin_service.py b/api/core/plugin/plugin_service.py
index 682cb28793e..37cc38e38cd 100644
--- a/api/core/plugin/plugin_service.py
+++ b/api/core/plugin/plugin_service.py
@@ -59,6 +59,7 @@ from core.plugin.entities.plugin_daemon import (
from core.plugin.impl.asset import PluginAssetManager
from core.plugin.impl.debugging import PluginDebuggingClient
from core.plugin.impl.endpoint import PluginEndpointClient
+from core.plugin.impl.exc import PluginDaemonClientSideError
from core.plugin.impl.model import PluginModelClient
from core.plugin.impl.plugin import PluginInstaller
from enums import DeploymentEdition
@@ -1069,7 +1070,7 @@ class PluginService:
manager.fetch_plugin_manifest(tenant_id, new_plugin_unique_identifier)
except Exception:
# plugin not downloaded yet, download and upload pkg
- pkg = download_plugin_pkg(new_plugin_unique_identifier)
+ pkg = PluginService._download_plugin_pkg(new_plugin_unique_identifier)
response = manager.upload_pkg(
tenant_id,
pkg,
@@ -1143,6 +1144,16 @@ class PluginService:
return response
+ @staticmethod
+ def _download_plugin_pkg(plugin_unique_identifier: str) -> bytes:
+ """Download a marketplace package, surfacing HTTP failures as client errors."""
+ try:
+ return download_plugin_pkg(plugin_unique_identifier)
+ except ValueError as e:
+ # download_with_size_limit() rejects 4xx/5xx responses. Let the controller translate
+ # the status to the console instead of reporting a generic plugin operation failure.
+ raise PluginDaemonClientSideError(str(e)) from e
+
@staticmethod
def upload_pkg_from_github(
tenant_id: str, repo: str, version: str, package: str, verify_signature: bool = False
@@ -1152,9 +1163,13 @@ class PluginService:
returns plugin_unique_identifier
"""
PluginService._check_marketplace_only_permission()
- pkg = download_with_size_limit(
- f"https://github.com/{repo}/releases/download/{version}/{package}", dify_config.PLUGIN_MAX_PACKAGE_SIZE
- )
+ try:
+ pkg = download_with_size_limit(
+ f"https://github.com/{repo}/releases/download/{version}/{package}",
+ dify_config.PLUGIN_MAX_PACKAGE_SIZE,
+ )
+ except ValueError as e:
+ raise PluginDaemonClientSideError(str(e)) from e
permission = PluginService._get_plugin_installation_permission()
manager = PluginInstaller()
@@ -1241,7 +1256,7 @@ class PluginService:
try:
declaration = manager.fetch_plugin_manifest(tenant_id, plugin_unique_identifier)
except Exception:
- pkg = download_plugin_pkg(plugin_unique_identifier)
+ pkg = PluginService._download_plugin_pkg(plugin_unique_identifier)
response = manager.upload_pkg(
tenant_id,
pkg,
@@ -1275,7 +1290,7 @@ class PluginService:
manager.fetch_plugin_manifest(tenant_id, plugin_unique_identifier)
except Exception:
# plugin not downloaded yet, download and upload pkg
- pkg = download_plugin_pkg(plugin_unique_identifier)
+ pkg = PluginService._download_plugin_pkg(plugin_unique_identifier)
response = manager.upload_pkg(
tenant_id,
pkg,
diff --git a/api/tests/unit_tests/controllers/console/workspace/test_plugin.py b/api/tests/unit_tests/controllers/console/workspace/test_plugin.py
index 0b27fa22601..364b640d93f 100644
--- a/api/tests/unit_tests/controllers/console/workspace/test_plugin.py
+++ b/api/tests/unit_tests/controllers/console/workspace/test_plugin.py
@@ -900,6 +900,22 @@ class TestPluginUploadFromGithubApi:
result = raised_error(lambda: method(api, ParserGithubUpload.model_validate(payload), "t1"))
assert result == ({"code": "plugin_error", "message": "error"}, 400)
+ def test_download_error_reports_status(self, app: Flask):
+ api = PluginUploadFromGithubApi()
+ method = unwrap(api.post)
+
+ payload = {"repo": "r", "version": "v", "package": "p"}
+
+ with (
+ app.test_request_context("/", json=payload),
+ patch(
+ "controllers.console.workspace.plugin.PluginService.upload_pkg_from_github",
+ side_effect=PluginDaemonClientSideError("Failed to download file: HTTP 403"),
+ ),
+ ):
+ result = raised_error(lambda: method(api, ParserGithubUpload.model_validate(payload), "t1"))
+ assert result == ({"code": "plugin_error", "message": "Failed to download file: HTTP 403"}, 400)
+
class TestPluginUploadFromBundleApi:
def test_success(self, app: Flask):
diff --git a/api/tests/unit_tests/core/helper/test_download.py b/api/tests/unit_tests/core/helper/test_download.py
index b293b6fbc34..3db0c77bf0f 100644
--- a/api/tests/unit_tests/core/helper/test_download.py
+++ b/api/tests/unit_tests/core/helper/test_download.py
@@ -4,14 +4,17 @@ import pytest
from pytest_mock import MockerFixture
from core.helper.download import download_with_size_limit
+from core.helper.ssrf_proxy import MaxRetriesExceededError
class _StubResponse:
def __init__(self, status_code: int, chunks: list[bytes]) -> None:
self.status_code = status_code
+ self.iter_bytes_called = False
self._chunks = chunks
def iter_bytes(self) -> Iterator[bytes]:
+ self.iter_bytes_called = True
return iter(self._chunks)
@@ -26,13 +29,45 @@ def test_download_with_size_limit_returns_content(mocker: MockerFixture) -> None
def test_download_with_size_limit_raises_for_404(mocker: MockerFixture) -> None:
+ response = _StubResponse(status_code=404, chunks=[])
+ mocker.patch("core.file.remote_fetcher.make_request", return_value=response)
+
+ with pytest.raises(ValueError, match="file not found"):
+ download_with_size_limit("https://example.com/missing.txt", max_download_size=10)
+
+ assert not response.iter_bytes_called
+
+
[email protected]("status_code", [403, 429, 500, 503])
+def test_download_with_size_limit_raises_for_error_status(mocker: MockerFixture, status_code: int) -> None:
+ response = _StubResponse(status_code=status_code, chunks=[b"<html>error page</html>"])
+ mocker.patch("core.file.remote_fetcher.make_request", return_value=response)
+
+ with pytest.raises(ValueError, match=f"HTTP {status_code}"):
+ download_with_size_limit("https://marketplace.example.com/api/v1/plugins/download-url", max_download_size=1024)
+
+ assert not response.iter_bytes_called
+
+
[email protected]("status_code", [429, 503])
+def test_download_with_size_limit_reports_an_exhausted_status(mocker: MockerFixture, status_code: int) -> None:
mocker.patch(
"core.file.remote_fetcher.make_request",
- return_value=_StubResponse(status_code=404, chunks=[]),
+ side_effect=MaxRetriesExceededError("Reached maximum retries (3) for URL u", status_code=status_code),
)
- with pytest.raises(ValueError, match="file not found"):
- download_with_size_limit("https://example.com/missing.txt", max_download_size=10)
+ with pytest.raises(ValueError, match=f"HTTP {status_code}"):
+ download_with_size_limit("https://marketplace.example.com/api/v1/plugins/download-url", max_download_size=1024)
+
+
+def test_download_with_size_limit_keeps_a_transport_failure(mocker: MockerFixture) -> None:
+ mocker.patch(
+ "core.file.remote_fetcher.make_request",
+ side_effect=MaxRetriesExceededError("Reached maximum retries (3) for URL u"),
+ )
+
+ with pytest.raises(MaxRetriesExceededError):
+ download_with_size_limit("https://marketplace.example.com/api/v1/plugins/download-url", max_download_size=1024)
def test_download_with_size_limit_raises_when_size_exceeds_limit(
diff --git a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py
index 3755c3857a8..160e2dba311 100644
--- a/api/tests/unit_tests/core/helper/test_ssrf_proxy.py
+++ b/api/tests/unit_tests/core/helper/test_ssrf_proxy.py
@@ -180,6 +180,59 @@ def test_retry_exhaustion_preserves_the_last_request_error(monkeypatch: pytest.M
assert sleep_calls == [BACKOFF_FACTOR]
+@patch("core.helper.ssrf_proxy._get_ssrf_client", autospec=True)
+def test_retry_exhaustion_keeps_the_last_status(mock_get_client, client_factory, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr("core.helper.ssrf_proxy.time.sleep", lambda _seconds: None)
+ mock_response = httpx.Response(503)
+ mock_client, send_calls = client_factory(mock_response)
+ mock_get_client.return_value = mock_client
+
+ with pytest.raises(max_retries_exceeded_error) as exc_info:
+ make_request("GET", "http://example.com", max_retries=2)
+
+ assert exc_info.value.status_code == 503
+ assert len(send_calls) == 3
+
+
+@patch("core.helper.ssrf_proxy._get_ssrf_client", autospec=True)
+def test_retry_exhaustion_without_a_response_keeps_no_status(mock_get_client, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr("core.helper.ssrf_proxy.time.sleep", lambda _seconds: None)
+ sends = 0
+
+ def refuse(request: httpx.Request) -> httpx.Response:
+ nonlocal sends
+ sends += 1
+ raise httpx.ConnectError("connection refused", request=request)
+
+ with httpx.Client(transport=httpx.MockTransport(refuse)) as client:
+ mock_get_client.return_value = client
+ with pytest.raises(max_retries_exceeded_error) as exc_info:
+ make_request("GET", "http://example.com", max_retries=2)
+
+ assert exc_info.value.status_code is None
+ assert sends == 3
+
+
+@patch("core.helper.ssrf_proxy._get_ssrf_client", autospec=True)
+def test_retry_then_success_returns_the_later_response(mock_get_client, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr("core.helper.ssrf_proxy.time.sleep", lambda _seconds: None)
+ responses = iter([httpx.Response(503), httpx.Response(200, content=b"ok")])
+ sends = 0
+
+ def answer(_request: httpx.Request) -> httpx.Response:
+ nonlocal sends
+ sends += 1
+ return next(responses)
+
+ with httpx.Client(transport=httpx.MockTransport(answer)) as client:
+ mock_get_client.return_value = client
+ response = make_request("GET", "http://example.com", max_retries=2)
+
+ assert response.status_code == 200
+ assert response.content == b"ok"
+ assert sends == 2
+
+
@patch("core.helper.ssrf_proxy._get_ssrf_client", autospec=True)
def test_force_list_response_returns_when_retries_disabled(mock_get_client, client_factory):
mock_response = httpx.Response(500)
diff --git a/api/tests/unit_tests/core/plugin/test_plugin_service_download_errors.py b/api/tests/unit_tests/core/plugin/test_plugin_service_download_errors.py
new file mode 100644
index 00000000000..7cc81e99e55
--- /dev/null
+++ b/api/tests/unit_tests/core/plugin/test_plugin_service_download_errors.py
@@ -0,0 +1,29 @@
+import pytest
+from pytest_mock import MockerFixture
+
+from core.plugin.impl.exc import PluginDaemonClientSideError
+from core.plugin.plugin_service import PluginService
+
+
+def test_upload_pkg_from_github_maps_download_failure(mocker: MockerFixture) -> None:
+ mocker.patch.object(PluginService, "_check_marketplace_only_permission")
+ mocker.patch(
+ "core.plugin.plugin_service.download_with_size_limit",
+ side_effect=ValueError("Failed to download file: HTTP 403"),
+ )
+ installer = mocker.patch("core.plugin.plugin_service.PluginInstaller")
+
+ with pytest.raises(PluginDaemonClientSideError, match="HTTP 403"):
+ PluginService.upload_pkg_from_github("tenant-id", "owner/repo", "v1.0.0", "plugin.difypkg")
+
+ installer.assert_not_called()
+
+
+def test_download_plugin_pkg_maps_download_failure(mocker: MockerFixture) -> None:
+ mocker.patch(
+ "core.plugin.plugin_service.download_plugin_pkg",
+ side_effect=ValueError("Failed to download file: HTTP 429"),
+ )
+
+ with pytest.raises(PluginDaemonClientSideError, match="HTTP 429"):
+ PluginService._download_plugin_pkg("langgenius-agent:0.0.1@abcdef")