Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat(agent): support package imports from external URLs (#42425)

林玮 (Jade Lin)223039128f1b
diff --git a/api/controllers/console/app/app_import.py b/api/controllers/console/app/app_import.py
index 4872947b4dd..a2592c81300 100644
--- a/api/controllers/console/app/app_import.py
+++ b/api/controllers/console/app/app_import.py
@@ -36,6 +36,7 @@ from services.app_dsl_service import (
Import,
PendingData,
)
+from services.app_import_source import download_app_import_source, try_read_yaml
from services.enterprise.enterprise_service import EnterpriseService
from services.entities.dsl_entities import CheckDependenciesResult, ImportStatus
from services.errors.account import NoPermissionError
@@ -128,24 +129,45 @@ class AppImportApi(Resource):
def post(self, current_user: Account):
if request.mimetype == "multipart/form-data":
return self._import_package(current_user)
- return self._import_dsl(validate_request(AppImportPayload), current_user)
+ payload = validate_request(AppImportPayload)
+ if payload.mode == "yaml-url" and payload.yaml_url:
+ return self._import_url(payload, current_user)
+ return self._import_dsl(payload, current_user)
+
+ @rbac_permission_required(
+ RBACCheck(RBACPermission.APP_IMPORT_EXPORT_DSL, Workspace()),
+ RBACCheck(RBACPermission.AGENT_IMPORT_EXPORT_DSL, Workspace()),
+ )
+ def _import_url(self, payload: AppImportPayload, current_user: Account):
+ with download_app_import_source(payload.yaml_url or "") as source:
+ content = try_read_yaml(source)
+ if content is not None:
+ return self._import_dsl(
+ payload.model_copy(update={"mode": "yaml-content", "yaml_content": content, "yaml_url": None}),
+ current_user,
+ )
+ return self._import_package(current_user, payload, source)
@rbac_permission_required(RBACCheck(RBACPermission.AGENT_CREATE, Workspace()))
@rbac_permission_required(RBACCheck(RBACPermission.AGENT_IMPORT_EXPORT_DSL, Workspace()))
- def _import_package(self, current_user: Account):
- uploaded = request.files.get("file")
- if uploaded is None or not uploaded.filename:
- raise InvalidRosterAgentPackageError("Roster Agent package file is required")
- if not uploaded.filename.lower().endswith(".ifpkg"):
- raise InvalidRosterAgentPackageError("Roster Agent package file must use the .ifpkg extension")
- if request.form.get("app_id"):
+ def _import_package(
+ self, current_user: Account, payload: AppImportPayload | None = None, source: BinaryIO | None = None
+ ):
+ app_id = payload.app_id if payload is not None else request.form.get("app_id")
+ if app_id:
raise InvalidRosterAgentPackageError("Roster Agent package import does not support overwriting an App")
account, tenant_id = _current_user_and_tenant_id(current_user)
if tenant_id is None:
raise Forbidden("Current workspace is required")
- result = RosterAgentPackageImporter().import_package(
- source=cast(BinaryIO, uploaded.stream), tenant_id=tenant_id, account=account
- )
+ importer = RosterAgentPackageImporter()
+ if source is None:
+ uploaded = request.files.get("file")
+ if uploaded is None or not uploaded.filename:
+ raise InvalidRosterAgentPackageError("Roster Agent package file is required")
+ if not uploaded.filename.lower().endswith(".ifpkg"):
+ raise InvalidRosterAgentPackageError("Roster Agent package file must use the .ifpkg extension")
+ source = cast(BinaryIO, uploaded.stream)
+ result = importer.import_package(source=source, tenant_id=tenant_id, account=account)
return Import(
id=str(uuid4()),
status=ImportStatus.COMPLETED_WITH_WARNINGS if result.warnings else ImportStatus.COMPLETED,
diff --git a/api/services/app_import_source.py b/api/services/app_import_source.py
new file mode 100644
index 00000000000..2d3457f16a9
--- /dev/null
+++ b/api/services/app_import_source.py
@@ -0,0 +1,74 @@
+"""Download URL imports once and detect YAML before falling back to package validation."""
+
+import tempfile
+from collections.abc import Generator
+from contextlib import contextmanager
+from typing import BinaryIO, cast
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+import yaml
+
+from configs import dify_config
+from core.file import remote_fetcher
+from core.tools.errors import ToolSSRFError
+from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError
+from services.dsl_content import DSL_MAX_SIZE
+
+
+@contextmanager
+def download_app_import_source(url: str) -> Generator[BinaryIO]:
+ """Fetch a bounded source without opening import transactions or creating resources."""
+ url = url.strip()
+ try:
+ parsed = urlsplit(url)
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+ raise ValueError("An absolute HTTP(S) URL is required")
+ except ValueError as exc:
+ raise InvalidRosterAgentPackageError("App import URL is invalid") from exc
+ if parsed.scheme == "https" and parsed.netloc == "github.com" and "/blob/" in parsed.path:
+ url = urlunsplit(
+ parsed._replace(netloc="raw.githubusercontent.com", path=parsed.path.replace("/blob/", "/", 1))
+ )
+
+ with tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b") as source:
+ try:
+ response = remote_fetcher.make_request(
+ "GET",
+ url,
+ stream_response=True,
+ follow_redirects=True,
+ timeout=(10, 10),
+ headers={"Accept-Encoding": "identity"},
+ )
+ try:
+ response.raise_for_status()
+ # Reject HTTP compression so the byte limit also bounds decompression memory.
+ if response.headers.get("content-encoding", "identity").strip().lower() not in {"", "identity"}:
+ raise InvalidRosterAgentPackageError("Compressed HTTP import responses are not supported")
+ size = 0
+ for chunk in response.iter_bytes(chunk_size=1024 * 1024):
+ size += len(chunk)
+ if size > max(dify_config.AGENT_PACKAGE_MAX_BYTES, DSL_MAX_SIZE):
+ raise RosterAgentPackageTooLargeError("App import download exceeds the size limit")
+ source.write(chunk)
+ finally:
+ response.close()
+ except (httpx.HTTPError, httpx.InvalidURL, remote_fetcher.max_retries_exceeded_error, ToolSSRFError) as exc:
+ raise InvalidRosterAgentPackageError("Could not download the App import from URL") from exc
+ source.seek(0)
+ yield cast(BinaryIO, source)
+
+
+def try_read_yaml(source: BinaryIO) -> str | None:
+ """Recognize a YAML mapping without executing an import; always rewind for the package fallback."""
+ try:
+ raw = source.read(DSL_MAX_SIZE + 1)
+ if len(raw) > DSL_MAX_SIZE:
+ return None
+ content = raw.decode("utf-8")
+ return content if isinstance(yaml.safe_load(content), dict) else None
+ except (UnicodeDecodeError, yaml.YAMLError):
+ return None
+ finally:
+ source.seek(0)
diff --git a/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py b/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py
index bb8b9c8ab06..e8e13be6424 100644
--- a/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py
+++ b/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py
@@ -4,6 +4,7 @@ from inspect import unwrap
from types import SimpleNamespace
from unittest.mock import Mock
+import httpx
import pytest
from flask import Flask
from werkzeug.exceptions import BadRequest, Forbidden
@@ -13,11 +14,20 @@ from controllers.console.app import app_import as import_module
from core.rbac import RBACPermission
from models.account import Account, Tenant
from models.model import App, AppMode
+from services import app_import_source
from services.agent.errors import InvalidRosterAgentPackageError
from services.agent.roster_package_importer import RosterAgentPackageImportResult
from services.entities.dsl_entities import DslImportWarning
+def _mock_download(monkeypatch: pytest.MonkeyPatch, content: bytes = b"PK\x00\xff") -> Mock:
+ fetch = Mock(
+ return_value=httpx.Response(200, content=content, request=httpx.Request("GET", "https://example.com/download"))
+ )
+ monkeypatch.setattr(app_import_source.remote_fetcher, "make_request", fetch)
+ return fetch
+
+
def _account() -> Account:
account = Account(name="Importer", email="[email protected]")
account._current_tenant = Tenant(name="Workspace")
@@ -43,6 +53,136 @@ def test_existing_import_route_dispatches_json_without_changing_payload(
)
+ "url",
+ [
+ "https://example.com/download?id=123",
+ "https://example.com/agent.ifpkg",
+ " https://example.com/agent.IFPKG?token=secret#download ",
+ ],
+)
+def test_package_url_uses_agent_import(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], url: str
+) -> None:
+ config_overrides(RBAC_ENABLED=False)
+ importer = Mock()
+ fetch = _mock_download(monkeypatch)
+ importer.import_package.return_value = RosterAgentPackageImportResult(
+ app_id="app-1", agent_id="agent-1", warnings=[]
+ )
+ monkeypatch.setattr(import_module, "RosterAgentPackageImporter", lambda: importer)
+ api = import_module.AppImportApi()
+ import_dsl = Mock()
+ monkeypatch.setattr(api, "_import_dsl", import_dsl)
+ account = _account()
+ with app.test_request_context(
+ "/console/api/apps/imports", method="POST", json={"mode": "yaml-url", "yaml_url": url}
+ ):
+ data, status = unwrap(api.post)(api, account)
+ assert status == 200
+ assert data["app_mode"] == "agent"
+ assert data["status"] == "completed"
+ fetch.assert_called_once()
+ assert importer.import_package.call_args.kwargs["tenant_id"] == "tenant-1"
+ assert importer.import_package.call_args.kwargs["account"] is account
+ import_dsl.assert_not_called()
+
+
+ "url", ["https://example.com/app.yaml", "https://example.com/download", "https://example.com/agent.ifpkg"]
+)
+def test_yaml_url_keeps_dsl_import_without_package_retry(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], url: str
+) -> None:
+ config_overrides(RBAC_ENABLED=False)
+ fetch = _mock_download(monkeypatch, b"app: {}")
+ api = import_module.AppImportApi()
+ import_dsl = Mock(return_value=({"status": "failed", "error": "Missing app data"}, 400))
+ import_package = Mock()
+ monkeypatch.setattr(api, "_import_dsl", import_dsl)
+ monkeypatch.setattr(api, "_import_package", import_package)
+ with app.test_request_context(
+ "/console/api/apps/imports",
+ method="POST",
+ json={"mode": "yaml-url", "yaml_url": url, "app_id": "existing", "name": "Renamed"},
+ ):
+ result = unwrap(api.post)(api, _account())
+ assert result[1] == 400
+ assert import_dsl.call_args.args[0] == import_module.AppImportPayload(
+ mode="yaml-content", yaml_content="app: {}", app_id="existing", name="Renamed"
+ )
+ fetch.assert_called_once()
+ import_package.assert_not_called()
+
+
+def test_url_without_any_import_permission_does_not_download(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]
+) -> None:
+ config_overrides(RBAC_ENABLED=True)
+ _mock_download(monkeypatch)
+ account = _account()
+ monkeypatch.setattr("controllers.common.wraps.current_account_with_tenant", lambda: (account, "tenant-1"))
+ monkeypatch.setattr("controllers.common.rbac.checks.RBACService.CheckAccess.check", lambda *_args, **_kwargs: False)
+ fetch = _mock_download(monkeypatch)
+ api = import_module.AppImportApi()
+ with app.test_request_context(
+ "/console/api/apps/imports",
+ method="POST",
+ json={"mode": "yaml-url", "yaml_url": "https://example.com/download"},
+ ):
+ with pytest.raises(Forbidden):
+ unwrap(api.post)(api, account)
+ fetch.assert_not_called()
+
+
[email protected]("is_yaml", [False, True])
+def test_url_quota_is_enforced_after_content_detection(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], is_yaml: bool
+) -> None:
+ config_overrides(RBAC_ENABLED=False, DEPLOYMENT_EDITION="CLOUD")
+ account = _account()
+ monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: (account, "tenant-1"))
+ features = Mock()
+ features.apps = SimpleNamespace(size=1, limit=1)
+ monkeypatch.setattr("controllers.console.wraps.FeatureService.get_features", lambda *_args, **_kwargs: features)
+ _mock_download(monkeypatch, b"app: {}" if is_yaml else b"PK\x00\xff")
+ importer = Mock()
+ importer.import_package.return_value = RosterAgentPackageImportResult(
+ app_id="app-1", agent_id="agent-1", warnings=[]
+ )
+ monkeypatch.setattr(import_module, "RosterAgentPackageImporter", lambda: importer)
+ api = import_module.AppImportApi()
+ with app.test_request_context(
+ "/console/api/apps/imports",
+ method="POST",
+ json={"mode": "yaml-url", "yaml_url": "https://example.com/download"},
+ ):
+ if is_yaml:
+ with pytest.raises(Forbidden, match="number of apps"):
+ unwrap(api.post)(api, account)
+ importer.import_package.assert_not_called()
+ else:
+ data, status = unwrap(api.post)(api, account)
+ assert status == 200
+ assert data["app_mode"] == "agent"
+
+
+def test_url_rejects_content_that_is_neither_yaml_nor_package(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]
+) -> None:
+ config_overrides(RBAC_ENABLED=False)
+ fetch = _mock_download(monkeypatch, b"[invalid")
+ api = import_module.AppImportApi()
+ with app.test_request_context(
+ "/console/api/apps/imports",
+ method="POST",
+ json={"mode": "yaml-url", "yaml_url": "https://example.com/download"},
+ ):
+ with pytest.raises(InvalidRosterAgentPackageError):
+ unwrap(api.post)(api, _account())
+ fetch.assert_called_once()
+
+
@pytest.mark.parametrize("has_warning", [False, True])
def test_existing_import_route_accepts_package_and_preserves_import_response(
app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], has_warning: bool
@@ -75,11 +215,17 @@ def test_existing_import_route_accepts_package_and_preserves_import_response(
assert import_package.call_args.kwargs["tenant_id"] == "tenant-1"
[email protected]("from_url", [False, True])
@pytest.mark.parametrize("denied", [RBACPermission.AGENT_CREATE, RBACPermission.AGENT_IMPORT_EXPORT_DSL])
def test_package_import_checks_agent_permissions_before_reading_payload(
- app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], denied: RBACPermission
+ app: Flask,
+ monkeypatch: pytest.MonkeyPatch,
+ config_overrides: Callable[..., None],
+ denied: RBACPermission,
+ from_url: bool,
) -> None:
config_overrides(RBAC_ENABLED=True)
+ _mock_download(monkeypatch)
account = _account()
monkeypatch.setattr("controllers.common.wraps.current_account_with_tenant", lambda: (account, "tenant-1"))
scenes = []
@@ -92,24 +238,33 @@ def test_package_import_checks_agent_permissions_before_reading_payload(
importer = Mock()
monkeypatch.setattr(import_module, "RosterAgentPackageImporter", importer)
api = import_module.AppImportApi()
- with app.test_request_context(
- "/console/api/apps/imports", method="POST", data={"file": (io.BytesIO(b"package"), "agent.ifpkg")}
- ):
+ request_kwargs = (
+ {"json": {"mode": "yaml-url", "yaml_url": "https://example.com/agent.ifpkg"}}
+ if from_url
+ else {"data": {"file": (io.BytesIO(b"package"), "agent.ifpkg")}}
+ )
+ with app.test_request_context("/console/api/apps/imports", method="POST", **request_kwargs):
with pytest.raises(Forbidden):
unwrap(api.post)(api, account)
assert denied in scenes
- assert RBACPermission.APP_IMPORT_EXPORT_DSL not in scenes
+ if not from_url:
+ assert RBACPermission.APP_IMPORT_EXPORT_DSL not in scenes
importer.assert_not_called()
-def test_package_import_rejects_overwrite(app: Flask, config_overrides: Callable[..., None]) -> None:
[email protected]("from_url", [False, True])
+def test_package_import_rejects_overwrite(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], from_url: bool
+) -> None:
config_overrides(RBAC_ENABLED=False)
+ _mock_download(monkeypatch)
api = import_module.AppImportApi()
- with app.test_request_context(
- "/console/api/apps/imports",
- method="POST",
- data={"app_id": "existing", "file": (io.BytesIO(b"package"), "agent.ifpkg")},
- ):
+ request_kwargs = (
+ {"json": {"mode": "yaml-url", "yaml_url": "https://example.com/agent.ifpkg", "app_id": "existing"}}
+ if from_url
+ else {"data": {"app_id": "existing", "file": (io.BytesIO(b"package"), "agent.ifpkg")}}
+ )
+ with app.test_request_context("/console/api/apps/imports", method="POST", **request_kwargs):
with pytest.raises(InvalidRosterAgentPackageError, match="overwriting"):
unwrap(api.post)(api, _account())
diff --git a/api/tests/unit_tests/services/test_app_import_source.py b/api/tests/unit_tests/services/test_app_import_source.py
new file mode 100644
index 00000000000..7d41932c3f7
--- /dev/null
+++ b/api/tests/unit_tests/services/test_app_import_source.py
@@ -0,0 +1,95 @@
+import io
+from collections.abc import Callable
+from unittest.mock import Mock
+
+import httpx
+import pytest
+
+from core.tools.errors import ToolSSRFError
+from services import app_import_source as module
+from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError
+
+
+ ("url", "download_url"),
+ [
+ (" https://example.com/agent.IFPKG?token=secret ", "https://example.com/agent.IFPKG?token=secret"),
+ (
+ "https://github.com/org/repo/blob/main/agent.ifpkg",
+ "https://raw.githubusercontent.com/org/repo/main/agent.ifpkg",
+ ),
+ ],
+)
+def test_download_preserves_binary_and_closes_resources(
+ monkeypatch: pytest.MonkeyPatch, url: str, download_url: str
+) -> None:
+ content = b"PK\x00\xff"
+ response = httpx.Response(200, stream=httpx.ByteStream(content), request=httpx.Request("GET", download_url))
+ fetch = Mock(return_value=response)
+ monkeypatch.setattr(module.remote_fetcher, "make_request", fetch)
+ with module.download_app_import_source(url) as source:
+ assert response.is_closed
+ assert source.read() == content
+ assert source.closed
+ fetch.assert_called_once_with(
+ "GET",
+ download_url,
+ stream_response=True,
+ follow_redirects=True,
+ timeout=(10, 10),
+ headers={"Accept-Encoding": "identity"},
+ )
+
+
[email protected]("failure", ["oversize", "encoding", "http", "network", "ssrf"])
+def test_download_failure_never_imports(
+ monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], failure: str
+) -> None:
+ config_overrides(AGENT_PACKAGE_MAX_BYTES=3)
+ monkeypatch.setattr(module, "DSL_MAX_SIZE", 3)
+ response = httpx.Response(
+ 404 if failure == "http" else 200,
+ headers={"content-encoding": "gzip"} if failure == "encoding" else {},
+ stream=httpx.ByteStream(b"1234"),
+ request=httpx.Request("GET", "https://example.com/agent.ifpkg"),
+ )
+ fetch = Mock(return_value=response)
+ if failure == "network":
+ fetch.side_effect = httpx.ReadTimeout("timeout")
+ elif failure == "ssrf":
+ fetch.side_effect = ToolSSRFError("blocked")
+ monkeypatch.setattr(module.remote_fetcher, "make_request", fetch)
+ error = RosterAgentPackageTooLargeError if failure == "oversize" else InvalidRosterAgentPackageError
+ with pytest.raises(error), module.download_app_import_source("https://example.com/download"):
+ pytest.fail("Failed downloads must not expose an import source")
+ if failure not in {"network", "ssrf"}:
+ assert response.is_closed
+
+
+ "url", ["file:///tmp/agent.ifpkg", "ftp://example.com/agent.ifpkg", "/agent.ifpkg", "https://[bad/agent.ifpkg"]
+)
+def test_invalid_url_never_downloads(monkeypatch: pytest.MonkeyPatch, url: str) -> None:
+ fetch = Mock()
+ monkeypatch.setattr(module.remote_fetcher, "make_request", fetch)
+ with pytest.raises(InvalidRosterAgentPackageError), module.download_app_import_source(url):
+ pytest.fail("Invalid URLs must not expose an import source")
+ fetch.assert_not_called()
+
+
+ ("content", "is_yaml"),
+ [(b"app: {}", True), (b"PK\x00\xff", False), (b"[broken", False), (b"hello", False), (b"", False)],
+)
+def test_yaml_detection_rewinds_for_package_fallback(content: bytes, is_yaml: bool) -> None:
+ source = io.BytesIO(content)
+ result = module.try_read_yaml(source)
+ assert (result is not None) is is_yaml
+ assert source.read() == content
+
+
+def test_yaml_detection_does_not_parse_oversized_content(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(module, "DSL_MAX_SIZE", 3)
+ source = io.BytesIO(b"app: {}")
+ assert module.try_read_yaml(source) is None
+ assert source.tell() == 0