Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
feat(agent): support package imports from external URLs (#42425)
林玮 (Jade Lin)223039128f1b
diff --git a/api/controllers/console/app/app_import.py b/api/controllers/console/app/app_import.pyindex 4872947b4dd..a2592c81300 100644--- a/api/controllers/console/app/app_import.py+++ b/api/controllers/console/app/app_import.py@@ -36,6 +36,7 @@ from services.app_dsl_service import (Import,PendingData,)+from services.app_import_source import download_app_import_source, try_read_yamlfrom services.enterprise.enterprise_service import EnterpriseServicefrom services.entities.dsl_entities import CheckDependenciesResult, ImportStatusfrom services.errors.account import NoPermissionError@@ -128,24 +129,45 @@ class AppImportApi(Resource):def post(self, current_user: Account):if request.mimetype == "multipart/form-data":return self._import_package(current_user)- return self._import_dsl(validate_request(AppImportPayload), current_user)+ payload = validate_request(AppImportPayload)+ if payload.mode == "yaml-url" and payload.yaml_url:+ return self._import_url(payload, current_user)+ return self._import_dsl(payload, current_user)++ @rbac_permission_required(+ RBACCheck(RBACPermission.APP_IMPORT_EXPORT_DSL, Workspace()),+ RBACCheck(RBACPermission.AGENT_IMPORT_EXPORT_DSL, Workspace()),+ )+ def _import_url(self, payload: AppImportPayload, current_user: Account):+ with download_app_import_source(payload.yaml_url or "") as source:+ content = try_read_yaml(source)+ if content is not None:+ return self._import_dsl(+ payload.model_copy(update={"mode": "yaml-content", "yaml_content": content, "yaml_url": None}),+ current_user,+ )+ return self._import_package(current_user, payload, source)@rbac_permission_required(RBACCheck(RBACPermission.AGENT_CREATE, Workspace()))@rbac_permission_required(RBACCheck(RBACPermission.AGENT_IMPORT_EXPORT_DSL, Workspace()))- def _import_package(self, current_user: Account):- uploaded = request.files.get("file")- if uploaded is None or not uploaded.filename:- raise InvalidRosterAgentPackageError("Roster Agent package file is required")- if not uploaded.filename.lower().endswith(".ifpkg"):- raise InvalidRosterAgentPackageError("Roster Agent package file must use the .ifpkg extension")- if request.form.get("app_id"):+ def _import_package(+ self, current_user: Account, payload: AppImportPayload | None = None, source: BinaryIO | None = None+ ):+ app_id = payload.app_id if payload is not None else request.form.get("app_id")+ if app_id:raise InvalidRosterAgentPackageError("Roster Agent package import does not support overwriting an App")account, tenant_id = _current_user_and_tenant_id(current_user)if tenant_id is None:raise Forbidden("Current workspace is required")- result = RosterAgentPackageImporter().import_package(- source=cast(BinaryIO, uploaded.stream), tenant_id=tenant_id, account=account- )+ importer = RosterAgentPackageImporter()+ if source is None:+ uploaded = request.files.get("file")+ if uploaded is None or not uploaded.filename:+ raise InvalidRosterAgentPackageError("Roster Agent package file is required")+ if not uploaded.filename.lower().endswith(".ifpkg"):+ raise InvalidRosterAgentPackageError("Roster Agent package file must use the .ifpkg extension")+ source = cast(BinaryIO, uploaded.stream)+ result = importer.import_package(source=source, tenant_id=tenant_id, account=account)return Import(id=str(uuid4()),status=ImportStatus.COMPLETED_WITH_WARNINGS if result.warnings else ImportStatus.COMPLETED,diff --git a/api/services/app_import_source.py b/api/services/app_import_source.pynew file mode 100644index 00000000000..2d3457f16a9--- /dev/null+++ b/api/services/app_import_source.py@@ -0,0 +1,74 @@+"""Download URL imports once and detect YAML before falling back to package validation."""++import tempfile+from collections.abc import Generator+from contextlib import contextmanager+from typing import BinaryIO, cast+from urllib.parse import urlsplit, urlunsplit++import httpx+import yaml++from configs import dify_config+from core.file import remote_fetcher+from core.tools.errors import ToolSSRFError+from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError+from services.dsl_content import DSL_MAX_SIZE+++@contextmanager+def download_app_import_source(url: str) -> Generator[BinaryIO]:+ """Fetch a bounded source without opening import transactions or creating resources."""+ url = url.strip()+ try:+ parsed = urlsplit(url)+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:+ raise ValueError("An absolute HTTP(S) URL is required")+ except ValueError as exc:+ raise InvalidRosterAgentPackageError("App import URL is invalid") from exc+ if parsed.scheme == "https" and parsed.netloc == "github.com" and "/blob/" in parsed.path:+ url = urlunsplit(+ parsed._replace(netloc="raw.githubusercontent.com", path=parsed.path.replace("/blob/", "/", 1))+ )++ with tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b") as source:+ try:+ response = remote_fetcher.make_request(+ "GET",+ url,+ stream_response=True,+ follow_redirects=True,+ timeout=(10, 10),+ headers={"Accept-Encoding": "identity"},+ )+ try:+ response.raise_for_status()+ # Reject HTTP compression so the byte limit also bounds decompression memory.+ if response.headers.get("content-encoding", "identity").strip().lower() not in {"", "identity"}:+ raise InvalidRosterAgentPackageError("Compressed HTTP import responses are not supported")+ size = 0+ for chunk in response.iter_bytes(chunk_size=1024 * 1024):+ size += len(chunk)+ if size > max(dify_config.AGENT_PACKAGE_MAX_BYTES, DSL_MAX_SIZE):+ raise RosterAgentPackageTooLargeError("App import download exceeds the size limit")+ source.write(chunk)+ finally:+ response.close()+ except (httpx.HTTPError, httpx.InvalidURL, remote_fetcher.max_retries_exceeded_error, ToolSSRFError) as exc:+ raise InvalidRosterAgentPackageError("Could not download the App import from URL") from exc+ source.seek(0)+ yield cast(BinaryIO, source)+++def try_read_yaml(source: BinaryIO) -> str | None:+ """Recognize a YAML mapping without executing an import; always rewind for the package fallback."""+ try:+ raw = source.read(DSL_MAX_SIZE + 1)+ if len(raw) > DSL_MAX_SIZE:+ return None+ content = raw.decode("utf-8")+ return content if isinstance(yaml.safe_load(content), dict) else None+ except (UnicodeDecodeError, yaml.YAMLError):+ return None+ finally:+ source.seek(0)diff --git a/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py b/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.pyindex bb8b9c8ab06..e8e13be6424 100644--- a/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py+++ b/api/tests/unit_tests/controllers/console/app/test_roster_package_routes.py@@ -4,6 +4,7 @@ from inspect import unwrapfrom types import SimpleNamespacefrom unittest.mock import Mock+import httpximport pytestfrom flask import Flaskfrom werkzeug.exceptions import BadRequest, Forbidden@@ -13,11 +14,20 @@ from controllers.console.app import app_import as import_modulefrom core.rbac import RBACPermissionfrom models.account import Account, Tenantfrom models.model import App, AppMode+from services import app_import_sourcefrom services.agent.errors import InvalidRosterAgentPackageErrorfrom services.agent.roster_package_importer import RosterAgentPackageImportResultfrom services.entities.dsl_entities import DslImportWarning+def _mock_download(monkeypatch: pytest.MonkeyPatch, content: bytes = b"PK\x00\xff") -> Mock:+ fetch = Mock(+ return_value=httpx.Response(200, content=content, request=httpx.Request("GET", "https://example.com/download"))+ )+ monkeypatch.setattr(app_import_source.remote_fetcher, "make_request", fetch)+ return fetch++def _account() -> Account:account = Account(name="Importer", email="[email protected]")account._current_tenant = Tenant(name="Workspace")@@ -43,6 +53,136 @@ def test_existing_import_route_dispatches_json_without_changing_payload()+ "url",+ [+ "https://example.com/download?id=123",+ "https://example.com/agent.ifpkg",+ " https://example.com/agent.IFPKG?token=secret#download ",+ ],+)+def test_package_url_uses_agent_import(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], url: str+) -> None:+ config_overrides(RBAC_ENABLED=False)+ importer = Mock()+ fetch = _mock_download(monkeypatch)+ importer.import_package.return_value = RosterAgentPackageImportResult(+ app_id="app-1", agent_id="agent-1", warnings=[]+ )+ monkeypatch.setattr(import_module, "RosterAgentPackageImporter", lambda: importer)+ api = import_module.AppImportApi()+ import_dsl = Mock()+ monkeypatch.setattr(api, "_import_dsl", import_dsl)+ account = _account()+ with app.test_request_context(+ "/console/api/apps/imports", method="POST", json={"mode": "yaml-url", "yaml_url": url}+ ):+ data, status = unwrap(api.post)(api, account)+ assert status == 200+ assert data["app_mode"] == "agent"+ assert data["status"] == "completed"+ fetch.assert_called_once()+ assert importer.import_package.call_args.kwargs["tenant_id"] == "tenant-1"+ assert importer.import_package.call_args.kwargs["account"] is account+ import_dsl.assert_not_called()+++ "url", ["https://example.com/app.yaml", "https://example.com/download", "https://example.com/agent.ifpkg"]+)+def test_yaml_url_keeps_dsl_import_without_package_retry(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], url: str+) -> None:+ config_overrides(RBAC_ENABLED=False)+ fetch = _mock_download(monkeypatch, b"app: {}")+ api = import_module.AppImportApi()+ import_dsl = Mock(return_value=({"status": "failed", "error": "Missing app data"}, 400))+ import_package = Mock()+ monkeypatch.setattr(api, "_import_dsl", import_dsl)+ monkeypatch.setattr(api, "_import_package", import_package)+ with app.test_request_context(+ "/console/api/apps/imports",+ method="POST",+ json={"mode": "yaml-url", "yaml_url": url, "app_id": "existing", "name": "Renamed"},+ ):+ result = unwrap(api.post)(api, _account())+ assert result[1] == 400+ assert import_dsl.call_args.args[0] == import_module.AppImportPayload(+ mode="yaml-content", yaml_content="app: {}", app_id="existing", name="Renamed"+ )+ fetch.assert_called_once()+ import_package.assert_not_called()+++def test_url_without_any_import_permission_does_not_download(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]+) -> None:+ config_overrides(RBAC_ENABLED=True)+ _mock_download(monkeypatch)+ account = _account()+ monkeypatch.setattr("controllers.common.wraps.current_account_with_tenant", lambda: (account, "tenant-1"))+ monkeypatch.setattr("controllers.common.rbac.checks.RBACService.CheckAccess.check", lambda *_args, **_kwargs: False)+ fetch = _mock_download(monkeypatch)+ api = import_module.AppImportApi()+ with app.test_request_context(+ "/console/api/apps/imports",+ method="POST",+ json={"mode": "yaml-url", "yaml_url": "https://example.com/download"},+ ):+ with pytest.raises(Forbidden):+ unwrap(api.post)(api, account)+ fetch.assert_not_called()++[email protected]("is_yaml", [False, True])+def test_url_quota_is_enforced_after_content_detection(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], is_yaml: bool+) -> None:+ config_overrides(RBAC_ENABLED=False, DEPLOYMENT_EDITION="CLOUD")+ account = _account()+ monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: (account, "tenant-1"))+ features = Mock()+ features.apps = SimpleNamespace(size=1, limit=1)+ monkeypatch.setattr("controllers.console.wraps.FeatureService.get_features", lambda *_args, **_kwargs: features)+ _mock_download(monkeypatch, b"app: {}" if is_yaml else b"PK\x00\xff")+ importer = Mock()+ importer.import_package.return_value = RosterAgentPackageImportResult(+ app_id="app-1", agent_id="agent-1", warnings=[]+ )+ monkeypatch.setattr(import_module, "RosterAgentPackageImporter", lambda: importer)+ api = import_module.AppImportApi()+ with app.test_request_context(+ "/console/api/apps/imports",+ method="POST",+ json={"mode": "yaml-url", "yaml_url": "https://example.com/download"},+ ):+ if is_yaml:+ with pytest.raises(Forbidden, match="number of apps"):+ unwrap(api.post)(api, account)+ importer.import_package.assert_not_called()+ else:+ data, status = unwrap(api.post)(api, account)+ assert status == 200+ assert data["app_mode"] == "agent"+++def test_url_rejects_content_that_is_neither_yaml_nor_package(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]+) -> None:+ config_overrides(RBAC_ENABLED=False)+ fetch = _mock_download(monkeypatch, b"[invalid")+ api = import_module.AppImportApi()+ with app.test_request_context(+ "/console/api/apps/imports",+ method="POST",+ json={"mode": "yaml-url", "yaml_url": "https://example.com/download"},+ ):+ with pytest.raises(InvalidRosterAgentPackageError):+ unwrap(api.post)(api, _account())+ fetch.assert_called_once()++@pytest.mark.parametrize("has_warning", [False, True])def test_existing_import_route_accepts_package_and_preserves_import_response(app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], has_warning: bool@@ -75,11 +215,17 @@ def test_existing_import_route_accepts_package_and_preserves_import_response(assert import_package.call_args.kwargs["tenant_id"] == "tenant-1"[email protected]("from_url", [False, True])@pytest.mark.parametrize("denied", [RBACPermission.AGENT_CREATE, RBACPermission.AGENT_IMPORT_EXPORT_DSL])def test_package_import_checks_agent_permissions_before_reading_payload(- app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], denied: RBACPermission+ app: Flask,+ monkeypatch: pytest.MonkeyPatch,+ config_overrides: Callable[..., None],+ denied: RBACPermission,+ from_url: bool,) -> None:config_overrides(RBAC_ENABLED=True)+ _mock_download(monkeypatch)account = _account()monkeypatch.setattr("controllers.common.wraps.current_account_with_tenant", lambda: (account, "tenant-1"))scenes = []@@ -92,24 +238,33 @@ def test_package_import_checks_agent_permissions_before_reading_payload(importer = Mock()monkeypatch.setattr(import_module, "RosterAgentPackageImporter", importer)api = import_module.AppImportApi()- with app.test_request_context(- "/console/api/apps/imports", method="POST", data={"file": (io.BytesIO(b"package"), "agent.ifpkg")}- ):+ request_kwargs = (+ {"json": {"mode": "yaml-url", "yaml_url": "https://example.com/agent.ifpkg"}}+ if from_url+ else {"data": {"file": (io.BytesIO(b"package"), "agent.ifpkg")}}+ )+ with app.test_request_context("/console/api/apps/imports", method="POST", **request_kwargs):with pytest.raises(Forbidden):unwrap(api.post)(api, account)assert denied in scenes- assert RBACPermission.APP_IMPORT_EXPORT_DSL not in scenes+ if not from_url:+ assert RBACPermission.APP_IMPORT_EXPORT_DSL not in scenesimporter.assert_not_called()-def test_package_import_rejects_overwrite(app: Flask, config_overrides: Callable[..., None]) -> None:[email protected]("from_url", [False, True])+def test_package_import_rejects_overwrite(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], from_url: bool+) -> None:config_overrides(RBAC_ENABLED=False)+ _mock_download(monkeypatch)api = import_module.AppImportApi()- with app.test_request_context(- "/console/api/apps/imports",- method="POST",- data={"app_id": "existing", "file": (io.BytesIO(b"package"), "agent.ifpkg")},- ):+ request_kwargs = (+ {"json": {"mode": "yaml-url", "yaml_url": "https://example.com/agent.ifpkg", "app_id": "existing"}}+ if from_url+ else {"data": {"app_id": "existing", "file": (io.BytesIO(b"package"), "agent.ifpkg")}}+ )+ with app.test_request_context("/console/api/apps/imports", method="POST", **request_kwargs):with pytest.raises(InvalidRosterAgentPackageError, match="overwriting"):unwrap(api.post)(api, _account())diff --git a/api/tests/unit_tests/services/test_app_import_source.py b/api/tests/unit_tests/services/test_app_import_source.pynew file mode 100644index 00000000000..7d41932c3f7--- /dev/null+++ b/api/tests/unit_tests/services/test_app_import_source.py@@ -0,0 +1,95 @@+import io+from collections.abc import Callable+from unittest.mock import Mock++import httpx+import pytest++from core.tools.errors import ToolSSRFError+from services import app_import_source as module+from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError+++ ("url", "download_url"),+ [+ (" https://example.com/agent.IFPKG?token=secret ", "https://example.com/agent.IFPKG?token=secret"),+ (+ "https://github.com/org/repo/blob/main/agent.ifpkg",+ "https://raw.githubusercontent.com/org/repo/main/agent.ifpkg",+ ),+ ],+)+def test_download_preserves_binary_and_closes_resources(+ monkeypatch: pytest.MonkeyPatch, url: str, download_url: str+) -> None:+ content = b"PK\x00\xff"+ response = httpx.Response(200, stream=httpx.ByteStream(content), request=httpx.Request("GET", download_url))+ fetch = Mock(return_value=response)+ monkeypatch.setattr(module.remote_fetcher, "make_request", fetch)+ with module.download_app_import_source(url) as source:+ assert response.is_closed+ assert source.read() == content+ assert source.closed+ fetch.assert_called_once_with(+ "GET",+ download_url,+ stream_response=True,+ follow_redirects=True,+ timeout=(10, 10),+ headers={"Accept-Encoding": "identity"},+ )++[email protected]("failure", ["oversize", "encoding", "http", "network", "ssrf"])+def test_download_failure_never_imports(+ monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], failure: str+) -> None:+ config_overrides(AGENT_PACKAGE_MAX_BYTES=3)+ monkeypatch.setattr(module, "DSL_MAX_SIZE", 3)+ response = httpx.Response(+ 404 if failure == "http" else 200,+ headers={"content-encoding": "gzip"} if failure == "encoding" else {},+ stream=httpx.ByteStream(b"1234"),+ request=httpx.Request("GET", "https://example.com/agent.ifpkg"),+ )+ fetch = Mock(return_value=response)+ if failure == "network":+ fetch.side_effect = httpx.ReadTimeout("timeout")+ elif failure == "ssrf":+ fetch.side_effect = ToolSSRFError("blocked")+ monkeypatch.setattr(module.remote_fetcher, "make_request", fetch)+ error = RosterAgentPackageTooLargeError if failure == "oversize" else InvalidRosterAgentPackageError+ with pytest.raises(error), module.download_app_import_source("https://example.com/download"):+ pytest.fail("Failed downloads must not expose an import source")+ if failure not in {"network", "ssrf"}:+ assert response.is_closed+++ "url", ["file:///tmp/agent.ifpkg", "ftp://example.com/agent.ifpkg", "/agent.ifpkg", "https://[bad/agent.ifpkg"]+)+def test_invalid_url_never_downloads(monkeypatch: pytest.MonkeyPatch, url: str) -> None:+ fetch = Mock()+ monkeypatch.setattr(module.remote_fetcher, "make_request", fetch)+ with pytest.raises(InvalidRosterAgentPackageError), module.download_app_import_source(url):+ pytest.fail("Invalid URLs must not expose an import source")+ fetch.assert_not_called()+++ ("content", "is_yaml"),+ [(b"app: {}", True), (b"PK\x00\xff", False), (b"[broken", False), (b"hello", False), (b"", False)],+)+def test_yaml_detection_rewinds_for_package_fallback(content: bytes, is_yaml: bool) -> None:+ source = io.BytesIO(content)+ result = module.try_read_yaml(source)+ assert (result is not None) is is_yaml+ assert source.read() == content+++def test_yaml_detection_does_not_parse_oversized_content(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(module, "DSL_MAX_SIZE", 3)+ source = io.BytesIO(b"app: {}")+ assert module.try_read_yaml(source) is None+ assert source.tell() == 0