Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: deliver real uploaded and tool files from local storage (#43208)

Asuka Minatoc797b8e99ef0
diff --git a/api/tests/unit_tests/services/test_tool_file_download_service.py b/api/tests/unit_tests/services/test_tool_file_download_service.py
index 43f6110e769..358b432e908 100644
--- a/api/tests/unit_tests/services/test_tool_file_download_service.py
+++ b/api/tests/unit_tests/services/test_tool_file_download_service.py
@@ -1,91 +1,104 @@
-from collections.abc import Iterator
-from unittest.mock import Mock, patch
+from pathlib import Path
+from unittest.mock import patch
import pytest
+from pytest_mock import MockerFixture
+from sqlalchemy.orm import Session, sessionmaker
-from graphon.file import File, FileTransferMethod, FileType
+from core.tools.tool_file_manager import ToolFileManager
+from extensions.ext_storage import Storage
+from extensions.storage.opendal_storage import OpenDALStorage
+from models.tools import ToolFile
from services.tool_file_download_service import (
- ToolFileDownload,
ToolFileDownloadAccessDeniedError,
ToolFileDownloadNotFoundError,
ToolFileDownloadService,
- ToolFileDownloadSource,
)
-def _file() -> File:
- return File(
- file_id="file-id",
- tenant_id="tenant-id",
- file_type=FileType.DOCUMENT,
- transfer_method=FileTransferMethod.TOOL_FILE,
- related_id="file-id",
- filename="tool.txt",
- extension=".txt",
- mime_type="text/plain",
- size=12,
- storage_key="tools/tenant-id/file.txt",
- )
+def storage(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Storage:
+ storage = Storage()
+ storage.storage_runner = OpenDALStorage(scheme="fs", root=str(tmp_path))
+ monkeypatch.setattr("core.tools.tool_file_manager.storage", storage)
+ return storage
@pytest.fixture
-def tool_files() -> Mock:
- return Mock(spec=ToolFileDownloadSource)
+def tool_files(sqlite_session_factory: sessionmaker[Session], storage: Storage) -> ToolFileManager:
+ storage.save("tools/tenant-id/file.txt", b"ab")
+ with sqlite_session_factory.begin() as session:
+ file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id=None,
+ file_key="tools/tenant-id/file.txt",
+ mimetype="text/plain",
+ name="tool.txt",
+ size=2,
+ )
+ file.id = "file-id"
+ session.add(file)
+ return ToolFileManager()
@pytest.fixture
-def service(tool_files: Mock) -> ToolFileDownloadService:
+def service(tool_files: ToolFileManager) -> ToolFileDownloadService:
return ToolFileDownloadService(tool_files=tool_files)
-def test_invalid_signature_does_not_load_file(service: ToolFileDownloadService, tool_files: Mock) -> None:
+def test_invalid_signature_does_not_load_file(
+ service: ToolFileDownloadService, tool_files: ToolFileManager, mocker: MockerFixture
+) -> None:
+ load = mocker.spy(tool_files, "get_file_generator_by_tool_file_id")
with patch("services.tool_file_download_service.verify_tool_file_signature", return_value=False) as verify:
with pytest.raises(ToolFileDownloadAccessDeniedError):
service.get_signed_file(file_id="file-id", timestamp="1", nonce="nonce", sign="invalid")
verify.assert_called_once_with(file_id="file-id", timestamp="1", nonce="nonce", sign="invalid")
- tool_files.get_file_generator_by_tool_file_id.assert_not_called()
+ load.assert_not_called()
def test_missing_file_is_reported_after_signature_validation(
service: ToolFileDownloadService,
- tool_files: Mock,
+ tool_files: ToolFileManager,
+ mocker: MockerFixture,
) -> None:
- tool_files.get_file_generator_by_tool_file_id.return_value = (None, None)
-
+ load = mocker.spy(tool_files, "get_file_generator_by_tool_file_id")
with patch("services.tool_file_download_service.verify_tool_file_signature", return_value=True):
with pytest.raises(ToolFileDownloadNotFoundError):
service.get_signed_file(file_id="missing", timestamp="1", nonce="nonce", sign="valid")
- tool_files.get_file_generator_by_tool_file_id.assert_called_once_with(tool_file_id="missing")
+ load.assert_called_once_with(tool_file_id="missing")
def test_signed_file_returns_stream_and_metadata(
service: ToolFileDownloadService,
- tool_files: Mock,
+ tool_files: ToolFileManager,
+ mocker: MockerFixture,
) -> None:
- stream: Iterator[bytes] = iter([b"a", b"b"])
- tool_files.get_file_generator_by_tool_file_id.return_value = (stream, _file())
-
+ load = mocker.spy(tool_files, "get_file_generator_by_tool_file_id")
with patch("services.tool_file_download_service.verify_tool_file_signature", return_value=True):
result = service.get_signed_file(file_id="file-id", timestamp="1", nonce="nonce", sign="valid")
- assert result == ToolFileDownload(
- content=stream,
- mime_type="text/plain",
- filename="tool.txt",
- size=12,
- )
- tool_files.get_file_generator_by_tool_file_id.assert_called_once_with(tool_file_id="file-id")
+ assert b"".join(result.content) == b"ab"
+ assert result.mime_type == "text/plain"
+ assert result.filename == "tool.txt"
+ assert result.size == 2
+ load.assert_called_once_with(tool_file_id="file-id")
@pytest.mark.parametrize("source_error", [RuntimeError("database unavailable"), OSError("storage unavailable")])
def test_source_error_is_not_converted(
service: ToolFileDownloadService,
- tool_files: Mock,
+ storage: Storage,
+ mocker: MockerFixture,
source_error: Exception,
) -> None:
- tool_files.get_file_generator_by_tool_file_id.side_effect = source_error
+ if isinstance(source_error, OSError):
+ mocker.patch.object(storage, "load_stream", side_effect=source_error)
+ else:
+ mocker.patch("core.tools.tool_file_manager.session_factory.create_session", side_effect=source_error)
with patch("services.tool_file_download_service.verify_tool_file_signature", return_value=True):
with pytest.raises(type(source_error)) as error_info:
diff --git a/api/tests/unit_tests/services/test_upload_file_delivery_service.py b/api/tests/unit_tests/services/test_upload_file_delivery_service.py
index b7ec99e508a..37381416123 100644
--- a/api/tests/unit_tests/services/test_upload_file_delivery_service.py
+++ b/api/tests/unit_tests/services/test_upload_file_delivery_service.py
@@ -1,21 +1,27 @@
-from unittest.mock import Mock, patch
+import json
+from pathlib import Path
+from unittest.mock import patch
import pytest
+from pytest_mock import MockerFixture
+from sqlalchemy.orm import Session, sessionmaker
+from extensions.ext_storage import Storage
+from extensions.storage.opendal_storage import OpenDALStorage
+from repositories.upload_file_delivery_repository import UploadFileDeliveryQueryRepository
from services.errors.file import UnsupportedFileTypeError
from services.upload_file_delivery_service import (
UploadFileDelivery,
UploadFileDeliveryNotFoundError,
- UploadFileDeliveryQuery,
UploadFileDeliveryRecord,
UploadFileDeliveryService,
- UploadFileStorage,
)
+from tests.unit_tests.model_factories import make_tenant, make_upload_file
def _record(*, extension: str = "png") -> UploadFileDeliveryRecord:
return UploadFileDeliveryRecord(
- key="upload_files/tenant-id/file-id",
+ key=f"upload_files/workspace-id/file.{extension}",
name=f"file.{extension}",
size=7,
extension=extension,
@@ -24,152 +30,182 @@ def _record(*, extension: str = "png") -> UploadFileDeliveryRecord:
@pytest.fixture
-def files() -> Mock:
- return Mock(spec=UploadFileDeliveryQuery)
+def files(sqlite_session_factory: sessionmaker[Session]) -> UploadFileDeliveryQueryRepository:
+ with sqlite_session_factory.begin() as session:
+ tenant = make_tenant(tenant_id="workspace-id")
+ tenant.custom_config = json.dumps({"replace_webapp_logo": "file-id"})
+ session.add(tenant)
+ for extension, file_id in [("png", "file-id"), ("txt", "text-id")]:
+ file = _record(extension=extension)
+ session.add(
+ make_upload_file(
+ file_id=file_id,
+ tenant_id=tenant.id,
+ key=file.key,
+ name=file.name,
+ size=file.size,
+ extension=extension,
+ mime_type="image/png" if extension == "png" else "text/plain",
+ )
+ )
+ return UploadFileDeliveryQueryRepository(session_factory=sqlite_session_factory)
@pytest.fixture
-def storage() -> Mock:
- return Mock(spec=UploadFileStorage)
+def storage(tmp_path: Path) -> Storage:
+ storage = Storage()
+ storage.storage_runner = OpenDALStorage(scheme="fs", root=str(tmp_path))
+ for extension in ("png", "txt"):
+ storage.save(_record(extension=extension).key, b"content")
+ return storage
@pytest.fixture
-def service(files: Mock, storage: Mock) -> UploadFileDeliveryService:
+def service(files: UploadFileDeliveryQueryRepository, storage: Storage) -> UploadFileDeliveryService:
return UploadFileDeliveryService(files=files, storage=storage)
def test_invalid_image_signature_does_not_query_or_load_file(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ files: UploadFileDeliveryQueryRepository,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
+ lookup = mocker.spy(files, "get_by_id")
+ load_stream = mocker.spy(storage, "load_stream")
+ load_once = mocker.spy(storage, "load_once")
with patch("services.upload_file_delivery_service.file_helpers.verify_image_signature", return_value=False):
with pytest.raises(UploadFileDeliveryNotFoundError):
service.get_signed_image_preview(file_id="file-id", timestamp="1", nonce="nonce", sign="invalid")
- files.get_by_id.assert_not_called()
- storage.load_stream.assert_not_called()
- storage.load_once.assert_not_called()
+ lookup.assert_not_called()
+ load_stream.assert_not_called()
+ load_once.assert_not_called()
def test_invalid_file_signature_does_not_query_or_load_file(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ files: UploadFileDeliveryQueryRepository,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
+ lookup = mocker.spy(files, "get_by_id")
+ load_stream = mocker.spy(storage, "load_stream")
+ load_once = mocker.spy(storage, "load_once")
with patch("services.upload_file_delivery_service.file_helpers.verify_file_signature", return_value=False):
with pytest.raises(UploadFileDeliveryNotFoundError):
service.get_signed_file_preview(file_id="file-id", timestamp="1", nonce="nonce", sign="invalid")
- files.get_by_id.assert_not_called()
- storage.load_stream.assert_not_called()
- storage.load_once.assert_not_called()
+ lookup.assert_not_called()
+ load_stream.assert_not_called()
+ load_once.assert_not_called()
def test_signed_image_preview_loads_image_stream(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ files: UploadFileDeliveryQueryRepository,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
- file = _record()
- content = iter((b"content",))
- files.get_by_id.return_value = file
- storage.load_stream.return_value = content
-
+ lookup = mocker.spy(files, "get_by_id")
+ load_stream = mocker.spy(storage, "load_stream")
with patch("services.upload_file_delivery_service.file_helpers.verify_image_signature", return_value=True):
result = service.get_signed_image_preview(file_id="file-id", timestamp="1", nonce="nonce", sign="valid")
- assert result == UploadFileDelivery(content=content, file=file)
- files.get_by_id.assert_called_once_with(file_id="file-id")
- storage.load_stream.assert_called_once_with(file.key)
+ assert result.file == _record()
+ assert not isinstance(result.content, bytes)
+ assert b"".join(result.content) == b"content"
+ lookup.assert_called_once_with(file_id="file-id")
+ load_stream.assert_called_once_with(_record().key)
def test_signed_image_preview_rejects_non_image_before_loading(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
- files.get_by_id.return_value = _record(extension="txt")
-
+ load_stream = mocker.spy(storage, "load_stream")
with patch("services.upload_file_delivery_service.file_helpers.verify_image_signature", return_value=True):
with pytest.raises(UnsupportedFileTypeError):
- service.get_signed_image_preview(file_id="file-id", timestamp="1", nonce="nonce", sign="valid")
+ service.get_signed_image_preview(file_id="text-id", timestamp="1", nonce="nonce", sign="valid")
- storage.load_stream.assert_not_called()
+ load_stream.assert_not_called()
def test_signed_file_preview_allows_non_image(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
- file = _record(extension="txt")
- content = iter((b"content",))
- files.get_by_id.return_value = file
- storage.load_stream.return_value = content
-
+ load_stream = mocker.spy(storage, "load_stream")
with patch("services.upload_file_delivery_service.file_helpers.verify_file_signature", return_value=True):
- result = service.get_signed_file_preview(file_id="file-id", timestamp="1", nonce="nonce", sign="valid")
+ result = service.get_signed_file_preview(file_id="text-id", timestamp="1", nonce="nonce", sign="valid")
- assert result == UploadFileDelivery(content=content, file=file)
- storage.load_stream.assert_called_once_with(file.key)
+ assert result.file == _record(extension="txt")
+ assert not isinstance(result.content, bytes)
+ assert b"".join(result.content) == b"content"
+ load_stream.assert_called_once_with(_record(extension="txt").key)
def test_signed_file_preview_reports_missing_file(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
- files.get_by_id.return_value = None
-
+ load_stream = mocker.spy(storage, "load_stream")
with patch("services.upload_file_delivery_service.file_helpers.verify_file_signature", return_value=True):
with pytest.raises(UploadFileDeliveryNotFoundError):
service.get_signed_file_preview(file_id="missing", timestamp="1", nonce="nonce", sign="valid")
- storage.load_stream.assert_not_called()
+ load_stream.assert_not_called()
def test_workspace_logo_loads_content_once(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ files: UploadFileDeliveryQueryRepository,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
- file = _record()
- files.get_workspace_logo.return_value = file
- storage.load_once.return_value = b"content"
+ lookup = mocker.spy(files, "get_workspace_logo")
+ load_once = mocker.spy(storage, "load_once")
+ load_stream = mocker.spy(storage, "load_stream")
result = service.get_workspace_webapp_logo(workspace_id="workspace-id")
- assert result == UploadFileDelivery(content=b"content", file=file)
- files.get_workspace_logo.assert_called_once_with(workspace_id="workspace-id")
- storage.load_once.assert_called_once_with(file.key)
- storage.load_stream.assert_not_called()
+ assert result == UploadFileDelivery(content=b"content", file=_record())
+ lookup.assert_called_once_with(workspace_id="workspace-id")
+ load_once.assert_called_once_with(_record().key)
+ load_stream.assert_not_called()
def test_workspace_logo_reports_missing_file(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ storage: Storage,
+ sqlite_session_factory: sessionmaker[Session],
+ mocker: MockerFixture,
) -> None:
- files.get_workspace_logo.return_value = None
+ with sqlite_session_factory.begin() as session:
+ tenant = make_tenant(tenant_id="missing-logo")
+ tenant.custom_config = json.dumps({"replace_webapp_logo": "missing-file"})
+ session.add(tenant)
+ load_once = mocker.spy(storage, "load_once")
with pytest.raises(UploadFileDeliveryNotFoundError):
- service.get_workspace_webapp_logo(workspace_id="workspace-id")
+ service.get_workspace_webapp_logo(workspace_id="missing-logo")
- storage.load_once.assert_not_called()
+ load_once.assert_not_called()
def test_storage_error_is_not_converted(
service: UploadFileDeliveryService,
- files: Mock,
- storage: Mock,
+ storage: Storage,
+ mocker: MockerFixture,
) -> None:
storage_error = OSError("storage unavailable")
- files.get_by_id.return_value = _record(extension="txt")
- storage.load_stream.side_effect = storage_error
+ mocker.patch.object(storage, "load_stream", side_effect=storage_error)
with patch("services.upload_file_delivery_service.file_helpers.verify_file_signature", return_value=True):
with pytest.raises(OSError) as error_info:
- service.get_signed_file_preview(file_id="file-id", timestamp="1", nonce="nonce", sign="valid")
+ service.get_signed_file_preview(file_id="text-id", timestamp="1", nonce="nonce", sign="valid")
assert error_info.value is storage_error