Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): enable strict typing for datasource manager tests (#44032)

Eddy ZHANG4de1be5342c6
diff --git a/api/tests/unit_tests/.ruff.toml b/api/tests/unit_tests/.ruff.toml
index 5751c9fa594..1be93700804 100644
--- a/api/tests/unit_tests/.ruff.toml
+++ b/api/tests/unit_tests/.ruff.toml
@@ -130,7 +130,6 @@ extend-select = ["ANN401", "ARG"]
"core/callback_handler/test_agent_tool_callback_handler.py" = ["ARG002"]
"core/callback_handler/test_workflow_tool_callback_handler.py" = ["ARG002"]
"core/datasource/__base/test_datasource_provider.py" = ["ARG002"]
-"core/datasource/test_datasource_file_manager.py" = ["ARG001", "ARG002"]
"core/datasource/test_notion_provider.py" = ["ARG002", "TID251"]
"core/datasource/test_website_crawl.py" = ["ARG002"]
"core/datasource/utils/test_message_transformer.py" = ["ARG002"]
diff --git a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py
index d7b4719bae5..5aeac8de816 100644
--- a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py
+++ b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py
@@ -1,5 +1,6 @@
+from collections.abc import Callable
from datetime import UTC, datetime
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from uuid import UUID
import httpx
@@ -7,6 +8,7 @@ import pytest
from sqlalchemy.orm import Session
from core.datasource.datasource_file_manager import DatasourceFileManager
+from graphon.file import FileTransferMethod, FileType
from models.enums import CreatorUserRole
from models.model import MessageFile, UploadFile
from models.tools import ToolFile
@@ -43,8 +45,8 @@ def _tool_file(id: str, *, key: str = "tool_key", mimetype: str = "image/png") -
def _message_file(id: str, *, url: str | None) -> MessageFile:
message_file = MessageFile(
message_id="message-1",
- type="image",
- transfer_method="remote_url",
+ type=FileType.IMAGE,
+ transfer_method=FileTransferMethod.REMOTE_URL,
created_by_role=CreatorUserRole.ACCOUNT,
created_by="user-1",
url=url,
@@ -55,7 +57,7 @@ def _message_file(id: str, *, url: str | None) -> MessageFile:
class TestDatasourceFileManager:
@pytest.fixture(autouse=True)
- def _file_config(self, config_overrides) -> None:
+ def _file_config(self, config_overrides: Callable[..., None]) -> None:
config_overrides(
FILES_URL="http://localhost:5001",
SECRET_KEY="test_secret",
@@ -64,7 +66,7 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.time.time")
@patch("core.datasource.datasource_file_manager.os.urandom")
- def test_sign_file(self, mock_urandom, mock_time):
+ def test_sign_file(self, mock_urandom: MagicMock, mock_time: MagicMock) -> None:
# Setup
mock_time.return_value = 1700000000
mock_urandom.return_value = b"1234567890abcdef" # 16 bytes
@@ -83,7 +85,7 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.storage")
@patch("core.datasource.datasource_file_manager.uuid4")
- def test_create_file_by_raw(self, mock_uuid, mock_storage, sqlite_session: Session):
+ def test_create_file_by_raw(self, mock_uuid: MagicMock, mock_storage: MagicMock, sqlite_session: Session) -> None:
# Setup
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
@@ -114,9 +116,9 @@ class TestDatasourceFileManager:
assert persisted_file is not None
assert persisted_file.key == upload_file.key
- @patch("core.datasource.datasource_file_manager.storage")
+ @patch("core.datasource.datasource_file_manager.storage", new=MagicMock())
@patch("core.datasource.datasource_file_manager.uuid4")
- def test_create_file_by_raw_filename_no_extension(self, mock_uuid, mock_storage):
+ def test_create_file_by_raw_filename_no_extension(self, mock_uuid: MagicMock) -> None:
# Setup
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
@@ -138,10 +140,10 @@ class TestDatasourceFileManager:
# Verify
assert upload_file.name == "test.png" # Should append extension
- @patch("core.datasource.datasource_file_manager.storage")
+ @patch("core.datasource.datasource_file_manager.storage", new=MagicMock())
@patch("core.datasource.datasource_file_manager.uuid4")
@patch("core.datasource.datasource_file_manager.guess_extension")
- def test_create_file_by_raw_unknown_extension(self, mock_guess_ext, mock_uuid, mock_storage):
+ def test_create_file_by_raw_unknown_extension(self, mock_guess_ext: MagicMock, mock_uuid: MagicMock) -> None:
# Setup
mock_guess_ext.return_value = None # Cannot guess
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
@@ -159,9 +161,9 @@ class TestDatasourceFileManager:
assert upload_file.extension == ".bin"
assert upload_file.name == "0123456789abcdef0123456789abcdef.bin"
- @patch("core.datasource.datasource_file_manager.storage")
+ @patch("core.datasource.datasource_file_manager.storage", new=MagicMock())
@patch("core.datasource.datasource_file_manager.uuid4")
- def test_create_file_by_raw_no_filename(self, mock_uuid, mock_storage):
+ def test_create_file_by_raw_no_filename(self, mock_uuid: MagicMock) -> None:
# Setup
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
@@ -179,9 +181,9 @@ class TestDatasourceFileManager:
assert upload_file.extension == ".pdf"
@patch("core.datasource.datasource_file_manager.remote_fetcher")
- @patch("core.datasource.datasource_file_manager.storage")
+ @patch("core.datasource.datasource_file_manager.storage", new=MagicMock())
@patch("core.datasource.datasource_file_manager.uuid4")
- def test_create_file_by_url_mimetype_from_guess(self, mock_uuid, mock_storage, mock_ssrf):
+ def test_create_file_by_url_mimetype_from_guess(self, mock_uuid: MagicMock, mock_ssrf: MagicMock) -> None:
# Setup
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
mock_response = httpx.Response(
@@ -200,9 +202,9 @@ class TestDatasourceFileManager:
assert tool_file.mimetype == "image/png" # Guessed from .png in URL
@patch("core.datasource.datasource_file_manager.remote_fetcher")
- @patch("core.datasource.datasource_file_manager.storage")
+ @patch("core.datasource.datasource_file_manager.storage", new=MagicMock())
@patch("core.datasource.datasource_file_manager.uuid4")
- def test_create_file_by_url_mimetype_default(self, mock_uuid, mock_storage, mock_ssrf):
+ def test_create_file_by_url_mimetype_default(self, mock_uuid: MagicMock, mock_ssrf: MagicMock) -> None:
# Setup
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
mock_response = httpx.Response(
@@ -223,7 +225,9 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.remote_fetcher")
@patch("core.datasource.datasource_file_manager.storage")
@patch("core.datasource.datasource_file_manager.uuid4")
- def test_create_file_by_url_success(self, mock_uuid, mock_storage, mock_ssrf):
+ def test_create_file_by_url_success(
+ self, mock_uuid: MagicMock, mock_storage: MagicMock, mock_ssrf: MagicMock
+ ) -> None:
# Setup
mock_uuid.return_value = UUID("0123456789abcdef0123456789abcdef")
mock_response = httpx.Response(
@@ -248,7 +252,7 @@ class TestDatasourceFileManager:
mock_storage.save.assert_called_once()
@patch("core.datasource.datasource_file_manager.remote_fetcher")
- def test_create_file_by_url_timeout(self, mock_ssrf):
+ def test_create_file_by_url_timeout(self, mock_ssrf: MagicMock) -> None:
# Setup
mock_ssrf.make_request.side_effect = httpx.TimeoutException("Timeout")
@@ -259,7 +263,7 @@ class TestDatasourceFileManager:
)
@patch("core.datasource.datasource_file_manager.storage")
- def test_get_file_binary(self, mock_storage, sqlite_session: Session):
+ def test_get_file_binary(self, mock_storage: MagicMock, sqlite_session: Session) -> None:
sqlite_session.add(_upload_file("file_id", key="some_key", mime_type="image/png"))
sqlite_session.commit()
@@ -273,7 +277,7 @@ class TestDatasourceFileManager:
assert DatasourceFileManager.get_file_binary("unknown") is None
@patch("core.datasource.datasource_file_manager.storage")
- def test_get_file_binary_by_message_file_id(self, mock_storage, sqlite_session: Session):
+ def test_get_file_binary_by_message_file_id(self, mock_storage: MagicMock, sqlite_session: Session) -> None:
sqlite_session.add_all(
[
_message_file("msg_file_id", url="http://localhost/files/tools/tool_id.png"),
@@ -289,7 +293,9 @@ class TestDatasourceFileManager:
assert result == (b"tool content", "image/png")
@patch("core.datasource.datasource_file_manager.storage")
- def test_get_file_binary_by_message_file_id_with_extension(self, mock_storage, sqlite_session: Session):
+ def test_get_file_binary_by_message_file_id_with_extension(
+ self, mock_storage: MagicMock, sqlite_session: Session
+ ) -> None:
sqlite_session.add_all(
[_message_file("m", url="http://localhost/files/tools/abcdef.png"), _tool_file("abcdef", key="tk")]
)
@@ -299,8 +305,8 @@ class TestDatasourceFileManager:
result = DatasourceFileManager.get_file_binary_by_message_file_id("m")
assert result == (b"bits", "image/png")
- @patch("core.datasource.datasource_file_manager.storage")
- def test_get_file_binary_by_message_file_id_failures(self, mock_storage, sqlite_session: Session):
+ @patch("core.datasource.datasource_file_manager.storage", new=MagicMock())
+ def test_get_file_binary_by_message_file_id_failures(self, sqlite_session: Session) -> None:
assert DatasourceFileManager.get_file_binary_by_message_file_id("none") is None
sqlite_session.add(_message_file("msg_id", url=None))
@@ -308,7 +314,7 @@ class TestDatasourceFileManager:
assert DatasourceFileManager.get_file_binary_by_message_file_id("msg_id") is None
@patch("core.datasource.datasource_file_manager.storage")
- def test_get_file_generator_by_upload_file_id(self, mock_storage, sqlite_session: Session):
+ def test_get_file_generator_by_upload_file_id(self, mock_storage: MagicMock, sqlite_session: Session) -> None:
sqlite_session.add(_upload_file("upload_id", key="upload_key", mime_type="text/plain"))
sqlite_session.commit()
@@ -319,6 +325,7 @@ class TestDatasourceFileManager:
# Verify
assert mimetype == "text/plain"
+ assert stream is not None
assert list(stream) == [b"chunk1", b"chunk2"]
stream, mimetype = DatasourceFileManager.get_file_generator_by_upload_file_id("none")
diff --git a/api/tests/unit_tests/core/datasource/test_datasource_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_manager.py
index f4e129c6ddd..5c6e0805abf 100644
--- a/api/tests/unit_tests/core/datasource/test_datasource_manager.py
+++ b/api/tests/unit_tests/core/datasource/test_datasource_manager.py
@@ -1,4 +1,4 @@
-from collections.abc import Generator
+from collections.abc import Callable, Generator
from datetime import UTC, datetime
import pytest
@@ -19,11 +19,13 @@ from core.datasource.entities.datasource_entities import (
OnlineDriveDownloadFileRequest,
)
from core.datasource.errors import DatasourceProviderNotFoundError
+from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
from core.datasource.online_document.online_document_provider import OnlineDocumentDatasourcePluginProviderController
from core.datasource.online_drive.online_drive_provider import OnlineDriveDatasourcePluginProviderController
from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity
from core.tools.entities.common_entities import I18nObject
from core.workflow.file_reference import parse_file_reference
+from core.workflow.nodes.datasource.entities import DatasourceParameter, OnlineDriveDownloadFileParam
from extensions.storage.storage_type import StorageType
from graphon.enums import WorkflowNodeExecutionStatus
from graphon.file import File, FileTransferMethod, FileType
@@ -55,7 +57,9 @@ def provider_entity() -> PluginDatasourceProviderEntity:
@pytest.fixture
-def provider_controller(provider_entity):
+def provider_controller(
+ provider_entity: PluginDatasourceProviderEntity,
+) -> OnlineDocumentDatasourcePluginProviderController:
return OnlineDocumentDatasourcePluginProviderController(
entity=provider_entity.declaration,
plugin_id=provider_entity.plugin_id,
@@ -65,7 +69,9 @@ def provider_controller(provider_entity):
@pytest.fixture
-def document_runtime(provider_controller):
+def document_runtime(
+ provider_controller: OnlineDocumentDatasourcePluginProviderController,
+) -> OnlineDocumentDatasourcePlugin:
return provider_controller.get_datasource("ds")
@@ -130,6 +136,11 @@ def _gen_messages_text_only(text: str) -> Generator[DatasourceMessage]:
)
+def _text(message: DatasourceMessage) -> str:
+ assert isinstance(message.message, DatasourceMessage.TextMessage)
+ return message.message.text
+
+
def _drain_generator(gen: Generator[DatasourceMessage, None, object]) -> tuple[list[DatasourceMessage], object | None]:
messages: list[DatasourceMessage] = []
try:
@@ -146,10 +157,12 @@ def _invalidate_recyclable_contextvars() -> None:
RecyclableContextVar.increment_thread_recycles()
-def test_get_icon_url_calls_runtime(mocker: MockerFixture, document_runtime, config_overrides):
+def test_get_icon_url_calls_runtime(
+ mocker: MockerFixture, document_runtime: OnlineDocumentDatasourcePlugin, config_overrides: Callable[..., None]
+) -> None:
config_overrides(CONSOLE_API_URL="https://console.example.com")
fake_runtime = document_runtime
- mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)
+ get_runtime = mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)
url = DatasourceManager.get_icon_url(
provider_id="p/x",
@@ -160,10 +173,12 @@ def test_get_icon_url_calls_runtime(mocker: MockerFixture, document_runtime, con
assert (
url == "https://console.example.com/console/api/workspaces/current/plugin/icon?tenant_id=t1&filename=icon.png"
)
- DatasourceManager.get_datasource_runtime.assert_called_once()
+ get_runtime.assert_called_once()
-def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerFixture, provider_controller):
+def test_get_datasource_runtime_delegates_to_provider_controller(
+ mocker: MockerFixture, provider_controller: OnlineDocumentDatasourcePluginProviderController
+) -> None:
get_datasource = mocker.spy(provider_controller, "get_datasource")
mocker.patch.object(DatasourceManager, "get_datasource_plugin_provider", return_value=provider_controller)
@@ -199,8 +214,11 @@ def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerF
],
)
def test_get_datasource_plugin_provider_creates_controller_and_caches(
- mocker: MockerFixture, datasource_type, controller_path, provider_entity
-):
+ mocker: MockerFixture,
+ datasource_type: DatasourceProviderType,
+ controller_path: str,
+ provider_entity: PluginDatasourceProviderEntity,
+) -> None:
_invalidate_recyclable_contextvars()
fetch = mocker.patch(
@@ -225,7 +243,7 @@ def test_get_datasource_plugin_provider_creates_controller_and_caches(
assert ctrl_cls.call_count == 1
-def test_get_datasource_plugin_provider_raises_when_provider_entity_missing(mocker: MockerFixture):
+def test_get_datasource_plugin_provider_raises_when_provider_entity_missing(mocker: MockerFixture) -> None:
_invalidate_recyclable_contextvars()
mocker.patch(
"core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",
@@ -240,7 +258,9 @@ def test_get_datasource_plugin_provider_raises_when_provider_entity_missing(mock
)
-def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: MockerFixture, provider_entity):
+def test_get_datasource_plugin_provider_raises_for_unsupported_type(
+ mocker: MockerFixture, provider_entity: PluginDatasourceProviderEntity
+) -> None:
_invalidate_recyclable_contextvars()
mocker.patch(
"core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",
@@ -251,11 +271,13 @@ def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: Mock
DatasourceManager.get_datasource_plugin_provider(
provider_id="prov/x",
tenant_id="t1",
- datasource_type="unsupported", # not a DatasourceProviderType at runtime
+ datasource_type="unsupported", # pyrefly: ignore[bad-argument-type] # not a DatasourceProviderType
)
-def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: MockerFixture, provider_entity):
+def test_get_datasource_plugin_provider_raises_when_controller_none(
+ mocker: MockerFixture, provider_entity: PluginDatasourceProviderEntity
+) -> None:
_invalidate_recyclable_contextvars()
mocker.patch(
"core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",
@@ -274,9 +296,11 @@ def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: Mock
)
-def test_stream_online_results_yields_messages_online_document(mocker: MockerFixture, document_runtime, monkeypatch):
+def test_stream_online_results_yields_messages_online_document(
+ mocker: MockerFixture, document_runtime: OnlineDocumentDatasourcePlugin, monkeypatch: pytest.MonkeyPatch
+) -> None:
# stub runtime to yield a text message
- def _doc_messages(**_):
+ def _doc_messages(**_) -> Generator[DatasourceMessage]:
yield from _gen_messages_text_only("hello")
fake_runtime = document_runtime
@@ -289,21 +313,26 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFix
datasource_type="online_document",
provider_id="p/x",
tenant_id="t1",
- datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=DatasourceParameter(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
msgs = list(gen)
assert len(msgs) == 1
- assert msgs[0].message.text == "hello"
+ assert _text(msgs[0]) == "hello"
def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_document(
- mocker: MockerFixture, document_runtime, monkeypatch
-):
+ mocker: MockerFixture, document_runtime: OnlineDocumentDatasourcePlugin, monkeypatch: pytest.MonkeyPatch
+) -> None:
runtime = document_runtime
- def page_content(**_kwargs):
+ received: list[GetOnlineDocumentPageContentRequest] = []
+
+ def page_content(
+ *, datasource_parameters: GetOnlineDocumentPageContentRequest, **_kwargs: object
+ ) -> Generator[DatasourceMessage]:
+ received.append(datasource_parameters)
yield from _gen_messages_text_only("hello")
monkeypatch.setattr(runtime, "get_online_document_page_content", page_content)
@@ -315,18 +344,21 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_do
datasource_type="online_document",
provider_id="p/x",
tenant_id="t1",
- datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=DatasourceParameter(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={"token": "t"},
)
messages, final_value = _drain_generator(gen)
assert runtime.runtime.credentials == {"token": "t"}
- assert [m.message.text for m in messages] == ["hello"]
+ assert received == [GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t")]
+ assert [_text(m) for m in messages] == ["hello"]
assert final_value == {}
-def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture, document_runtime):
+def test_stream_online_results_raises_when_missing_params(
+ mocker: MockerFixture, document_runtime: OnlineDocumentDatasourcePlugin
+) -> None:
mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=document_runtime)
with pytest.raises(ValueError, match="datasource_param is required for ONLINE_DOCUMENT streaming"):
@@ -359,8 +391,8 @@ def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture,
def test_stream_online_results_yields_messages_and_returns_empty_dict_online_drive(
- mocker: MockerFixture, provider_entity, monkeypatch
-):
+ mocker: MockerFixture, provider_entity: PluginDatasourceProviderEntity, monkeypatch: pytest.MonkeyPatch
+) -> None:
controller = OnlineDriveDatasourcePluginProviderController(
entity=provider_entity.declaration,
plugin_id=provider_entity.plugin_id,
@@ -369,7 +401,10 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_dri
)
runtime = controller.get_datasource("ds")
- def download_file(**_kwargs):
+ received: list[OnlineDriveDownloadFileRequest] = []
+
+ def download_file(*, request: OnlineDriveDownloadFileRequest, **_kwargs: object) -> Generator[DatasourceMessage]:
+ received.append(request)
yield from _gen_messages_text_only("drive")
monkeypatch.setattr(runtime, "online_drive_download_file", download_file)
@@ -382,17 +417,20 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_dri
provider_id="p/x",
tenant_id="t1",
datasource_param=None,
- online_drive_request=OnlineDriveDownloadFileRequest(id="fid", bucket="b"),
+ online_drive_request=OnlineDriveDownloadFileParam(id="fid", bucket="b", result=[]),
credentials={"token": "t"},
)
messages, final_value = _drain_generator(gen)
assert runtime.runtime.credentials == {"token": "t"}
- assert [m.message.text for m in messages] == ["drive"]
+ assert received == [OnlineDriveDownloadFileRequest(id="fid", bucket="b")]
+ assert [_text(m) for m in messages] == ["drive"]
assert final_value == {}
-def test_stream_online_results_raises_for_unsupported_stream_type(mocker: MockerFixture, document_runtime):
+def test_stream_online_results_raises_for_unsupported_stream_type(
+ mocker: MockerFixture, document_runtime: OnlineDocumentDatasourcePlugin
+) -> None:
mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=document_runtime)
with pytest.raises(ValueError, match="Unsupported datasource type for streaming"):
@@ -410,7 +448,7 @@ def test_stream_online_results_raises_for_unsupported_stream_type(mocker: Mocker
)
-def test_stream_node_events_emits_events_online_document(mocker: MockerFixture):
+def test_stream_node_events_emits_events_online_document(mocker: MockerFixture) -> None:
# make manager's low-level stream produce TEXT only
mocker.patch.object(
DatasourceManager,
@@ -429,7 +467,7 @@ def test_stream_node_events_emits_events_online_document(mocker: MockerFixture):
parameters_for_log={"k": "v"},
datasource_info={"user_id": "u1"},
variable_pool=VariablePool(),
- datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=DatasourceParameter(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
@@ -441,11 +479,13 @@ def test_stream_node_events_emits_events_online_document(mocker: MockerFixture):
assert events[-1].node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
-def test_stream_node_events_builds_file_and_variables_from_messages(mocker: MockerFixture, tool_file_session: Session):
+def test_stream_node_events_builds_file_and_variables_from_messages(
+ mocker: MockerFixture, tool_file_session: Session
+) -> None:
_persist_tool_file(tool_file_session, file_id="tool_file_1", tenant_id="t1")
mocker.patch.object(DatasourceManager, "stream_online_results", return_value=_gen_messages_text_only("ignored"))
- def _transformed(**_kwargs):
+ def _transformed(**_kwargs) -> Generator[DatasourceMessage]:
yield DatasourceMessage(
type=DatasourceMessage.MessageType.IMAGE_LINK,
message=DatasourceMessage.TextMessage(text="/files/datasources/tool_file_1.png"),
@@ -514,7 +554,7 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mock
parameters_for_log={"k": "v"},
datasource_info={"info": "x"},
variable_pool=variable_pool,
- datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=DatasourceParameter(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
@@ -535,11 +575,11 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mock
assert events[-1].node_run_result.outputs["x"] == 1
-def test_stream_node_events_raises_when_toolfile_missing(mocker: MockerFixture, tool_file_session: Session):
+def test_stream_node_events_raises_when_toolfile_missing(mocker: MockerFixture, tool_file_session: Session) -> None:
_persist_tool_file(tool_file_session, file_id="missing", tenant_id="other-tenant")
mocker.patch.object(DatasourceManager, "stream_online_results", return_value=_gen_messages_text_only("ignored"))
- def _transformed(**_kwargs):
+ def _transformed(**_kwargs) -> Generator[DatasourceMessage]:
yield DatasourceMessage(
type=DatasourceMessage.MessageType.IMAGE_LINK,
message=DatasourceMessage.TextMessage(text="/files/datasources/missing.png"),
@@ -563,14 +603,14 @@ def test_stream_node_events_raises_when_toolfile_missing(mocker: MockerFixture,
parameters_for_log={},
datasource_info={},
variable_pool=VariablePool(),
- datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=DatasourceParameter(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
)
-def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(mocker: MockerFixture):
+def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(mocker: MockerFixture) -> None:
mocker.patch.object(DatasourceManager, "stream_online_results", return_value=_gen_messages_text_only("ignored"))
file_in = File(
@@ -582,7 +622,7 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(moc
storage_key="k",
)
- def _transformed(**_kwargs):
+ def _transformed(**_kwargs) -> Generator[DatasourceMessage]:
yield DatasourceMessage(
type=DatasourceMessage.MessageType.FILE,
message=DatasourceMessage.FileMessage(file_marker="file_marker"),
@@ -607,7 +647,7 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(moc
datasource_info={"k": "v"},
variable_pool=variable_pool,
datasource_param=None,
- online_drive_request=OnlineDriveDownloadFileRequest(id="id", bucket="b"),
+ online_drive_request=OnlineDriveDownloadFileParam(id="id", bucket="b", result=[]),
credentials={},
)
)
@@ -622,10 +662,10 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(moc
assert completed.node_run_result.outputs["datasource_type"] == DatasourceProviderType.ONLINE_DRIVE
-def test_stream_node_events_skips_file_build_for_non_online_types(mocker: MockerFixture):
+def test_stream_node_events_skips_file_build_for_non_online_types(mocker: MockerFixture) -> None:
mocker.patch.object(DatasourceManager, "stream_online_results", return_value=_gen_messages_text_only("ignored"))
- def _transformed(**_kwargs):
+ def _transformed(**_kwargs) -> Generator[DatasourceMessage]:
yield DatasourceMessage(
type=DatasourceMessage.MessageType.IMAGE_LINK,
message=DatasourceMessage.TextMessage(text="/files/datasources/tool_file_1.png"),
@@ -660,17 +700,19 @@ def test_stream_node_events_skips_file_build_for_non_online_types(mocker: Mocker
assert events[-1].node_run_result.outputs["file"] is None
-def test_get_upload_file_by_id_builds_file(datasource_session: Session):
+def test_get_upload_file_by_id_builds_file(datasource_session: Session) -> None:
_persist_upload_file(datasource_session, file_id="fid", tenant_id="t1")
f = DatasourceManager.get_upload_file_by_id(file_id="fid", tenant_id="t1")
assert f.related_id == "fid"
assert f.extension == ".txt"
- assert parse_file_reference(f.reference).storage_key is None
+ reference = parse_file_reference(f.reference)
+ assert reference is not None
+ assert reference.storage_key is None
assert f.storage_key == "files/file.txt"
-def test_get_upload_file_by_id_raises_when_missing(datasource_session: Session):
+def test_get_upload_file_by_id_raises_when_missing(datasource_session: Session) -> None:
_persist_upload_file(datasource_session, file_id="fid", tenant_id="other-tenant")
with pytest.raises(ValueError, match="UploadFile not found for file_id=fid, tenant_id=t1"):
diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml
index b54a2ce586b..c4d737c4820 100644
--- a/api/tests/unit_tests/pyrefly.toml
+++ b/api/tests/unit_tests/pyrefly.toml
@@ -263,8 +263,6 @@ project-excludes = [
"core/datasource/entities/test_datasource_entities.py",
"core/datasource/online_document/test_online_document_plugin.py",
"core/datasource/online_drive/test_online_drive_plugin.py",
- "core/datasource/test_datasource_file_manager.py",
- "core/datasource/test_datasource_manager.py",
"core/datasource/test_notion_provider.py",
"core/datasource/test_website_crawl.py",
"core/entities/test_entities_mcp_provider.py",