Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: use real datasource manager runtime fixtures (#43410)
Asuka Minato324e47189c4a
diff --git a/api/tests/unit_tests/core/datasource/test_datasource_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_manager.pyindex f8ec07e3cd3..f4e129c6ddd 100644--- a/api/tests/unit_tests/core/datasource/test_datasource_manager.py+++ b/api/tests/unit_tests/core/datasource/test_datasource_manager.py@@ -1,4 +1,3 @@-import typesfrom collections.abc import Generatorfrom datetime import UTC, datetime@@ -9,18 +8,67 @@ from sqlalchemy.orm import Session, sessionmakerfrom contexts.wrapper import RecyclableContextVarfrom core.datasource import datasource_manager as datasource_manager_modulefrom core.datasource.datasource_manager import DatasourceManager-from core.datasource.entities.datasource_entities import DatasourceMessage, DatasourceProviderType+from core.datasource.entities.datasource_entities import (+ DatasourceEntity,+ DatasourceIdentity,+ DatasourceMessage,+ DatasourceProviderEntityWithPlugin,+ DatasourceProviderIdentity,+ DatasourceProviderType,+ GetOnlineDocumentPageContentRequest,+ OnlineDriveDownloadFileRequest,+)from core.datasource.errors import DatasourceProviderNotFoundError+from core.datasource.online_document.online_document_provider import OnlineDocumentDatasourcePluginProviderController+from core.datasource.online_drive.online_drive_provider import OnlineDriveDatasourcePluginProviderController+from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity+from core.tools.entities.common_entities import I18nObjectfrom core.workflow.file_reference import parse_file_referencefrom extensions.storage.storage_type import StorageTypefrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.file import File, FileTransferMethod, FileTypefrom graphon.node_events import StreamChunkEvent, StreamCompletedEvent+from graphon.runtime import VariablePoolfrom models.enums import CreatorUserRolefrom models.model import UploadFilefrom models.tools import ToolFile+def provider_entity() -> PluginDatasourceProviderEntity:+ label = I18nObject(en_US="Datasource")+ declaration = DatasourceProviderEntityWithPlugin(+ identity=DatasourceProviderIdentity(+ author="author", name="provider", description=label, icon="icon.png", label=label+ ),+ provider_type=DatasourceProviderType.ONLINE_DOCUMENT,+ datasources=[+ DatasourceEntity(+ identity=DatasourceIdentity(author="author", name="ds", label=label, provider="provider"),+ description=label,+ )+ ],+ )+ return PluginDatasourceProviderEntity(+ provider="provider", declaration=declaration, plugin_id="plugin", plugin_unique_identifier="uniq"+ )+++def provider_controller(provider_entity):+ return OnlineDocumentDatasourcePluginProviderController(+ entity=provider_entity.declaration,+ plugin_id=provider_entity.plugin_id,+ plugin_unique_identifier=provider_entity.plugin_unique_identifier,+ tenant_id="t1",+ )+++def document_runtime(provider_controller):+ return provider_controller.get_datasource("ds")++@pytest.fixturedef datasource_session(sqlite_session: Session,@@ -98,9 +146,9 @@ def _invalidate_recyclable_contextvars() -> None:RecyclableContextVar.increment_thread_recycles()-def test_get_icon_url_calls_runtime(mocker: MockerFixture):- fake_runtime = mocker.Mock()- fake_runtime.get_icon_url.return_value = "https://icon"+def test_get_icon_url_calls_runtime(mocker: MockerFixture, document_runtime, config_overrides):+ config_overrides(CONSOLE_API_URL="https://console.example.com")+ fake_runtime = document_runtimemocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)url = DatasourceManager.get_icon_url(@@ -109,13 +157,14 @@ def test_get_icon_url_calls_runtime(mocker: MockerFixture):datasource_name="ds",datasource_type="online_document",)- assert url == "https://icon"+ assert (+ url == "https://console.example.com/console/api/workspaces/current/plugin/icon?tenant_id=t1&filename=icon.png"+ )DatasourceManager.get_datasource_runtime.assert_called_once()-def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerFixture):- provider_controller = mocker.Mock()- provider_controller.get_datasource.return_value = object()+def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerFixture, provider_controller):+ get_datasource = mocker.spy(provider_controller, "get_datasource")mocker.patch.object(DatasourceManager, "get_datasource_plugin_provider", return_value=provider_controller)runtime = DatasourceManager.get_datasource_runtime(@@ -124,8 +173,8 @@ def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerFtenant_id="t1",datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,)- assert runtime is provider_controller.get_datasource.return_value- provider_controller.get_datasource.assert_called_once_with("ds")+ assert runtime is get_datasource.spy_return+ get_datasource.assert_called_once_with("ds")@pytest.mark.parametrize(@@ -150,11 +199,10 @@ def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerF],)def test_get_datasource_plugin_provider_creates_controller_and_caches(- mocker: MockerFixture, datasource_type, controller_path+ mocker: MockerFixture, datasource_type, controller_path, provider_entity):_invalidate_recyclable_contextvars()- provider_entity = types.SimpleNamespace(declaration=object(), plugin_id="plugin", plugin_unique_identifier="uniq")fetch = mocker.patch("core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",return_value=provider_entity,@@ -192,9 +240,8 @@ def test_get_datasource_plugin_provider_raises_when_provider_entity_missing(mock)-def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: MockerFixture):+def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: MockerFixture, provider_entity):_invalidate_recyclable_contextvars()- provider_entity = types.SimpleNamespace(declaration=object(), plugin_id="plugin", plugin_unique_identifier="uniq")mocker.patch("core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",return_value=provider_entity,@@ -204,13 +251,12 @@ def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: MockDatasourceManager.get_datasource_plugin_provider(provider_id="prov/x",tenant_id="t1",- datasource_type=types.SimpleNamespace(), # not a DatasourceProviderType at runtime+ datasource_type="unsupported", # not a DatasourceProviderType at runtime)-def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: MockerFixture):+def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: MockerFixture, provider_entity):_invalidate_recyclable_contextvars()- provider_entity = types.SimpleNamespace(declaration=object(), plugin_id="plugin", plugin_unique_identifier="uniq")mocker.patch("core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",return_value=provider_entity,@@ -228,13 +274,13 @@ def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: Mock)-def test_stream_online_results_yields_messages_online_document(mocker: MockerFixture):+def test_stream_online_results_yields_messages_online_document(mocker: MockerFixture, document_runtime, monkeypatch):# stub runtime to yield a text messagedef _doc_messages(**_):yield from _gen_messages_text_only("hello")- fake_runtime = mocker.Mock()- fake_runtime.get_online_document_page_content.side_effect = _doc_messages+ fake_runtime = document_runtime+ monkeypatch.setattr(fake_runtime, "get_online_document_page_content", _doc_messages)mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)gen = DatasourceManager.stream_online_results(@@ -243,7 +289,7 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFixdatasource_type="online_document",provider_id="p/x",tenant_id="t1",- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,credentials={},)@@ -252,15 +298,15 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFixassert msgs[0].message.text == "hello"-def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_document(mocker: MockerFixture):- class _Runtime:- def __init__(self) -> None:- self.runtime = types.SimpleNamespace(credentials=None)+def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_document(+ mocker: MockerFixture, document_runtime, monkeypatch+):+ runtime = document_runtime- def get_online_document_page_content(self, **_kwargs):- yield from _gen_messages_text_only("hello")+ def page_content(**_kwargs):+ yield from _gen_messages_text_only("hello")- runtime = _Runtime()+ monkeypatch.setattr(runtime, "get_online_document_page_content", page_content)mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=runtime)gen = DatasourceManager.stream_online_results(@@ -269,7 +315,7 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_dodatasource_type="online_document",provider_id="p/x",tenant_id="t1",- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,credentials={"token": "t"},)@@ -280,18 +326,8 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_doassert final_value == {}-def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture):- class _Runtime:- def __init__(self) -> None:- self.runtime = types.SimpleNamespace(credentials=None)-- def get_online_document_page_content(self, **_kwargs):- yield from _gen_messages_text_only("never")-- def online_drive_download_file(self, **_kwargs):- yield from _gen_messages_text_only("never")-- mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=_Runtime())+def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture, document_runtime):+ mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=document_runtime)with pytest.raises(ValueError, match="datasource_param is required for ONLINE_DOCUMENT streaming"):list(@@ -322,15 +358,21 @@ def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture))-def test_stream_online_results_yields_messages_and_returns_empty_dict_online_drive(mocker: MockerFixture):- class _Runtime:- def __init__(self) -> None:- self.runtime = types.SimpleNamespace(credentials=None)+def test_stream_online_results_yields_messages_and_returns_empty_dict_online_drive(+ mocker: MockerFixture, provider_entity, monkeypatch+):+ controller = OnlineDriveDatasourcePluginProviderController(+ entity=provider_entity.declaration,+ plugin_id=provider_entity.plugin_id,+ plugin_unique_identifier=provider_entity.plugin_unique_identifier,+ tenant_id="t1",+ )+ runtime = controller.get_datasource("ds")- def online_drive_download_file(self, **_kwargs):- yield from _gen_messages_text_only("drive")+ def download_file(**_kwargs):+ yield from _gen_messages_text_only("drive")- runtime = _Runtime()+ monkeypatch.setattr(runtime, "online_drive_download_file", download_file)mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=runtime)gen = DatasourceManager.stream_online_results(@@ -340,7 +382,7 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_driprovider_id="p/x",tenant_id="t1",datasource_param=None,- online_drive_request=types.SimpleNamespace(id="fid", bucket="b"),+ online_drive_request=OnlineDriveDownloadFileRequest(id="fid", bucket="b"),credentials={"token": "t"},)messages, final_value = _drain_generator(gen)@@ -350,8 +392,8 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_driassert final_value == {}-def test_stream_online_results_raises_for_unsupported_stream_type(mocker: MockerFixture):- mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=mocker.Mock())+def test_stream_online_results_raises_for_unsupported_stream_type(mocker: MockerFixture, document_runtime):+ mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=document_runtime)with pytest.raises(ValueError, match="Unsupported datasource type for streaming"):list(@@ -386,8 +428,8 @@ def test_stream_node_events_emits_events_online_document(mocker: MockerFixture):tenant_id="t1",parameters_for_log={"k": "v"},datasource_info={"user_id": "u1"},- variable_pool=mocker.Mock(),- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),+ variable_pool=VariablePool(),+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,credentials={},)@@ -459,7 +501,7 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mockreturn_value=built,)- variable_pool = mocker.Mock()+ variable_pool = VariablePool()events = list(DatasourceManager.stream_node_events(@@ -472,14 +514,14 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mockparameters_for_log={"k": "v"},datasource_info={"info": "x"},variable_pool=variable_pool,- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,credentials={},))build_from_mapping.assert_called_once()- variable_pool.add.assert_not_called()+ assert variable_pool.variable_dictionary == {}assert any(isinstance(e, StreamChunkEvent) and e.chunk == "hello" for e in events)assert any(isinstance(e, StreamChunkEvent) and e.chunk.startswith("Link: http") for e in events)@@ -520,8 +562,8 @@ def test_stream_node_events_raises_when_toolfile_missing(mocker: MockerFixture,tenant_id="t1",parameters_for_log={},datasource_info={},- variable_pool=mocker.Mock(),- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),+ variable_pool=VariablePool(),+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),online_drive_request=None,credentials={},)@@ -552,7 +594,7 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(mocside_effect=_transformed,)- variable_pool = mocker.Mock()+ variable_pool = VariablePool()events = list(DatasourceManager.stream_node_events(node_id="nodeA",@@ -565,14 +607,14 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(mocdatasource_info={"k": "v"},variable_pool=variable_pool,datasource_param=None,- online_drive_request=types.SimpleNamespace(id="id", bucket="b"),+ online_drive_request=OnlineDriveDownloadFileRequest(id="id", bucket="b"),credentials={},))- variable_pool.add.assert_called_once()- assert variable_pool.add.call_args[0][0] == ["nodeA", "file"]- assert variable_pool.add.call_args[0][1] == file_in+ stored_file = variable_pool.get(["nodeA", "file"])+ assert stored_file is not None+ assert stored_file.value == file_incompleted = events[-1]assert isinstance(completed, StreamCompletedEvent)@@ -606,7 +648,7 @@ def test_stream_node_events_skips_file_build_for_non_online_types(mocker: Mockertenant_id="t1",parameters_for_log={},datasource_info={},- variable_pool=mocker.Mock(),+ variable_pool=VariablePool(),datasource_param=None,online_drive_request=None,credentials={},