Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use real datasource manager runtime fixtures (#43410)

Asuka Minato324e47189c4a
diff --git a/api/tests/unit_tests/core/datasource/test_datasource_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_manager.py
index f8ec07e3cd3..f4e129c6ddd 100644
--- a/api/tests/unit_tests/core/datasource/test_datasource_manager.py
+++ b/api/tests/unit_tests/core/datasource/test_datasource_manager.py
@@ -1,4 +1,3 @@
-import types
from collections.abc import Generator
from datetime import UTC, datetime
@@ -9,18 +8,67 @@ from sqlalchemy.orm import Session, sessionmaker
from contexts.wrapper import RecyclableContextVar
from core.datasource import datasource_manager as datasource_manager_module
from core.datasource.datasource_manager import DatasourceManager
-from core.datasource.entities.datasource_entities import DatasourceMessage, DatasourceProviderType
+from core.datasource.entities.datasource_entities import (
+ DatasourceEntity,
+ DatasourceIdentity,
+ DatasourceMessage,
+ DatasourceProviderEntityWithPlugin,
+ DatasourceProviderIdentity,
+ DatasourceProviderType,
+ GetOnlineDocumentPageContentRequest,
+ OnlineDriveDownloadFileRequest,
+)
from core.datasource.errors import DatasourceProviderNotFoundError
+from core.datasource.online_document.online_document_provider import OnlineDocumentDatasourcePluginProviderController
+from core.datasource.online_drive.online_drive_provider import OnlineDriveDatasourcePluginProviderController
+from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity
+from core.tools.entities.common_entities import I18nObject
from core.workflow.file_reference import parse_file_reference
from extensions.storage.storage_type import StorageType
from graphon.enums import WorkflowNodeExecutionStatus
from graphon.file import File, FileTransferMethod, FileType
from graphon.node_events import StreamChunkEvent, StreamCompletedEvent
+from graphon.runtime import VariablePool
from models.enums import CreatorUserRole
from models.model import UploadFile
from models.tools import ToolFile
+def provider_entity() -> PluginDatasourceProviderEntity:
+ label = I18nObject(en_US="Datasource")
+ declaration = DatasourceProviderEntityWithPlugin(
+ identity=DatasourceProviderIdentity(
+ author="author", name="provider", description=label, icon="icon.png", label=label
+ ),
+ provider_type=DatasourceProviderType.ONLINE_DOCUMENT,
+ datasources=[
+ DatasourceEntity(
+ identity=DatasourceIdentity(author="author", name="ds", label=label, provider="provider"),
+ description=label,
+ )
+ ],
+ )
+ return PluginDatasourceProviderEntity(
+ provider="provider", declaration=declaration, plugin_id="plugin", plugin_unique_identifier="uniq"
+ )
+
+
+def provider_controller(provider_entity):
+ return OnlineDocumentDatasourcePluginProviderController(
+ entity=provider_entity.declaration,
+ plugin_id=provider_entity.plugin_id,
+ plugin_unique_identifier=provider_entity.plugin_unique_identifier,
+ tenant_id="t1",
+ )
+
+
+def document_runtime(provider_controller):
+ return provider_controller.get_datasource("ds")
+
+
@pytest.fixture
def datasource_session(
sqlite_session: Session,
@@ -98,9 +146,9 @@ def _invalidate_recyclable_contextvars() -> None:
RecyclableContextVar.increment_thread_recycles()
-def test_get_icon_url_calls_runtime(mocker: MockerFixture):
- fake_runtime = mocker.Mock()
- fake_runtime.get_icon_url.return_value = "https://icon"
+def test_get_icon_url_calls_runtime(mocker: MockerFixture, document_runtime, config_overrides):
+ config_overrides(CONSOLE_API_URL="https://console.example.com")
+ fake_runtime = document_runtime
mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)
url = DatasourceManager.get_icon_url(
@@ -109,13 +157,14 @@ def test_get_icon_url_calls_runtime(mocker: MockerFixture):
datasource_name="ds",
datasource_type="online_document",
)
- assert url == "https://icon"
+ assert (
+ url == "https://console.example.com/console/api/workspaces/current/plugin/icon?tenant_id=t1&filename=icon.png"
+ )
DatasourceManager.get_datasource_runtime.assert_called_once()
-def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerFixture):
- provider_controller = mocker.Mock()
- provider_controller.get_datasource.return_value = object()
+def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerFixture, provider_controller):
+ get_datasource = mocker.spy(provider_controller, "get_datasource")
mocker.patch.object(DatasourceManager, "get_datasource_plugin_provider", return_value=provider_controller)
runtime = DatasourceManager.get_datasource_runtime(
@@ -124,8 +173,8 @@ def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerF
tenant_id="t1",
datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
)
- assert runtime is provider_controller.get_datasource.return_value
- provider_controller.get_datasource.assert_called_once_with("ds")
+ assert runtime is get_datasource.spy_return
+ get_datasource.assert_called_once_with("ds")
@pytest.mark.parametrize(
@@ -150,11 +199,10 @@ def test_get_datasource_runtime_delegates_to_provider_controller(mocker: MockerF
],
)
def test_get_datasource_plugin_provider_creates_controller_and_caches(
- mocker: MockerFixture, datasource_type, controller_path
+ mocker: MockerFixture, datasource_type, controller_path, provider_entity
):
_invalidate_recyclable_contextvars()
- provider_entity = types.SimpleNamespace(declaration=object(), plugin_id="plugin", plugin_unique_identifier="uniq")
fetch = mocker.patch(
"core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",
return_value=provider_entity,
@@ -192,9 +240,8 @@ def test_get_datasource_plugin_provider_raises_when_provider_entity_missing(mock
)
-def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: MockerFixture):
+def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: MockerFixture, provider_entity):
_invalidate_recyclable_contextvars()
- provider_entity = types.SimpleNamespace(declaration=object(), plugin_id="plugin", plugin_unique_identifier="uniq")
mocker.patch(
"core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",
return_value=provider_entity,
@@ -204,13 +251,12 @@ def test_get_datasource_plugin_provider_raises_for_unsupported_type(mocker: Mock
DatasourceManager.get_datasource_plugin_provider(
provider_id="prov/x",
tenant_id="t1",
- datasource_type=types.SimpleNamespace(), # not a DatasourceProviderType at runtime
+ datasource_type="unsupported", # not a DatasourceProviderType at runtime
)
-def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: MockerFixture):
+def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: MockerFixture, provider_entity):
_invalidate_recyclable_contextvars()
- provider_entity = types.SimpleNamespace(declaration=object(), plugin_id="plugin", plugin_unique_identifier="uniq")
mocker.patch(
"core.datasource.datasource_manager.PluginDatasourceManager.fetch_datasource_provider",
return_value=provider_entity,
@@ -228,13 +274,13 @@ def test_get_datasource_plugin_provider_raises_when_controller_none(mocker: Mock
)
-def test_stream_online_results_yields_messages_online_document(mocker: MockerFixture):
+def test_stream_online_results_yields_messages_online_document(mocker: MockerFixture, document_runtime, monkeypatch):
# stub runtime to yield a text message
def _doc_messages(**_):
yield from _gen_messages_text_only("hello")
- fake_runtime = mocker.Mock()
- fake_runtime.get_online_document_page_content.side_effect = _doc_messages
+ fake_runtime = document_runtime
+ monkeypatch.setattr(fake_runtime, "get_online_document_page_content", _doc_messages)
mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=fake_runtime)
gen = DatasourceManager.stream_online_results(
@@ -243,7 +289,7 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFix
datasource_type="online_document",
provider_id="p/x",
tenant_id="t1",
- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
@@ -252,15 +298,15 @@ def test_stream_online_results_yields_messages_online_document(mocker: MockerFix
assert msgs[0].message.text == "hello"
-def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_document(mocker: MockerFixture):
- class _Runtime:
- def __init__(self) -> None:
- self.runtime = types.SimpleNamespace(credentials=None)
+def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_document(
+ mocker: MockerFixture, document_runtime, monkeypatch
+):
+ runtime = document_runtime
- def get_online_document_page_content(self, **_kwargs):
- yield from _gen_messages_text_only("hello")
+ def page_content(**_kwargs):
+ yield from _gen_messages_text_only("hello")
- runtime = _Runtime()
+ monkeypatch.setattr(runtime, "get_online_document_page_content", page_content)
mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=runtime)
gen = DatasourceManager.stream_online_results(
@@ -269,7 +315,7 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_do
datasource_type="online_document",
provider_id="p/x",
tenant_id="t1",
- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={"token": "t"},
)
@@ -280,18 +326,8 @@ def test_stream_online_results_sets_credentials_and_returns_empty_dict_online_do
assert final_value == {}
-def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture):
- class _Runtime:
- def __init__(self) -> None:
- self.runtime = types.SimpleNamespace(credentials=None)
-
- def get_online_document_page_content(self, **_kwargs):
- yield from _gen_messages_text_only("never")
-
- def online_drive_download_file(self, **_kwargs):
- yield from _gen_messages_text_only("never")
-
- mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=_Runtime())
+def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture, document_runtime):
+ mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=document_runtime)
with pytest.raises(ValueError, match="datasource_param is required for ONLINE_DOCUMENT streaming"):
list(
@@ -322,15 +358,21 @@ def test_stream_online_results_raises_when_missing_params(mocker: MockerFixture)
)
-def test_stream_online_results_yields_messages_and_returns_empty_dict_online_drive(mocker: MockerFixture):
- class _Runtime:
- def __init__(self) -> None:
- self.runtime = types.SimpleNamespace(credentials=None)
+def test_stream_online_results_yields_messages_and_returns_empty_dict_online_drive(
+ mocker: MockerFixture, provider_entity, monkeypatch
+):
+ controller = OnlineDriveDatasourcePluginProviderController(
+ entity=provider_entity.declaration,
+ plugin_id=provider_entity.plugin_id,
+ plugin_unique_identifier=provider_entity.plugin_unique_identifier,
+ tenant_id="t1",
+ )
+ runtime = controller.get_datasource("ds")
- def online_drive_download_file(self, **_kwargs):
- yield from _gen_messages_text_only("drive")
+ def download_file(**_kwargs):
+ yield from _gen_messages_text_only("drive")
- runtime = _Runtime()
+ monkeypatch.setattr(runtime, "online_drive_download_file", download_file)
mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=runtime)
gen = DatasourceManager.stream_online_results(
@@ -340,7 +382,7 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_dri
provider_id="p/x",
tenant_id="t1",
datasource_param=None,
- online_drive_request=types.SimpleNamespace(id="fid", bucket="b"),
+ online_drive_request=OnlineDriveDownloadFileRequest(id="fid", bucket="b"),
credentials={"token": "t"},
)
messages, final_value = _drain_generator(gen)
@@ -350,8 +392,8 @@ def test_stream_online_results_yields_messages_and_returns_empty_dict_online_dri
assert final_value == {}
-def test_stream_online_results_raises_for_unsupported_stream_type(mocker: MockerFixture):
- mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=mocker.Mock())
+def test_stream_online_results_raises_for_unsupported_stream_type(mocker: MockerFixture, document_runtime):
+ mocker.patch.object(DatasourceManager, "get_datasource_runtime", return_value=document_runtime)
with pytest.raises(ValueError, match="Unsupported datasource type for streaming"):
list(
@@ -386,8 +428,8 @@ def test_stream_node_events_emits_events_online_document(mocker: MockerFixture):
tenant_id="t1",
parameters_for_log={"k": "v"},
datasource_info={"user_id": "u1"},
- variable_pool=mocker.Mock(),
- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),
+ variable_pool=VariablePool(),
+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
@@ -459,7 +501,7 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mock
return_value=built,
)
- variable_pool = mocker.Mock()
+ variable_pool = VariablePool()
events = list(
DatasourceManager.stream_node_events(
@@ -472,14 +514,14 @@ def test_stream_node_events_builds_file_and_variables_from_messages(mocker: Mock
parameters_for_log={"k": "v"},
datasource_info={"info": "x"},
variable_pool=variable_pool,
- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),
+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
)
build_from_mapping.assert_called_once()
- variable_pool.add.assert_not_called()
+ assert variable_pool.variable_dictionary == {}
assert any(isinstance(e, StreamChunkEvent) and e.chunk == "hello" for e in events)
assert any(isinstance(e, StreamChunkEvent) and e.chunk.startswith("Link: http") for e in events)
@@ -520,8 +562,8 @@ def test_stream_node_events_raises_when_toolfile_missing(mocker: MockerFixture,
tenant_id="t1",
parameters_for_log={},
datasource_info={},
- variable_pool=mocker.Mock(),
- datasource_param=types.SimpleNamespace(workspace_id="w", page_id="pg", type="t"),
+ variable_pool=VariablePool(),
+ datasource_param=GetOnlineDocumentPageContentRequest(workspace_id="w", page_id="pg", type="t"),
online_drive_request=None,
credentials={},
)
@@ -552,7 +594,7 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(moc
side_effect=_transformed,
)
- variable_pool = mocker.Mock()
+ variable_pool = VariablePool()
events = list(
DatasourceManager.stream_node_events(
node_id="nodeA",
@@ -565,14 +607,14 @@ def test_stream_node_events_online_drive_sets_variable_pool_file_and_outputs(moc
datasource_info={"k": "v"},
variable_pool=variable_pool,
datasource_param=None,
- online_drive_request=types.SimpleNamespace(id="id", bucket="b"),
+ online_drive_request=OnlineDriveDownloadFileRequest(id="id", bucket="b"),
credentials={},
)
)
- variable_pool.add.assert_called_once()
- assert variable_pool.add.call_args[0][0] == ["nodeA", "file"]
- assert variable_pool.add.call_args[0][1] == file_in
+ stored_file = variable_pool.get(["nodeA", "file"])
+ assert stored_file is not None
+ assert stored_file.value == file_in
completed = events[-1]
assert isinstance(completed, StreamCompletedEvent)
@@ -606,7 +648,7 @@ def test_stream_node_events_skips_file_build_for_non_online_types(mocker: Mocker
tenant_id="t1",
parameters_for_log={},
datasource_info={},
- variable_pool=mocker.Mock(),
+ variable_pool=VariablePool(),
datasource_param=None,
online_drive_request=None,
credentials={},