Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use real completion runner collaborators (#43547)

Asuka Minato1b577d73d6f8
diff --git a/api/tests/unit_tests/core/app/apps/completion/test_app_runner.py b/api/tests/unit_tests/core/app/apps/completion/test_app_runner.py
index 0af69b54b7c..1a86344a329 100644
--- a/api/tests/unit_tests/core/app/apps/completion/test_app_runner.py
+++ b/api/tests/unit_tests/core/app/apps/completion/test_app_runner.py
@@ -1,17 +1,29 @@
+from collections.abc import Callable
+from dataclasses import dataclass, field
from decimal import Decimal
from types import SimpleNamespace
-from unittest.mock import ANY, MagicMock
import pytest
-from pytest_mock import MockerFixture
from sqlalchemy import event
from sqlalchemy.orm import Session
import core.app.apps.completion.app_runner as module
+from core.app.app_config.entities import (
+ AppAdditionalFeatures,
+ DatasetEntity,
+ DatasetRetrieveConfigEntity,
+ PromptTemplateEntity,
+)
+from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
+from core.app.apps.completion.app_config_manager import CompletionAppConfig
from core.app.apps.completion.app_runner import CompletionAppRunner
-from core.app.entities.app_invoke_entities import InvokeFrom
+from core.app.entities.app_invoke_entities import InvokeFrom, ModelConfigWithCredentialsEntity
from core.credit_usage import CreditUsageAppType, CreditUsageCreatedBy
+from core.model_manager import ModelInstance, ModelManager
from core.moderation.base import ModerationError
+from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
+from graphon.file import FileUploadConfig
+from graphon.file.models import ImageConfig
from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent
from graphon.model_runtime.entities.model_entities import ModelType
from models.enums import ConversationFromSource
@@ -21,26 +33,95 @@ APP_ID = "00000000-0000-0000-0000-000000000001"
TENANT_ID = "00000000-0000-0000-0000-000000000002"
+@dataclass(frozen=True)
+class RecordedCall:
+ args: tuple[object, ...]
+ kwargs: dict[str, object]
+
+
+@dataclass
+class CallRecorder:
+ result: object = None
+ results: list[object] | None = None
+ error: Exception | None = None
+ implementation: Callable[..., object] | None = None
+ calls: list[RecordedCall] = field(default_factory=list)
+
+ def __call__(self, *args: object, **kwargs: object) -> object:
+ self.calls.append(RecordedCall(args=args, kwargs=kwargs))
+ if self.error is not None:
+ raise self.error
+ if self.implementation is not None:
+ return self.implementation(*args, **kwargs)
+ if self.results is not None:
+ return self.results.pop(0)
+ return self.result
+
+
+class RecordingQueueManager(AppQueueManager):
+ def __init__(self) -> None:
+ self.published: list[tuple[object, PublishFrom]] = []
+
+ def _publish(self, event: object, pub_from: PublishFrom) -> None:
+ self.published.append((event, pub_from))
+
+
+class RecordingDatasetRetrieval(DatasetRetrieval):
+ def __init__(self, result: tuple[object, list[object]]) -> None:
+ self.result = result
+ self.calls: list[RecordedCall] = []
+
+ def retrieve(self, *args: object, **kwargs: object) -> tuple[object, list[object]]:
+ self.calls.append(RecordedCall(args=args, kwargs=kwargs))
+ return self.result
+
+
+class RecordingModelInstance(ModelInstance):
+ def __init__(self, result: object = "invoke_result", implementation: Callable[..., object] | None = None) -> None:
+ self.result = result
+ self.implementation = implementation
+ self.calls: list[RecordedCall] = []
+
+ def invoke_llm(self, *args: object, **kwargs: object) -> object:
+ self.calls.append(RecordedCall(args=args, kwargs=kwargs))
+ if self.implementation is not None:
+ return self.implementation(*args, **kwargs)
+ return self.result
+
+
+class RecordingModelManager(ModelManager):
+ def __init__(self, model_instance: RecordingModelInstance) -> None:
+ self.model_instance = model_instance
+ self.calls: list[RecordedCall] = []
+
+ def get_model_instance(self, *args: object, **kwargs: object) -> RecordingModelInstance:
+ self.calls.append(RecordedCall(args=args, kwargs=kwargs))
+ return self.model_instance
+
+
@pytest.fixture
def runner():
return CompletionAppRunner()
def _build_app_config(dataset=None, external_tools=None, additional_features=None):
- app_config = MagicMock()
- app_config.app_id = APP_ID
- app_config.tenant_id = TENANT_ID
- app_config.app_mode = AppMode.COMPLETION
- app_config.prompt_template = MagicMock()
- app_config.dataset = dataset
- app_config.external_data_variables = external_tools or []
- app_config.additional_features = additional_features
- app_config.app_model_config_dict = {"file_upload": {"enabled": True}}
- return app_config
+ return CompletionAppConfig.model_construct(
+ app_id=APP_ID,
+ tenant_id=TENANT_ID,
+ app_mode=AppMode.COMPLETION,
+ prompt_template=PromptTemplateEntity(
+ prompt_type=PromptTemplateEntity.PromptType.SIMPLE,
+ simple_prompt_template="Answer the query.",
+ ),
+ dataset=dataset,
+ external_data_variables=external_tools or [],
+ additional_features=additional_features,
+ app_model_config_dict={"file_upload": {"enabled": True}},
+ )
def _build_generate_entity(app_config, file_upload_config=None):
- model_conf = MagicMock(
+ model_conf = ModelConfigWithCredentialsEntity.model_construct(
provider="provider",
provider_model_bundle="bundle",
model="model",
@@ -56,7 +137,7 @@ def _build_generate_entity(app_config, file_upload_config=None):
file_upload_config=file_upload_config,
stream=True,
user_id="user",
- invoke_from=MagicMock(),
+ invoke_from=InvokeFrom.SERVICE_API,
)
@@ -109,91 +190,104 @@ def _persist_records(session: Session) -> tuple[App, Message]:
class TestCompletionAppRunner:
- def test_run_app_not_found(self, runner, mocker: MockerFixture, sqlite_session: Session):
+ def test_run_app_not_found(self, runner, sqlite_session: Session):
app_config = _build_app_config()
app_generate_entity = _build_generate_entity(app_config)
with pytest.raises(ValueError):
- runner.run(app_generate_entity, MagicMock(), _message(), sqlite_session)
+ runner.run(app_generate_entity, RecordingQueueManager(), _message(), sqlite_session)
- def test_run_moderation_error_outputs_direct(self, runner, mocker: MockerFixture, sqlite_session: Session):
+ def test_run_moderation_error_outputs_direct(self, runner, sqlite_session: Session):
_, message = _persist_records(sqlite_session)
app_config = _build_app_config()
app_generate_entity = _build_generate_entity(app_config)
- runner.organize_prompt_messages = MagicMock(return_value=([], None))
- runner.moderation_for_inputs = MagicMock(side_effect=ModerationError("blocked"))
- runner.direct_output = MagicMock()
- runner._handle_invoke_result = MagicMock()
+ runner.organize_prompt_messages = CallRecorder(result=([], None)) # type: ignore[method-assign]
+ runner.moderation_for_inputs = CallRecorder(error=ModerationError("blocked")) # type: ignore[method-assign]
+ direct_output = CallRecorder()
+ handle_invoke_result = CallRecorder()
+ runner.direct_output = direct_output # type: ignore[method-assign]
+ runner._handle_invoke_result = handle_invoke_result # type: ignore[method-assign]
- runner.run(app_generate_entity, MagicMock(), message, sqlite_session)
+ runner.run(app_generate_entity, RecordingQueueManager(), message, sqlite_session)
- runner.direct_output.assert_called_once()
- runner._handle_invoke_result.assert_not_called()
+ assert len(direct_output.calls) == 1
+ assert handle_invoke_result.calls == []
- def test_run_hosting_moderation_stops(self, runner, mocker: MockerFixture, sqlite_session: Session):
+ def test_run_hosting_moderation_stops(self, runner, sqlite_session: Session):
_, message = _persist_records(sqlite_session)
app_config = _build_app_config()
app_generate_entity = _build_generate_entity(app_config)
- runner.organize_prompt_messages = MagicMock(return_value=([], None))
- runner.moderation_for_inputs = MagicMock(return_value=(None, app_generate_entity.inputs, "query"))
- runner.check_hosting_moderation = MagicMock(return_value=True)
- runner._handle_invoke_result = MagicMock()
+ runner.organize_prompt_messages = CallRecorder(result=([], None)) # type: ignore[method-assign]
+ runner.moderation_for_inputs = CallRecorder( # type: ignore[method-assign]
+ result=(None, app_generate_entity.inputs, "query")
+ )
+ runner.check_hosting_moderation = CallRecorder(result=True) # type: ignore[method-assign]
+ handle_invoke_result = CallRecorder()
+ runner._handle_invoke_result = handle_invoke_result # type: ignore[method-assign]
- runner.run(app_generate_entity, MagicMock(), message, sqlite_session)
+ runner.run(app_generate_entity, RecordingQueueManager(), message, sqlite_session)
- runner._handle_invoke_result.assert_not_called()
+ assert handle_invoke_result.calls == []
- def test_run_dataset_and_external_tools_flow(self, runner, mocker: MockerFixture, sqlite_session: Session):
+ def test_run_dataset_and_external_tools_flow(
+ self, runner, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
+ ):
_, message = _persist_records(sqlite_session)
- retrieve_config = MagicMock(query_variable="qvar")
- dataset_config = MagicMock(dataset_ids=["ds"], retrieve_config=retrieve_config)
- additional_features = MagicMock(show_retrieve_source=True)
+ retrieve_config = DatasetRetrieveConfigEntity(
+ query_variable="qvar",
+ retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE,
+ )
+ dataset_config = DatasetEntity(dataset_ids=["ds"], retrieve_config=retrieve_config)
+ additional_features = AppAdditionalFeatures(show_retrieve_source=True)
app_config = _build_app_config(
dataset=dataset_config,
external_tools=["tool"],
additional_features=additional_features,
)
- file_upload_config = MagicMock()
- file_upload_config.image_config.detail = ImagePromptMessageContent.DETAIL.HIGH
+ file_upload_config = FileUploadConfig(image_config=ImageConfig(detail=ImagePromptMessageContent.DETAIL.HIGH))
app_generate_entity = _build_generate_entity(app_config, file_upload_config=file_upload_config)
- runner.organize_prompt_messages = MagicMock(side_effect=[(["pm1"], ["stop"]), (["pm2"], ["stop"])])
- runner.moderation_for_inputs = MagicMock(return_value=(None, app_generate_entity.inputs, "query"))
- runner.fill_in_inputs_from_external_data_tools = MagicMock(return_value=app_generate_entity.inputs)
- runner.check_hosting_moderation = MagicMock(return_value=False)
- runner.recalc_llm_max_tokens = MagicMock()
- runner._handle_invoke_result = MagicMock()
+ runner.organize_prompt_messages = CallRecorder( # type: ignore[method-assign]
+ results=[(["pm1"], ["stop"]), (["pm2"], ["stop"])]
+ )
+ runner.moderation_for_inputs = CallRecorder( # type: ignore[method-assign]
+ result=(None, app_generate_entity.inputs, "query")
+ )
+ runner.fill_in_inputs_from_external_data_tools = CallRecorder( # type: ignore[method-assign]
+ result=app_generate_entity.inputs
+ )
+ runner.check_hosting_moderation = CallRecorder(result=False) # type: ignore[method-assign]
+ runner.recalc_llm_max_tokens = CallRecorder() # type: ignore[method-assign]
+ handle_invoke_result = CallRecorder()
+ runner._handle_invoke_result = handle_invoke_result # type: ignore[method-assign]
- dataset_retrieval = MagicMock()
- dataset_retrieval.retrieve.return_value = ("ctx", ["file1"])
- mocker.patch.object(module, "DatasetRetrieval", return_value=dataset_retrieval)
+ dataset_retrieval = RecordingDatasetRetrieval(("ctx", ["file1"]))
+ monkeypatch.setattr(module, "DatasetRetrieval", lambda _: dataset_retrieval)
- model_instance = MagicMock()
- model_instance.invoke_llm.return_value = "invoke_result"
- model_manager = MagicMock()
- model_manager.get_model_instance.return_value = model_instance
- mocker.patch.object(module.ModelManager, "for_tenant", return_value=model_manager)
+ model_instance = RecordingModelInstance()
+ model_manager = RecordingModelManager(model_instance)
+ monkeypatch.setattr(module.ModelManager, "for_tenant", staticmethod(lambda tenant_id: model_manager))
- runner.run(app_generate_entity, MagicMock(), message, sqlite_session)
+ runner.run(app_generate_entity, RecordingQueueManager(), message, sqlite_session)
- dataset_retrieval.retrieve.assert_called_once()
- assert dataset_retrieval.retrieve.call_args.kwargs["query"] == "query_from_input"
- runner._handle_invoke_result.assert_called_once()
+ assert len(dataset_retrieval.calls) == 1
+ assert dataset_retrieval.calls[0].kwargs["query"] == "query_from_input"
+ assert len(handle_invoke_result.calls) == 1
def test_run_closes_explicit_session_before_stream_consumption(
- self, runner, mocker: MockerFixture, sqlite_session: Session
+ self, runner, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
):
_, message = _persist_records(sqlite_session)
app_config = _build_app_config()
app_generate_entity = _build_generate_entity(app_config)
- queue_manager = MagicMock()
+ queue_manager = RecordingQueueManager()
events = []
session = sqlite_session
@@ -206,13 +300,14 @@ class TestCompletionAppRunner:
events.append("close")
original_close()
- runner.organize_prompt_messages = MagicMock(return_value=([], None))
- runner.moderation_for_inputs = MagicMock(return_value=(None, app_generate_entity.inputs, "query"))
- runner.check_hosting_moderation = MagicMock(return_value=False)
- runner.recalc_llm_max_tokens = MagicMock()
- runner._handle_invoke_result = MagicMock(side_effect=lambda invoke_result, **kwargs: list(invoke_result))
-
- model_instance = MagicMock()
+ runner.organize_prompt_messages = CallRecorder(result=([], None)) # type: ignore[method-assign]
+ runner.moderation_for_inputs = CallRecorder( # type: ignore[method-assign]
+ result=(None, app_generate_entity.inputs, "query")
+ )
+ runner.check_hosting_moderation = CallRecorder(result=False) # type: ignore[method-assign]
+ runner.recalc_llm_max_tokens = CallRecorder() # type: ignore[method-assign]
+ handle_invoke_result = CallRecorder(implementation=lambda invoke_result, **kwargs: list(invoke_result))
+ runner._handle_invoke_result = handle_invoke_result # type: ignore[method-assign]
def invoke_stream():
events.append("first-chunk")
@@ -222,11 +317,10 @@ class TestCompletionAppRunner:
events.append("invoke")
return invoke_stream()
- model_instance.invoke_llm.side_effect = invoke_llm
- model_manager = MagicMock()
- model_manager.get_model_instance.return_value = model_instance
- mocker.patch.object(module.ModelManager, "for_tenant", return_value=model_manager)
- mocker.patch.object(session, "close", side_effect=record_close)
+ model_instance = RecordingModelInstance(implementation=invoke_llm)
+ model_manager = RecordingModelManager(model_instance)
+ monkeypatch.setattr(module.ModelManager, "for_tenant", staticmethod(lambda tenant_id: model_manager))
+ monkeypatch.setattr(session, "close", record_close)
event.listen(session, "after_commit", record_commit)
try:
@@ -235,20 +329,24 @@ class TestCompletionAppRunner:
event.remove(session, "after_commit", record_commit)
assert events == ["commit", "close", "invoke", "first-chunk"]
- runner._handle_invoke_result.assert_called_once_with(
- invoke_result=ANY,
- queue_manager=queue_manager,
- stream=True,
- message_id="msg",
- user_id="user",
- tenant_id=TENANT_ID,
- )
+ assert len(handle_invoke_result.calls) == 1
+ handle_call = handle_invoke_result.calls[0]
+ assert handle_call.args == ()
+ assert handle_call.kwargs["queue_manager"] is queue_manager
+ assert handle_call.kwargs | {"invoke_result": None} == {
+ "invoke_result": None,
+ "queue_manager": queue_manager,
+ "stream": True,
+ "message_id": "msg",
+ "user_id": "user",
+ "tenant_id": TENANT_ID,
+ }
@pytest.mark.parametrize("stream", [False, True])
def test_run_invokes_model_resolved_by_model_manager(
self,
runner,
- mocker: MockerFixture,
+ monkeypatch: pytest.MonkeyPatch,
sqlite_session: Session,
stream: bool,
):
@@ -257,52 +355,63 @@ class TestCompletionAppRunner:
app_generate_entity = _build_generate_entity(app_config)
app_generate_entity.stream = stream
- runner.organize_prompt_messages = MagicMock(return_value=(["prompt"], ["stop"]))
- runner.moderation_for_inputs = MagicMock(return_value=(None, app_generate_entity.inputs, "query"))
- runner.check_hosting_moderation = MagicMock(return_value=False)
- runner.recalc_llm_max_tokens = MagicMock()
- runner._handle_invoke_result = MagicMock()
-
- model_instance = MagicMock()
- model_instance.invoke_llm.return_value = "invoke_result"
- model_manager = MagicMock()
- model_manager.get_model_instance.return_value = model_instance
- model_manager_factory = mocker.patch.object(module.ModelManager, "for_tenant", return_value=model_manager)
-
- runner.run(app_generate_entity, MagicMock(), message, sqlite_session)
-
- model_manager_factory.assert_called_once_with(tenant_id=TENANT_ID)
- model_manager.get_model_instance.assert_called_once_with(
- tenant_id=TENANT_ID,
- provider="provider",
- model_type=ModelType.LLM,
- model="model",
+ runner.organize_prompt_messages = CallRecorder(result=(["prompt"], ["stop"])) # type: ignore[method-assign]
+ runner.moderation_for_inputs = CallRecorder( # type: ignore[method-assign]
+ result=(None, app_generate_entity.inputs, "query")
)
- model_instance.invoke_llm.assert_called_once_with(
- prompt_messages=["prompt"],
- model_parameters={"max_tokens": 10},
- stop=["stop"],
- stream=stream,
- request_metadata={
- "app_id": APP_ID,
- "app_type": CreditUsageAppType.COMPLETION,
- "created_by": CreditUsageCreatedBy.APP,
- },
- )
-
- def test_run_uses_low_image_detail_default(self, runner, mocker: MockerFixture, sqlite_session: Session):
+ runner.check_hosting_moderation = CallRecorder(result=False) # type: ignore[method-assign]
+ runner.recalc_llm_max_tokens = CallRecorder() # type: ignore[method-assign]
+ runner._handle_invoke_result = CallRecorder() # type: ignore[method-assign]
+
+ model_instance = RecordingModelInstance()
+ model_manager = RecordingModelManager(model_instance)
+ model_manager_factory = CallRecorder(result=model_manager)
+ monkeypatch.setattr(module.ModelManager, "for_tenant", staticmethod(model_manager_factory))
+
+ runner.run(app_generate_entity, RecordingQueueManager(), message, sqlite_session)
+
+ assert model_manager_factory.calls == [RecordedCall(args=(), kwargs={"tenant_id": TENANT_ID})]
+ assert model_manager.calls == [
+ RecordedCall(
+ args=(),
+ kwargs={
+ "tenant_id": TENANT_ID,
+ "provider": "provider",
+ "model_type": ModelType.LLM,
+ "model": "model",
+ },
+ )
+ ]
+ assert model_instance.calls == [
+ RecordedCall(
+ args=(),
+ kwargs={
+ "prompt_messages": ["prompt"],
+ "model_parameters": {"max_tokens": 10},
+ "stop": ["stop"],
+ "stream": stream,
+ "request_metadata": {
+ "app_id": APP_ID,
+ "app_type": CreditUsageAppType.COMPLETION,
+ "created_by": CreditUsageCreatedBy.APP,
+ },
+ },
+ )
+ ]
+
+ def test_run_uses_low_image_detail_default(self, runner, sqlite_session: Session):
_, message = _persist_records(sqlite_session)
app_config = _build_app_config()
app_generate_entity = _build_generate_entity(app_config, file_upload_config=None)
- runner.organize_prompt_messages = MagicMock(return_value=([], None))
- runner.moderation_for_inputs = MagicMock(return_value=(None, app_generate_entity.inputs, "query"))
- runner.check_hosting_moderation = MagicMock(return_value=True)
+ organize_prompt_messages = CallRecorder(result=([], None))
+ runner.organize_prompt_messages = organize_prompt_messages # type: ignore[method-assign]
+ runner.moderation_for_inputs = CallRecorder( # type: ignore[method-assign]
+ result=(None, app_generate_entity.inputs, "query")
+ )
+ runner.check_hosting_moderation = CallRecorder(result=True) # type: ignore[method-assign]
- runner.run(app_generate_entity, MagicMock(), message, sqlite_session)
+ runner.run(app_generate_entity, RecordingQueueManager(), message, sqlite_session)
- assert (
- runner.organize_prompt_messages.call_args.kwargs["image_detail_config"]
- == ImagePromptMessageContent.DETAIL.LOW
- )
+ assert organize_prompt_messages.calls[-1].kwargs["image_detail_config"] == ImagePromptMessageContent.DETAIL.LOW