Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: exercise real prompt model runtime (#43567)

Asuka Minatob31c4dbeeea4
diff --git a/api/tests/unit_tests/core/prompt/test_prompt_transform.py b/api/tests/unit_tests/core/prompt/test_prompt_transform.py
index 3d71e734960..5b3de360c16 100644
--- a/api/tests/unit_tests/core/prompt/test_prompt_transform.py
+++ b/api/tests/unit_tests/core/prompt/test_prompt_transform.py
@@ -1,18 +1,13 @@
-from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
-
import pytest
+from core.entities.provider_entities import CustomProviderConfiguration
+from core.model_manager import ModelInstance
+from core.prompt.entities.advanced_prompt_entities import MemoryConfig
from core.prompt.prompt_transform import PromptTransform
-from graphon.model_runtime.entities.model_entities import ModelPropertyKey
-
-# from core.app.app_config.entities import ModelConfigEntity
-# from core.entities.provider_configuration import ProviderConfiguration, ProviderModelBundle
-# from graphon.model_runtime.entities.message_entities import UserPromptMessage
-# from graphon.model_runtime.entities.model_entities import AIModelEntity, ModelPropertyKey, ParameterRule
-# from graphon.model_runtime.entities.provider_entities import ProviderEntity
-# from graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModel
-# from core.prompt.prompt_transform import PromptTransform
+from graphon.model_runtime.entities.common_entities import I18nObject
+from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage
+from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ParameterRule, ParameterType
+from tests.unit_tests.core.model_fixtures import make_model_config, make_token_buffer_memory
class TestPromptTransform:
@@ -22,58 +17,41 @@ class TestPromptTransform:
with pytest.raises(ValueError, match="Either model_config or model_instance must be provided."):
transform._resolve_model_runtime()
- def test_resolve_model_runtime_builds_model_instance_from_model_config(self):
+ def test_resolve_model_runtime_builds_model_instance_from_model_config(self, monkeypatch: pytest.MonkeyPatch):
transform = PromptTransform()
- fake_model_schema = SimpleNamespace(model_properties={}, parameter_rules=[])
- fake_model_type_instance = MagicMock()
- fake_model_type_instance.get_model_schema.return_value = fake_model_schema
- fake_model_instance = SimpleNamespace(
- model_type_instance=fake_model_type_instance,
- model_name="resolved-model",
- credentials=None,
- parameters=None,
- stop=None,
- )
- model_config = SimpleNamespace(
- provider_model_bundle=object(),
- model="config-model",
- credentials={"api_key": "secret"},
- parameters={"temperature": 0.1},
- stop=["END"],
- model_schema=SimpleNamespace(model_properties={}, parameter_rules=[]),
+ model_config = make_model_config(provider="openai", model="config-model", mode="chat")
+ model_config.credentials = {"api_key": "secret"}
+ model_config.parameters = {"temperature": 0.1}
+ model_config.stop = ["END"]
+ bundle = model_config.provider_model_bundle
+ bundle.configuration.custom_configuration.provider = CustomProviderConfiguration(
+ credentials={"api_key": "provider-secret"}
)
+ model_schema = model_config.model_schema.model_copy(deep=True)
+ schema_calls = []
- with patch(
- "core.prompt.prompt_transform.ModelInstance", return_value=fake_model_instance
- ) as model_instance_cls:
- model_instance, model_schema = transform._resolve_model_runtime(model_config=model_config)
+ def get_model_schema(*, model, credentials):
+ schema_calls.append((model, credentials))
+ return model_schema
- model_instance_cls.assert_called_once_with(
- provider_model_bundle=model_config.provider_model_bundle,
- model=model_config.model,
- )
- fake_model_type_instance.get_model_schema.assert_called_once_with(
- model="resolved-model",
- credentials={"api_key": "secret"},
- )
- assert model_instance is fake_model_instance
+ monkeypatch.setattr(bundle.model_type_instance, "get_model_schema", get_model_schema)
+
+ model_instance, resolved_schema = transform._resolve_model_runtime(model_config=model_config)
+
+ assert isinstance(model_instance, ModelInstance)
+ assert model_instance.provider_model_bundle is bundle
+ assert model_instance.model_name == "config-model"
+ assert schema_calls == [("config-model", {"api_key": "secret"})]
assert model_instance.credentials == {"api_key": "secret"}
assert model_instance.parameters == {"temperature": 0.1}
assert model_instance.stop == ["END"]
- assert model_schema is fake_model_schema
+ assert resolved_schema is model_schema
- def test_resolve_model_runtime_uses_model_config_schema_fallback(self):
+ def test_resolve_model_runtime_uses_model_config_schema_fallback(self, monkeypatch: pytest.MonkeyPatch):
transform = PromptTransform()
- fallback_schema = SimpleNamespace(model_properties={}, parameter_rules=[])
- fake_model_type_instance = MagicMock()
- fake_model_type_instance.get_model_schema.return_value = None
- model_instance = SimpleNamespace(
- model_type_instance=fake_model_type_instance,
- model_name="resolved-model",
- credentials={"api_key": "secret"},
- parameters={},
- )
- model_config = SimpleNamespace(model_schema=fallback_schema)
+ model_config = make_model_config(provider="openai", model="resolved-model", mode="chat")
+ model_instance = make_token_buffer_memory(model_config).model_instance
+ monkeypatch.setattr(model_instance.model_type_instance, "get_model_schema", lambda **_kwargs: None)
resolved_model_instance, resolved_schema = transform._resolve_model_runtime(
model_config=model_config,
@@ -81,92 +59,96 @@ class TestPromptTransform:
)
assert resolved_model_instance is model_instance
- assert resolved_schema is fallback_schema
+ assert resolved_schema is model_config.model_schema
- def test_resolve_model_runtime_raises_when_schema_missing_without_model_config(self):
+ def test_resolve_model_runtime_raises_when_schema_missing_without_model_config(
+ self, monkeypatch: pytest.MonkeyPatch
+ ):
transform = PromptTransform()
- fake_model_type_instance = MagicMock()
- fake_model_type_instance.get_model_schema.return_value = None
- model_instance = SimpleNamespace(
- model_type_instance=fake_model_type_instance,
- model_name="resolved-model",
- credentials={"api_key": "secret"},
- parameters={},
- )
+ model_config = make_model_config(provider="openai", model="resolved-model", mode="chat")
+ model_instance = make_token_buffer_memory(model_config).model_instance
+ monkeypatch.setattr(model_instance.model_type_instance, "get_model_schema", lambda **_kwargs: None)
with pytest.raises(ValueError, match="Model schema not found for the provided model instance."):
transform._resolve_model_runtime(model_instance=model_instance)
- def test_calculate_rest_token_defaults_when_context_size_missing(self):
+ def test_calculate_rest_token_defaults_when_context_size_missing(self, monkeypatch: pytest.MonkeyPatch):
transform = PromptTransform()
- fake_model_instance = SimpleNamespace(parameters={}, get_llm_num_tokens=lambda _: 0)
- fake_model_schema = SimpleNamespace(model_properties={}, parameter_rules=[])
- transform._resolve_model_runtime = MagicMock(return_value=(fake_model_instance, fake_model_schema))
- model_config = SimpleNamespace(
- model_schema=SimpleNamespace(model_properties={}, parameter_rules=[]),
- provider_model_bundle=object(),
- model="test-model",
- parameters={},
+ model_config = make_model_config(provider="openai", model="test-model", mode="chat")
+ model_instance = make_token_buffer_memory(model_config).model_instance
+ monkeypatch.setattr(
+ model_instance.model_type_instance, "get_model_schema", lambda **_kwargs: model_config.model_schema
)
- rest = transform._calculate_rest_token([], model_config=model_config)
-
- assert rest == 2000
+ def unexpected_token_count(_messages):
+ pytest.fail("A model without a context size must use the default token budget.")
- def test_calculate_rest_token_uses_max_tokens_and_clamps_to_zero(self):
- transform = PromptTransform()
-
- parameter_rule = SimpleNamespace(name="max_tokens", use_template=None)
- fake_model_instance = SimpleNamespace(parameters={"max_tokens": 50}, get_llm_num_tokens=lambda _: 95)
- fake_model_schema = SimpleNamespace(
- model_properties={ModelPropertyKey.CONTEXT_SIZE: 100},
- parameter_rules=[parameter_rule],
- )
- transform._resolve_model_runtime = MagicMock(return_value=(fake_model_instance, fake_model_schema))
- model_config = SimpleNamespace(
- model_schema=SimpleNamespace(
- model_properties={ModelPropertyKey.CONTEXT_SIZE: 100},
- parameter_rules=[parameter_rule],
- ),
- provider_model_bundle=object(),
- model="test-model",
- parameters={"max_tokens": 50},
- )
+ monkeypatch.setattr(model_instance, "get_llm_num_tokens", unexpected_token_count)
- rest = transform._calculate_rest_token([SimpleNamespace()], model_config=model_config)
+ rest = transform._calculate_rest_token([], model_instance=model_instance)
- assert rest == 0
+ assert rest == 2000
- def test_calculate_rest_token_supports_use_template_parameter(self):
+ @pytest.mark.parametrize(
+ ("parameter_name", "use_template", "context_size", "max_tokens", "message_tokens", "expected"),
+ [
+ ("max_tokens", None, 100, 50, 95, 0),
+ ("generation_max", "max_tokens", 200, 30, 20, 150),
+ ],
+ ids=["clamps-to-zero", "template-parameter"],
+ )
+ def test_calculate_rest_token_uses_max_tokens(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ parameter_name: str,
+ use_template: str | None,
+ context_size: int,
+ max_tokens: int,
+ message_tokens: int,
+ expected: int,
+ ):
transform = PromptTransform()
-
- parameter_rule = SimpleNamespace(name="generation_max", use_template="max_tokens")
- fake_model_instance = SimpleNamespace(parameters={"max_tokens": 30}, get_llm_num_tokens=lambda _: 20)
- fake_model_schema = SimpleNamespace(
- model_properties={ModelPropertyKey.CONTEXT_SIZE: 200},
- parameter_rules=[parameter_rule],
- )
- transform._resolve_model_runtime = MagicMock(return_value=(fake_model_instance, fake_model_schema))
- model_config = SimpleNamespace(
- model_schema=SimpleNamespace(
- model_properties={ModelPropertyKey.CONTEXT_SIZE: 200},
- parameter_rules=[parameter_rule],
- ),
- provider_model_bundle=object(),
- model="test-model",
- parameters={"max_tokens": 30},
+ model_config = make_model_config(provider="openai", model="test-model", mode="chat")
+ model_config.model_schema.model_properties = {ModelPropertyKey.CONTEXT_SIZE: context_size}
+ model_config.model_schema.parameter_rules = [
+ ParameterRule(
+ name=parameter_name,
+ use_template=use_template,
+ label=I18nObject(en_US="Maximum tokens"),
+ type=ParameterType.INT,
+ )
+ ]
+ model_instance = make_token_buffer_memory(model_config).model_instance
+ model_instance.parameters = {"max_tokens": max_tokens}
+ messages = [UserPromptMessage(content="hello")]
+ token_calls = []
+
+ def get_llm_num_tokens(prompt_messages):
+ token_calls.append(prompt_messages)
+ return message_tokens
+
+ monkeypatch.setattr(model_instance, "get_llm_num_tokens", get_llm_num_tokens)
+ monkeypatch.setattr(
+ model_instance.model_type_instance, "get_model_schema", lambda **_kwargs: model_config.model_schema
)
- rest = transform._calculate_rest_token([SimpleNamespace()], model_config=model_config)
+ rest = transform._calculate_rest_token(messages, model_instance=model_instance)
- assert rest == 150
+ assert rest == expected
+ assert token_calls == [messages]
- def test_get_history_messages_from_memory_with_and_without_window(self):
+ def test_get_history_messages_from_memory_with_and_without_window(self, monkeypatch: pytest.MonkeyPatch):
transform = PromptTransform()
- memory = MagicMock()
- memory.get_history_prompt_text.return_value = "history"
+ model_config = make_model_config(provider="openai", model="test-model", mode="chat")
+ memory = make_token_buffer_memory(model_config)
+ history_calls = []
+
+ def get_history_prompt_text(**kwargs):
+ history_calls.append(kwargs)
+ return "history"
- memory_config_with_window = SimpleNamespace(window=SimpleNamespace(enabled=True, size=3))
+ monkeypatch.setattr(memory, "get_history_prompt_text", get_history_prompt_text)
+ memory_config_with_window = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=True, size=3))
result = transform._get_history_messages_from_memory(
memory=memory,
memory_config=memory_config_with_window,
@@ -176,56 +158,70 @@ class TestPromptTransform:
)
assert result == "history"
- memory.get_history_prompt_text.assert_called_with(
- max_token_limit=100,
- human_prefix="Human",
- ai_prefix="Assistant",
- message_limit=3,
- )
+ assert history_calls == [
+ {"max_token_limit": 100, "human_prefix": "Human", "ai_prefix": "Assistant", "message_limit": 3}
+ ]
- memory.reset_mock()
- memory_config_no_window = SimpleNamespace(window=SimpleNamespace(enabled=False, size=2))
- transform._get_history_messages_from_memory(
+ memory_config_no_window = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=False, size=2))
+ result = transform._get_history_messages_from_memory(
memory=memory,
memory_config=memory_config_no_window,
max_token_limit=50,
)
- memory.get_history_prompt_text.assert_called_with(max_token_limit=50)
- def test_get_history_messages_list_from_memory_with_and_without_window(self):
+ assert result == "history"
+ assert history_calls[1:] == [{"max_token_limit": 50}]
+
+ def test_get_history_messages_list_from_memory_with_and_without_window(self, monkeypatch: pytest.MonkeyPatch):
transform = PromptTransform()
- memory = MagicMock()
- memory.get_history_prompt_messages.return_value = ["m1", "m2"]
+ model_config = make_model_config(provider="openai", model="test-model", mode="chat")
+ memory = make_token_buffer_memory(model_config)
+ messages = [UserPromptMessage(content="m1"), AssistantPromptMessage(content="m2")]
+ history_calls = []
+
+ def get_history_prompt_messages(**kwargs):
+ history_calls.append(kwargs)
+ return messages
- memory_config_window = SimpleNamespace(window=SimpleNamespace(enabled=True, size=2))
+ monkeypatch.setattr(memory, "get_history_prompt_messages", get_history_prompt_messages)
+ memory_config_window = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=True, size=2))
result = transform._get_history_messages_list_from_memory(memory, memory_config_window, 120)
- assert result == ["m1", "m2"]
- memory.get_history_prompt_messages.assert_called_with(max_token_limit=120, message_limit=2)
+ assert result == messages
+ assert history_calls == [{"max_token_limit": 120, "message_limit": 2}]
- memory.reset_mock()
- memory.get_history_prompt_messages.return_value = ["only"]
- memory_config_no_window = SimpleNamespace(window=SimpleNamespace(enabled=True, size=0))
+ messages = [UserPromptMessage(content="only")]
+ memory_config_no_window = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=True, size=0))
result = transform._get_history_messages_list_from_memory(memory, memory_config_no_window, 10)
- assert result == ["only"]
- memory.get_history_prompt_messages.assert_called_with(max_token_limit=10, message_limit=None)
+ assert result == messages
+ assert history_calls[1:] == [{"max_token_limit": 10, "message_limit": None}]
def test_append_chat_histories_extends_prompt_messages(self, monkeypatch: pytest.MonkeyPatch):
transform = PromptTransform()
- memory = MagicMock()
- memory_config = SimpleNamespace(window=SimpleNamespace(enabled=False, size=None))
-
- monkeypatch.setattr(transform, "_calculate_rest_token", lambda prompt_messages, **kwargs: 99)
+ model_config = make_model_config(provider="openai", model="test-model", mode="chat")
+ memory = make_token_buffer_memory(model_config)
+ memory_config = MemoryConfig(window=MemoryConfig.WindowConfig(enabled=False, size=None))
+ histories = [UserPromptMessage(content="h1"), AssistantPromptMessage(content="h2")]
+ messages = [UserPromptMessage(content="p1")]
+ history_calls = []
+
+ def get_history_prompt_messages(**kwargs):
+ history_calls.append(kwargs)
+ return histories
+
+ monkeypatch.setattr(memory, "get_history_prompt_messages", get_history_prompt_messages)
monkeypatch.setattr(
- transform,
- "_get_history_messages_list_from_memory",
- lambda memory, memory_config, max_token_limit: ["h1", "h2"],
+ memory.model_instance.model_type_instance,
+ "get_model_schema",
+ lambda **_kwargs: model_config.model_schema,
)
result = transform._append_chat_histories(
memory=memory,
memory_config=memory_config,
- prompt_messages=["p1"],
- model_config=SimpleNamespace(),
+ prompt_messages=messages,
+ model_instance=memory.model_instance,
)
- assert result == ["p1", "h1", "h2"]
+ assert result is messages
+ assert result == [UserPromptMessage(content="p1"), *histories]
+ assert history_calls == [{"max_token_limit": 2000, "message_limit": None}]
diff --git a/api/tests/unit_tests/core/prompt/test_simple_prompt_transform.py b/api/tests/unit_tests/core/prompt/test_simple_prompt_transform.py
index 4b58e002ef2..e657b8eb08c 100644
--- a/api/tests/unit_tests/core/prompt/test_simple_prompt_transform.py
+++ b/api/tests/unit_tests/core/prompt/test_simple_prompt_transform.py
@@ -1,8 +1,8 @@
-from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
+from core.app.app_config.entities import PromptTemplateEntity
from core.prompt.prompt_templates.advanced_prompt_templates import (
CHAT_APP_CHAT_PROMPT_CONFIG,
CHAT_APP_COMPLETION_PROMPT_CONFIG,
@@ -11,6 +11,7 @@ from core.prompt.prompt_templates.advanced_prompt_templates import (
CONTEXT,
)
from core.prompt.simple_prompt_transform import SimplePromptTransform
+from graphon.file import File, FileTransferMethod, FileType
from graphon.model_runtime.entities.message_entities import (
AssistantPromptMessage,
ImagePromptMessageContent,
@@ -204,7 +205,9 @@ def test_get_prompt_dispatches_chat_and_completion():
transform = SimplePromptTransform()
model_config_chat = make_model_config(provider="openai", model="gpt-4", mode="chat")
model_config_completion = make_model_config(provider="openai", model="gpt-3.5-turbo-instruct", mode="completion")
- prompt_entity = SimpleNamespace(simple_prompt_template="hello")
+ prompt_entity = PromptTemplateEntity(
+ prompt_type=PromptTemplateEntity.PromptType.SIMPLE, simple_prompt_template="hello"
+ )
transform._get_chat_model_prompt_messages = MagicMock(return_value=(["chat-msg"], None))
transform._get_completion_model_prompt_messages = MagicMock(return_value=(["completion-msg"], ["stop"]))
@@ -326,8 +329,12 @@ def test_completion_model_prompt_messages_empty_stops_becomes_none():
def test_get_last_user_message_with_files_and_context_files():
transform = SimplePromptTransform()
- file = SimpleNamespace()
- context_file = SimpleNamespace()
+ file = File(
+ type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/a.jpg"
+ )
+ context_file = File(
+ type=FileType.IMAGE, transfer_method=FileTransferMethod.REMOTE_URL, remote_url="https://example.com/b.jpg"
+ )
with patch("core.prompt.simple_prompt_transform.file_manager.to_prompt_message_content") as to_content:
to_content.side_effect = [