Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use real structured output model fixtures (#43154)

Asuka Minato51fa4a25f945
diff --git a/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py b/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py
index 6ed9ddb476c..be224ae2370 100644
--- a/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py
+++ b/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py
@@ -1,5 +1,8 @@
+import inspect
import json
-from unittest.mock import MagicMock, patch
+from collections.abc import Iterator
+from types import SimpleNamespace
+from unittest.mock import patch
import pytest
@@ -16,6 +19,7 @@ from core.llm_generator.output_parser.structured_output import (
remove_additional_properties,
)
from core.model_manager import ModelInstance
+from graphon.model_runtime.entities.common_entities import I18nObject
from graphon.model_runtime.entities.llm_entities import (
LLMResult,
LLMResultChunk,
@@ -29,7 +33,35 @@ from graphon.model_runtime.entities.message_entities import (
TextPromptMessageContent,
UserPromptMessage,
)
-from graphon.model_runtime.entities.model_entities import AIModelEntity, ParameterRule, ParameterType
+from graphon.model_runtime.entities.model_entities import (
+ AIModelEntity,
+ FetchFrom,
+ ModelFeature,
+ ModelType,
+ ParameterRule,
+ ParameterType,
+)
+
+
+def make_model_schema(model: str) -> AIModelEntity:
+ return AIModelEntity(
+ model=model,
+ label=I18nObject(en_US=model),
+ model_type=ModelType.LLM,
+ fetch_from=FetchFrom.PREDEFINED_MODEL,
+ model_properties={},
+ )
+
+
+class _ModelInstance(ModelInstance):
+ """Return a configured response while checking the invocation contract."""
+
+ def __init__(self, response: LLMResult | Iterator[LLMResultChunk | SimpleNamespace]) -> None:
+ self.response = response
+
+ def invoke_llm(self, **kwargs: object):
+ inspect.signature(ModelInstance.invoke_llm).bind(self, **kwargs)
+ return self.response
class TestStructuredOutput:
@@ -140,8 +172,7 @@ class TestStructuredOutput:
def test_handle_native_json_schema(self):
provider = "openai"
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.model = "gpt-4"
+ model_schema = make_model_schema("gpt-4")
structured_output_schema = {"type": "object"}
model_parameters = {}
rules = [
@@ -164,8 +195,7 @@ class TestStructuredOutput:
def test_handle_native_json_schema_no_format_rule(self):
provider = "openai"
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.model = "gpt-4"
+ model_schema = make_model_schema("gpt-4")
structured_output_schema = {"type": "object"}
model_parameters = {}
rules = []
@@ -204,32 +234,29 @@ class TestStructuredOutput:
assert isinstance(result[1], UserPromptMessage)
def test_prepare_schema_for_model_gemini(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.model = "gemini-1.5-pro"
+ model_schema = make_model_schema("gemini-1.5-pro")
schema = {"type": "object", "additionalProperties": False}
result = _prepare_schema_for_model("google", model_schema, schema)
assert "additionalProperties" not in result
def test_prepare_schema_for_model_ollama(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.model = "llama3"
+ model_schema = make_model_schema("llama3")
schema = {"type": "object"}
result = _prepare_schema_for_model("ollama", model_schema, schema)
assert result == schema
def test_prepare_schema_for_model_default(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.model = "gpt-4"
+ model_schema = make_model_schema("gpt-4")
schema = {"type": "object"}
result = _prepare_schema_for_model("openai", model_schema, schema)
assert result == {"schema": schema, "name": "llm_response"}
def test_invoke_llm_with_structured_output_no_stream_native(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.support_structure_output = True
+ model_schema = make_model_schema("gpt-4")
+ model_schema.features = [ModelFeature.STRUCTURED_OUTPUT]
model_schema.parameter_rules = [
ParameterRule(
name="response_format",
@@ -241,15 +268,15 @@ class TestStructuredOutput:
]
model_schema.model = "gpt-4o"
- model_instance = MagicMock(spec=ModelInstance)
- mock_result = MagicMock(spec=LLMResult)
- mock_result.message = AssistantPromptMessage(content='{"result": "success"}')
- mock_result.model = "gpt-4o"
- mock_result.usage = LLMUsage.empty_usage()
- mock_result.system_fingerprint = "fp_native"
- mock_result.prompt_messages = [UserPromptMessage(content="hi")]
+ llm_response = LLMResult(
+ model="gpt-4o",
+ message=AssistantPromptMessage(content='{"result": "success"}'),
+ usage=LLMUsage.empty_usage(),
+ system_fingerprint="fp_native",
+ prompt_messages=[UserPromptMessage(content="hi")],
+ )
- model_instance.invoke_llm.return_value = mock_result
+ model_instance = _ModelInstance(llm_response)
result = invoke_llm_with_structured_output(
provider="openai",
@@ -265,8 +292,8 @@ class TestStructuredOutput:
assert result.system_fingerprint == "fp_native"
def test_invoke_llm_with_structured_output_no_stream_prompt_based(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.support_structure_output = False
+ model_schema = make_model_schema("gpt-4")
+ model_schema.features = []
model_schema.parameter_rules = [
ParameterRule(
name="response_format",
@@ -278,15 +305,15 @@ class TestStructuredOutput:
]
model_schema.model = "claude-3"
- model_instance = MagicMock(spec=ModelInstance)
- mock_result = MagicMock(spec=LLMResult)
- mock_result.message = AssistantPromptMessage(content='{"result": "success"}')
- mock_result.model = "claude-3"
- mock_result.usage = LLMUsage.empty_usage()
- mock_result.system_fingerprint = "fp_prompt"
- mock_result.prompt_messages = []
+ llm_response = LLMResult(
+ model="claude-3",
+ message=AssistantPromptMessage(content='{"result": "success"}'),
+ usage=LLMUsage.empty_usage(),
+ system_fingerprint="fp_prompt",
+ prompt_messages=[],
+ )
- model_instance.invoke_llm.return_value = mock_result
+ model_instance = _ModelInstance(llm_response)
result = invoke_llm_with_structured_output(
provider="anthropic",
@@ -302,15 +329,17 @@ class TestStructuredOutput:
assert result.system_fingerprint == "fp_prompt"
def test_invoke_llm_with_structured_output_no_string_error(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.support_structure_output = False
+ model_schema = make_model_schema("gpt-4")
+ model_schema.features = []
model_schema.parameter_rules = []
- model_instance = MagicMock(spec=ModelInstance)
- mock_result = MagicMock(spec=LLMResult)
- mock_result.message = AssistantPromptMessage(content=[TextPromptMessageContent(data="not a string")])
+ llm_response = LLMResult(
+ model=model_schema.model,
+ message=AssistantPromptMessage(content=[TextPromptMessageContent(data="not a string")]),
+ usage=LLMUsage.empty_usage(),
+ )
- model_instance.invoke_llm.return_value = mock_result
+ model_instance = _ModelInstance(llm_response)
with pytest.raises(OutputParserError) as excinfo:
invoke_llm_with_structured_output(
@@ -324,42 +353,41 @@ class TestStructuredOutput:
assert "Failed to parse structured output, LLM result is not a string" in str(excinfo.value)
def test_invoke_llm_with_structured_output_stream(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.support_structure_output = False
+ model_schema = make_model_schema("gpt-4")
+ model_schema.features = []
model_schema.parameter_rules = []
model_schema.model = "gpt-4"
- model_instance = MagicMock(spec=ModelInstance)
+ # Model chunks carry validated deltas and stream metadata.
+ chunk1 = LLMResultChunk(
+ model=model_schema.model,
+ delta=LLMResultChunkDelta(
+ index=0, message=AssistantPromptMessage(content='{"key": '), usage=LLMUsage.empty_usage()
+ ),
+ prompt_messages=[UserPromptMessage(content="hi")],
+ system_fingerprint="fp1",
+ )
- # Mock chunks
- chunk1 = MagicMock(spec=LLMResultChunk)
- chunk1.delta = LLMResultChunkDelta(
- index=0, message=AssistantPromptMessage(content='{"key": '), usage=LLMUsage.empty_usage()
+ chunk2 = LLMResultChunk(
+ model=model_schema.model,
+ delta=LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content='"value"}')),
+ prompt_messages=[UserPromptMessage(content="hi")],
+ system_fingerprint="fp1",
)
- chunk1.prompt_messages = [UserPromptMessage(content="hi")]
- chunk1.system_fingerprint = "fp1"
-
- chunk2 = MagicMock(spec=LLMResultChunk)
- chunk2.delta = LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content='"value"}'))
- chunk2.prompt_messages = [UserPromptMessage(content="hi")]
- chunk2.system_fingerprint = "fp1"
-
- chunk3 = MagicMock(spec=LLMResultChunk)
- chunk3.delta = LLMResultChunkDelta(
- index=0,
- message=AssistantPromptMessage(
- content=[
- TextPromptMessageContent(data=" "),
- ]
+
+ chunk3 = LLMResultChunk(
+ model=model_schema.model,
+ delta=LLMResultChunkDelta(
+ index=0, message=AssistantPromptMessage(content=[TextPromptMessageContent(data=" ")])
),
+ prompt_messages=[UserPromptMessage(content="hi")],
+ system_fingerprint="fp1",
)
- chunk3.prompt_messages = [UserPromptMessage(content="hi")]
- chunk3.system_fingerprint = "fp1"
- event4 = MagicMock()
- event4.delta = LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content=""))
+ # Preserve coverage for a delta-bearing event that is not an LLMResultChunk.
+ event4 = SimpleNamespace(delta=LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content="")))
- model_instance.invoke_llm.return_value = [chunk1, chunk2, chunk3, event4]
+ model_instance = _ModelInstance(iter([chunk1, chunk2, chunk3, event4]))
generator = invoke_llm_with_structured_output(
provider="openai",
@@ -377,13 +405,12 @@ class TestStructuredOutput:
assert chunks[-1].prompt_messages == [UserPromptMessage(content="hi")]
def test_invoke_llm_with_structured_output_stream_no_id_events(self):
- model_schema = MagicMock(spec=AIModelEntity)
- model_schema.support_structure_output = False
+ model_schema = make_model_schema("gpt-4")
+ model_schema.features = []
model_schema.parameter_rules = []
model_schema.model = "gpt-4"
- model_instance = MagicMock(spec=ModelInstance)
- model_instance.invoke_llm.return_value = []
+ model_instance = _ModelInstance(iter([]))
generator = invoke_llm_with_structured_output(
provider="openai",