Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test(workflow): cover structured output validation across model adapter (#44731)

ShiYuan Xiong4ff13e4340c6
diff --git a/api/tests/unit_tests/core/workflow/test_node_runtime.py b/api/tests/unit_tests/core/workflow/test_node_runtime.py
index 89d184a71f4..0f97bb0339c 100644
--- a/api/tests/unit_tests/core/workflow/test_node_runtime.py
+++ b/api/tests/unit_tests/core/workflow/test_node_runtime.py
@@ -180,6 +180,103 @@ def _llm_node_execution_behavior_mock_db_session_close(monkeypatch: pytest.Monke
class TestLlmNodeExecutionBehaviorCases:
+ @staticmethod
+ @pytest.mark.parametrize("streamed", [False, True], ids=["blocking", "streaming"])
+ @pytest.mark.parametrize("response_kind", ["valid", "schema_echo", "missing_field", "wrong_type"])
+ def test_structured_output_validation_crosses_dify_model_boundary(
+ monkeypatch: pytest.MonkeyPatch, streamed: bool, response_kind: str
+ ) -> None:
+ """A parsed JSON object must satisfy the node schema before success (#44722)."""
+ from graphon.model_runtime.entities.llm_entities import (
+ LLMResult,
+ LLMResultChunk,
+ LLMResultChunkDelta,
+ LLMUsage,
+ )
+ from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage
+
+ output_schema = {
+ "type": "object",
+ "properties": {"platform_name": {"type": "string"}, "post_content": {"type": "string"}},
+ "required": ["platform_name", "post_content"],
+ "additionalProperties": False,
+ }
+ responses = {
+ "valid": {"platform_name": "RED", "post_content": "A short post"},
+ "schema_echo": output_schema,
+ "missing_field": {"platform_name": "RED"},
+ "wrong_type": {"platform_name": "RED", "post_content": 42},
+ }
+ response = responses[response_kind]
+ text = json.dumps(response)
+ model_instance = make_model_instance(provider="openai", model="gpt-3.5-turbo")
+ model_schema = make_model_config(provider="openai", model="gpt-3.5-turbo", mode="chat").model_schema
+ monkeypatch.setattr(
+ model_instance.model_type_instance, "get_model_schema", lambda *_args, **_kwargs: model_schema
+ )
+ usage = LLMUsage.empty_usage()
+
+ def invoke_model(**kwargs: object) -> LLMResult | Generator[LLMResultChunk]:
+ assert kwargs["stream"] is True
+ if not streamed:
+ return LLMResult(
+ model=model_schema.model,
+ prompt_messages=[],
+ message=AssistantPromptMessage(content=text),
+ usage=usage,
+ )
+ return (
+ LLMResultChunk(
+ model=model_schema.model,
+ prompt_messages=[],
+ delta=LLMResultChunkDelta(
+ index=index,
+ message=AssistantPromptMessage(content=part),
+ usage=usage if index == 1 else None,
+ finish_reason="stop" if index == 1 else None,
+ ),
+ )
+ for index, part in enumerate((text[: len(text) // 2], text[len(text) // 2 :]))
+ )
+
+ monkeypatch.setattr(model_instance, "invoke_llm", invoke_model)
+ node = init_llm_node(
+ config={
+ "id": "llm",
+ "data": {
+ "title": "Structured post",
+ "type": "llm",
+ "model": {"provider": "openai", "name": "gpt-3.5-turbo", "mode": "chat", "completion_params": {}},
+ "prompt_template": [{"role": "user", "text": "Write a post"}],
+ "context": {"enabled": False},
+ "vision": {"enabled": False},
+ "structured_output_enabled": True,
+ "structured_output": {"schema": output_schema},
+ },
+ }
+ )
+ node._model_instance = DifyPreparedLLM(model_instance)
+ _llm_node_execution_behavior_mock_db_session_close(monkeypatch)
+ monkeypatch.setattr(
+ "graphon.nodes.llm.llm_utils.fetch_prompt_messages",
+ lambda **_: ([UserPromptMessage(content="Write a post")], []),
+ )
+
+ completions = [event.node_run_result for event in node._run() if isinstance(event, StreamCompletedEvent)]
+
+ assert len(completions) == 1
+ result = completions[0]
+ if response_kind == "valid":
+ assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED, result.error
+ assert result.outputs is not None
+ assert result.outputs["structured_output"] == response
+ else:
+ assert result.status == WorkflowNodeExecutionStatus.FAILED
+ assert result.error_type == "LLMNodeError"
+ assert result.error is not None
+ assert "Structured output validation failed (stage=result, path=$" in result.error
+ assert not result.outputs
+
@staticmethod
def test_execute_llm(monkeypatch: pytest.MonkeyPatch) -> None:
node = init_llm_node(