Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: use real structured output model fixtures (#43154)
Asuka Minato51fa4a25f945
diff --git a/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py b/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.pyindex 6ed9ddb476c..be224ae2370 100644--- a/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py+++ b/api/tests/unit_tests/core/llm_generator/output_parser/test_structured_output.py@@ -1,5 +1,8 @@+import inspectimport json-from unittest.mock import MagicMock, patch+from collections.abc import Iterator+from types import SimpleNamespace+from unittest.mock import patchimport pytest@@ -16,6 +19,7 @@ from core.llm_generator.output_parser.structured_output import (remove_additional_properties,)from core.model_manager import ModelInstance+from graphon.model_runtime.entities.common_entities import I18nObjectfrom graphon.model_runtime.entities.llm_entities import (LLMResult,LLMResultChunk,@@ -29,7 +33,35 @@ from graphon.model_runtime.entities.message_entities import (TextPromptMessageContent,UserPromptMessage,)-from graphon.model_runtime.entities.model_entities import AIModelEntity, ParameterRule, ParameterType+from graphon.model_runtime.entities.model_entities import (+ AIModelEntity,+ FetchFrom,+ ModelFeature,+ ModelType,+ ParameterRule,+ ParameterType,+)+++def make_model_schema(model: str) -> AIModelEntity:+ return AIModelEntity(+ model=model,+ label=I18nObject(en_US=model),+ model_type=ModelType.LLM,+ fetch_from=FetchFrom.PREDEFINED_MODEL,+ model_properties={},+ )+++class _ModelInstance(ModelInstance):+ """Return a configured response while checking the invocation contract."""++ def __init__(self, response: LLMResult | Iterator[LLMResultChunk | SimpleNamespace]) -> None:+ self.response = response++ def invoke_llm(self, **kwargs: object):+ inspect.signature(ModelInstance.invoke_llm).bind(self, **kwargs)+ return self.responseclass TestStructuredOutput:@@ -140,8 +172,7 @@ class TestStructuredOutput:def test_handle_native_json_schema(self):provider = "openai"- model_schema = MagicMock(spec=AIModelEntity)- model_schema.model = "gpt-4"+ model_schema = make_model_schema("gpt-4")structured_output_schema = {"type": "object"}model_parameters = {}rules = [@@ -164,8 +195,7 @@ class TestStructuredOutput:def test_handle_native_json_schema_no_format_rule(self):provider = "openai"- model_schema = MagicMock(spec=AIModelEntity)- model_schema.model = "gpt-4"+ model_schema = make_model_schema("gpt-4")structured_output_schema = {"type": "object"}model_parameters = {}rules = []@@ -204,32 +234,29 @@ class TestStructuredOutput:assert isinstance(result[1], UserPromptMessage)def test_prepare_schema_for_model_gemini(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.model = "gemini-1.5-pro"+ model_schema = make_model_schema("gemini-1.5-pro")schema = {"type": "object", "additionalProperties": False}result = _prepare_schema_for_model("google", model_schema, schema)assert "additionalProperties" not in resultdef test_prepare_schema_for_model_ollama(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.model = "llama3"+ model_schema = make_model_schema("llama3")schema = {"type": "object"}result = _prepare_schema_for_model("ollama", model_schema, schema)assert result == schemadef test_prepare_schema_for_model_default(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.model = "gpt-4"+ model_schema = make_model_schema("gpt-4")schema = {"type": "object"}result = _prepare_schema_for_model("openai", model_schema, schema)assert result == {"schema": schema, "name": "llm_response"}def test_invoke_llm_with_structured_output_no_stream_native(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.support_structure_output = True+ model_schema = make_model_schema("gpt-4")+ model_schema.features = [ModelFeature.STRUCTURED_OUTPUT]model_schema.parameter_rules = [ParameterRule(name="response_format",@@ -241,15 +268,15 @@ class TestStructuredOutput:]model_schema.model = "gpt-4o"- model_instance = MagicMock(spec=ModelInstance)- mock_result = MagicMock(spec=LLMResult)- mock_result.message = AssistantPromptMessage(content='{"result": "success"}')- mock_result.model = "gpt-4o"- mock_result.usage = LLMUsage.empty_usage()- mock_result.system_fingerprint = "fp_native"- mock_result.prompt_messages = [UserPromptMessage(content="hi")]+ llm_response = LLMResult(+ model="gpt-4o",+ message=AssistantPromptMessage(content='{"result": "success"}'),+ usage=LLMUsage.empty_usage(),+ system_fingerprint="fp_native",+ prompt_messages=[UserPromptMessage(content="hi")],+ )- model_instance.invoke_llm.return_value = mock_result+ model_instance = _ModelInstance(llm_response)result = invoke_llm_with_structured_output(provider="openai",@@ -265,8 +292,8 @@ class TestStructuredOutput:assert result.system_fingerprint == "fp_native"def test_invoke_llm_with_structured_output_no_stream_prompt_based(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.support_structure_output = False+ model_schema = make_model_schema("gpt-4")+ model_schema.features = []model_schema.parameter_rules = [ParameterRule(name="response_format",@@ -278,15 +305,15 @@ class TestStructuredOutput:]model_schema.model = "claude-3"- model_instance = MagicMock(spec=ModelInstance)- mock_result = MagicMock(spec=LLMResult)- mock_result.message = AssistantPromptMessage(content='{"result": "success"}')- mock_result.model = "claude-3"- mock_result.usage = LLMUsage.empty_usage()- mock_result.system_fingerprint = "fp_prompt"- mock_result.prompt_messages = []+ llm_response = LLMResult(+ model="claude-3",+ message=AssistantPromptMessage(content='{"result": "success"}'),+ usage=LLMUsage.empty_usage(),+ system_fingerprint="fp_prompt",+ prompt_messages=[],+ )- model_instance.invoke_llm.return_value = mock_result+ model_instance = _ModelInstance(llm_response)result = invoke_llm_with_structured_output(provider="anthropic",@@ -302,15 +329,17 @@ class TestStructuredOutput:assert result.system_fingerprint == "fp_prompt"def test_invoke_llm_with_structured_output_no_string_error(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.support_structure_output = False+ model_schema = make_model_schema("gpt-4")+ model_schema.features = []model_schema.parameter_rules = []- model_instance = MagicMock(spec=ModelInstance)- mock_result = MagicMock(spec=LLMResult)- mock_result.message = AssistantPromptMessage(content=[TextPromptMessageContent(data="not a string")])+ llm_response = LLMResult(+ model=model_schema.model,+ message=AssistantPromptMessage(content=[TextPromptMessageContent(data="not a string")]),+ usage=LLMUsage.empty_usage(),+ )- model_instance.invoke_llm.return_value = mock_result+ model_instance = _ModelInstance(llm_response)with pytest.raises(OutputParserError) as excinfo:invoke_llm_with_structured_output(@@ -324,42 +353,41 @@ class TestStructuredOutput:assert "Failed to parse structured output, LLM result is not a string" in str(excinfo.value)def test_invoke_llm_with_structured_output_stream(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.support_structure_output = False+ model_schema = make_model_schema("gpt-4")+ model_schema.features = []model_schema.parameter_rules = []model_schema.model = "gpt-4"- model_instance = MagicMock(spec=ModelInstance)+ # Model chunks carry validated deltas and stream metadata.+ chunk1 = LLMResultChunk(+ model=model_schema.model,+ delta=LLMResultChunkDelta(+ index=0, message=AssistantPromptMessage(content='{"key": '), usage=LLMUsage.empty_usage()+ ),+ prompt_messages=[UserPromptMessage(content="hi")],+ system_fingerprint="fp1",+ )- # Mock chunks- chunk1 = MagicMock(spec=LLMResultChunk)- chunk1.delta = LLMResultChunkDelta(- index=0, message=AssistantPromptMessage(content='{"key": '), usage=LLMUsage.empty_usage()+ chunk2 = LLMResultChunk(+ model=model_schema.model,+ delta=LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content='"value"}')),+ prompt_messages=[UserPromptMessage(content="hi")],+ system_fingerprint="fp1",)- chunk1.prompt_messages = [UserPromptMessage(content="hi")]- chunk1.system_fingerprint = "fp1"-- chunk2 = MagicMock(spec=LLMResultChunk)- chunk2.delta = LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content='"value"}'))- chunk2.prompt_messages = [UserPromptMessage(content="hi")]- chunk2.system_fingerprint = "fp1"-- chunk3 = MagicMock(spec=LLMResultChunk)- chunk3.delta = LLMResultChunkDelta(- index=0,- message=AssistantPromptMessage(- content=[- TextPromptMessageContent(data=" "),- ]++ chunk3 = LLMResultChunk(+ model=model_schema.model,+ delta=LLMResultChunkDelta(+ index=0, message=AssistantPromptMessage(content=[TextPromptMessageContent(data=" ")])),+ prompt_messages=[UserPromptMessage(content="hi")],+ system_fingerprint="fp1",)- chunk3.prompt_messages = [UserPromptMessage(content="hi")]- chunk3.system_fingerprint = "fp1"- event4 = MagicMock()- event4.delta = LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content=""))+ # Preserve coverage for a delta-bearing event that is not an LLMResultChunk.+ event4 = SimpleNamespace(delta=LLMResultChunkDelta(index=0, message=AssistantPromptMessage(content="")))- model_instance.invoke_llm.return_value = [chunk1, chunk2, chunk3, event4]+ model_instance = _ModelInstance(iter([chunk1, chunk2, chunk3, event4]))generator = invoke_llm_with_structured_output(provider="openai",@@ -377,13 +405,12 @@ class TestStructuredOutput:assert chunks[-1].prompt_messages == [UserPromptMessage(content="hi")]def test_invoke_llm_with_structured_output_stream_no_id_events(self):- model_schema = MagicMock(spec=AIModelEntity)- model_schema.support_structure_output = False+ model_schema = make_model_schema("gpt-4")+ model_schema.features = []model_schema.parameter_rules = []model_schema.model = "gpt-4"- model_instance = MagicMock(spec=ModelInstance)- model_instance.invoke_llm.return_value = []+ model_instance = _ModelInstance(iter([]))generator = invoke_llm_with_structured_output(provider="openai",