Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(agent): omit prompt history from LLM gateway streams (#45137)
zyssyz123efe37c59ae52
diff --git a/api/controllers/inner_api/agent/llm.py b/api/controllers/inner_api/agent/llm.pyindex 27c5fe35325..bb7ba8ef385 100644--- a/api/controllers/inner_api/agent/llm.py+++ b/api/controllers/inner_api/agent/llm.py@@ -76,7 +76,13 @@ class AgentLLMInvokeApi(Resource):def generate() -> Generator[str]:try:for chunk in service.invoke(prepared):- envelope = {"code": 0, "message": "", "data": chunk.model_dump(mode="json")}+ # Agent owns its history. Do not reserialize opaque request+ # messages through the runtime's typed PromptMessage field.+ envelope = {+ "code": 0,+ "message": "",+ "data": chunk.model_dump(mode="json", exclude={"prompt_messages"}),+ }yield f"data: {json.dumps(envelope, ensure_ascii=False, separators=(',', ':'))}\n\n"except GeneratorExit:raisediff --git a/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py b/api/tests/unit_tests/controllers/inner_api/agent/test_llm.pyindex ff017978e7e..932a1abb281 100644--- a/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py+++ b/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py@@ -1,21 +1,30 @@"""Contract tests for the trusted Agent LLM streaming gateway."""import json+import warningsfrom collections.abc import Generatorfrom contextlib import contextmanager+from copy import deepcopyfrom unittest.mock import MagicMock, patchfrom uuid import uuid4+import pytestfrom flask import Flaskfrom pydantic import JsonValuefrom controllers.inner_api import bp as inner_api_bpfrom core.errors.error import QuotaExceededError-from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta-from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage-from services.agent_llm_inner_service import AgentLLMInnerServiceError, PreparedAgentLLMInvocation+from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta, LLMUsage+from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage+from graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModel+from services.agent_llm_inner_service import (+ AgentLLMInnerService,+ AgentLLMInnerServiceError,+ PreparedAgentLLMInvocation,+)from services.entities.agent_llm_inner import AgentLLMInvokeRequestfrom tests.helpers.unit.config_override import config_overrides_context+from tests.helpers.unit.core.model_fixtures import make_model_instancedef _payload() -> dict[str, object]:@@ -92,7 +101,10 @@ def test_post_streams_plugin_compatible_envelope() -> None:prepare.assert_called_once()-def test_post_preserves_prompt_messages_without_transport_validation() -> None:[email protected]("stream_error", [False, True], ids=["completed", "provider-error"])+def test_post_preserves_raw_prompts_without_serializing_them(+ monkeypatch: pytest.MonkeyPatch, stream_error: bool+) -> None:payload = _payload()prompt_messages: list[dict[str, JsonValue]] = [{@@ -118,21 +130,69 @@ def test_post_preserves_prompt_messages_without_transport_validation() -> None:"name": "search","tool_call_id": "call-1",},+ {+ "role": "user",+ "content": [+ {"type": "text", "data": "Inspect this image"},+ {+ "type": "image",+ "format": "png",+ "url": "https://example.com/image.png",+ "mime_type": "image/png",+ "detail": "high",+ "provider_extension": {"quality": "original"},+ },+ ],+ },]target = payload["target"]assert isinstance(target, dict)target["prompt_messages"] = prompt_messages- captured: dict[str, AgentLLMInvokeRequest] = {}+ original_prompt_messages = deepcopy(prompt_messages)+ model_instance = make_model_instance(provider="openai", model="gpt-test")+ assert isinstance(model_instance.model_type_instance, LargeLanguageModel)+ captured_requests: list[AgentLLMInvokeRequest] = []+ runtime_prompt_messages: list[list[PromptMessage]] = []+ usage = LLMUsage.from_metadata({"prompt_tokens": 11, "completion_tokens": 2, "total_price": "0.002"})+ chunks = [+ LLMResultChunk(+ model="gpt-test",+ system_fingerprint="fp-test",+ delta=LLMResultChunkDelta(index=index, message=AssistantPromptMessage(content=content)),+ )+ for index, content in enumerate(["Dify", " ready", ""])+ ]+ chunks[1].delta.message.tool_calls = [+ AssistantPromptMessage.ToolCall(+ id="call-2",+ type="function",+ function=AssistantPromptMessage.ToolCall.ToolCallFunction(name="search", arguments='{"query":"Dify"}'),+ )+ ]+ chunks[-1].delta.usage = usage+ chunks[-1].delta.finish_reason = "tool_calls"++ def prepare(_service: AgentLLMInnerService, request: AgentLLMInvokeRequest) -> PreparedAgentLLMInvocation:+ captured_requests.append(request)+ return PreparedAgentLLMInvocation(request=request, model_instance=model_instance)++ def provider_stream(*, prompt_messages: list[PromptMessage], **_kwargs: object) -> Generator[LLMResultChunk]:+ runtime_prompt_messages.append(prompt_messages)+ yield from chunks[:2]+ if stream_error:+ raise ValueError("provider stream failed")+ yield chunks[-1]- def prepare(request: AgentLLMInvokeRequest) -> PreparedAgentLLMInvocation:- captured["request"] = request- return PreparedAgentLLMInvocation(request=request, model_instance=MagicMock())+ # Replace only preflight database resolution and the external provider boundary;+ # the service, model instance, and Graphon streaming wrapper execute normally.+ monkeypatch.setattr(AgentLLMInnerService, "prepare", prepare)+ monkeypatch.setattr(model_instance.model_type_instance.model_runtime, "invoke_llm", provider_stream)with (_agent_inner_auth(),- patch("controllers.inner_api.agent.llm.AgentLLMInnerService.prepare", side_effect=prepare),- patch("controllers.inner_api.agent.llm.AgentLLMInnerService.invoke", return_value=iter([])),+ warnings.catch_warnings(record=True) as recorded_warnings,):+ warnings.simplefilter("always")response = (_flask_app().test_client()@@ -142,9 +202,40 @@ def test_post_preserves_prompt_messages_without_transport_validation() -> None:headers={"X-Inner-Api-Key": "inner-key"},))+ envelopes = [+ json.loads(event.removeprefix("data: ")) for event in response.get_data(as_text=True).strip().split("\n\n")+ ]assert response.status_code == 200- assert captured["request"].target.prompt_messages == prompt_messages+ assert not any("PydanticSerializationUnexpectedValue" in str(warning.message) for warning in recorded_warnings)+ assert captured_requests[0].target.prompt_messages == original_prompt_messages+ assert runtime_prompt_messages == [original_prompt_messages]+ assert prompt_messages == original_prompt_messages++ successful_envelopes = envelopes[:-1] if stream_error else envelopes+ expected_chunks = chunks[:2] if stream_error else chunks+ assert len(successful_envelopes) == len(expected_chunks)+ for envelope, chunk in zip(successful_envelopes, expected_chunks, strict=True):+ assert envelope["code"] == 0+ assert envelope["message"] == ""+ assert "prompt_messages" not in envelope["data"]+ assert envelope["data"]["delta"] == chunk.delta.model_dump(mode="json")+ assert envelope["data"]["system_fingerprint"] == "fp-test"+ assert LLMResultChunk.model_validate(envelope["data"]).model == "gpt-test"+ # Graphon must have restored the raw history before the controller serializes.+ assert chunk.prompt_messages == original_prompt_messages+ assert all(isinstance(message, dict) for message in chunk.prompt_messages)++ if stream_error:+ assert envelopes[-1]["code"] == -500+ assert envelopes[-1]["data"] is None+ assert json.loads(envelopes[-1]["message"]) == {+ "error_type": "ValueError",+ "message": "provider stream failed",+ }+ else:+ assert envelopes[-1]["data"]["delta"]["usage"] == usage.model_dump(mode="json")+ assert envelopes[-1]["data"]["delta"]["finish_reason"] == "tool_calls"def test_post_rejects_invalid_body_before_model_resolution() -> None: