Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(agent): omit prompt history from LLM gateway streams (#45137)

zyssyz123efe37c59ae52
diff --git a/api/controllers/inner_api/agent/llm.py b/api/controllers/inner_api/agent/llm.py
index 27c5fe35325..bb7ba8ef385 100644
--- a/api/controllers/inner_api/agent/llm.py
+++ b/api/controllers/inner_api/agent/llm.py
@@ -76,7 +76,13 @@ class AgentLLMInvokeApi(Resource):
def generate() -> Generator[str]:
try:
for chunk in service.invoke(prepared):
- envelope = {"code": 0, "message": "", "data": chunk.model_dump(mode="json")}
+ # Agent owns its history. Do not reserialize opaque request
+ # messages through the runtime's typed PromptMessage field.
+ envelope = {
+ "code": 0,
+ "message": "",
+ "data": chunk.model_dump(mode="json", exclude={"prompt_messages"}),
+ }
yield f"data: {json.dumps(envelope, ensure_ascii=False, separators=(',', ':'))}\n\n"
except GeneratorExit:
raise
diff --git a/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py b/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py
index ff017978e7e..932a1abb281 100644
--- a/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py
+++ b/api/tests/unit_tests/controllers/inner_api/agent/test_llm.py
@@ -1,21 +1,30 @@
"""Contract tests for the trusted Agent LLM streaming gateway."""
import json
+import warnings
from collections.abc import Generator
from contextlib import contextmanager
+from copy import deepcopy
from unittest.mock import MagicMock, patch
from uuid import uuid4
+import pytest
from flask import Flask
from pydantic import JsonValue
from controllers.inner_api import bp as inner_api_bp
from core.errors.error import QuotaExceededError
-from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta
-from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage
-from services.agent_llm_inner_service import AgentLLMInnerServiceError, PreparedAgentLLMInvocation
+from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
+from graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModel
+from services.agent_llm_inner_service import (
+ AgentLLMInnerService,
+ AgentLLMInnerServiceError,
+ PreparedAgentLLMInvocation,
+)
from services.entities.agent_llm_inner import AgentLLMInvokeRequest
from tests.helpers.unit.config_override import config_overrides_context
+from tests.helpers.unit.core.model_fixtures import make_model_instance
def _payload() -> dict[str, object]:
@@ -92,7 +101,10 @@ def test_post_streams_plugin_compatible_envelope() -> None:
prepare.assert_called_once()
-def test_post_preserves_prompt_messages_without_transport_validation() -> None:
[email protected]("stream_error", [False, True], ids=["completed", "provider-error"])
+def test_post_preserves_raw_prompts_without_serializing_them(
+ monkeypatch: pytest.MonkeyPatch, stream_error: bool
+) -> None:
payload = _payload()
prompt_messages: list[dict[str, JsonValue]] = [
{
@@ -118,21 +130,69 @@ def test_post_preserves_prompt_messages_without_transport_validation() -> None:
"name": "search",
"tool_call_id": "call-1",
},
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "data": "Inspect this image"},
+ {
+ "type": "image",
+ "format": "png",
+ "url": "https://example.com/image.png",
+ "mime_type": "image/png",
+ "detail": "high",
+ "provider_extension": {"quality": "original"},
+ },
+ ],
+ },
]
target = payload["target"]
assert isinstance(target, dict)
target["prompt_messages"] = prompt_messages
- captured: dict[str, AgentLLMInvokeRequest] = {}
+ original_prompt_messages = deepcopy(prompt_messages)
+ model_instance = make_model_instance(provider="openai", model="gpt-test")
+ assert isinstance(model_instance.model_type_instance, LargeLanguageModel)
+ captured_requests: list[AgentLLMInvokeRequest] = []
+ runtime_prompt_messages: list[list[PromptMessage]] = []
+ usage = LLMUsage.from_metadata({"prompt_tokens": 11, "completion_tokens": 2, "total_price": "0.002"})
+ chunks = [
+ LLMResultChunk(
+ model="gpt-test",
+ system_fingerprint="fp-test",
+ delta=LLMResultChunkDelta(index=index, message=AssistantPromptMessage(content=content)),
+ )
+ for index, content in enumerate(["Dify", " ready", ""])
+ ]
+ chunks[1].delta.message.tool_calls = [
+ AssistantPromptMessage.ToolCall(
+ id="call-2",
+ type="function",
+ function=AssistantPromptMessage.ToolCall.ToolCallFunction(name="search", arguments='{"query":"Dify"}'),
+ )
+ ]
+ chunks[-1].delta.usage = usage
+ chunks[-1].delta.finish_reason = "tool_calls"
+
+ def prepare(_service: AgentLLMInnerService, request: AgentLLMInvokeRequest) -> PreparedAgentLLMInvocation:
+ captured_requests.append(request)
+ return PreparedAgentLLMInvocation(request=request, model_instance=model_instance)
+
+ def provider_stream(*, prompt_messages: list[PromptMessage], **_kwargs: object) -> Generator[LLMResultChunk]:
+ runtime_prompt_messages.append(prompt_messages)
+ yield from chunks[:2]
+ if stream_error:
+ raise ValueError("provider stream failed")
+ yield chunks[-1]
- def prepare(request: AgentLLMInvokeRequest) -> PreparedAgentLLMInvocation:
- captured["request"] = request
- return PreparedAgentLLMInvocation(request=request, model_instance=MagicMock())
+ # Replace only preflight database resolution and the external provider boundary;
+ # the service, model instance, and Graphon streaming wrapper execute normally.
+ monkeypatch.setattr(AgentLLMInnerService, "prepare", prepare)
+ monkeypatch.setattr(model_instance.model_type_instance.model_runtime, "invoke_llm", provider_stream)
with (
_agent_inner_auth(),
- patch("controllers.inner_api.agent.llm.AgentLLMInnerService.prepare", side_effect=prepare),
- patch("controllers.inner_api.agent.llm.AgentLLMInnerService.invoke", return_value=iter([])),
+ warnings.catch_warnings(record=True) as recorded_warnings,
):
+ warnings.simplefilter("always")
response = (
_flask_app()
.test_client()
@@ -142,9 +202,40 @@ def test_post_preserves_prompt_messages_without_transport_validation() -> None:
headers={"X-Inner-Api-Key": "inner-key"},
)
)
+ envelopes = [
+ json.loads(event.removeprefix("data: ")) for event in response.get_data(as_text=True).strip().split("\n\n")
+ ]
assert response.status_code == 200
- assert captured["request"].target.prompt_messages == prompt_messages
+ assert not any("PydanticSerializationUnexpectedValue" in str(warning.message) for warning in recorded_warnings)
+ assert captured_requests[0].target.prompt_messages == original_prompt_messages
+ assert runtime_prompt_messages == [original_prompt_messages]
+ assert prompt_messages == original_prompt_messages
+
+ successful_envelopes = envelopes[:-1] if stream_error else envelopes
+ expected_chunks = chunks[:2] if stream_error else chunks
+ assert len(successful_envelopes) == len(expected_chunks)
+ for envelope, chunk in zip(successful_envelopes, expected_chunks, strict=True):
+ assert envelope["code"] == 0
+ assert envelope["message"] == ""
+ assert "prompt_messages" not in envelope["data"]
+ assert envelope["data"]["delta"] == chunk.delta.model_dump(mode="json")
+ assert envelope["data"]["system_fingerprint"] == "fp-test"
+ assert LLMResultChunk.model_validate(envelope["data"]).model == "gpt-test"
+ # Graphon must have restored the raw history before the controller serializes.
+ assert chunk.prompt_messages == original_prompt_messages
+ assert all(isinstance(message, dict) for message in chunk.prompt_messages)
+
+ if stream_error:
+ assert envelopes[-1]["code"] == -500
+ assert envelopes[-1]["data"] is None
+ assert json.loads(envelopes[-1]["message"]) == {
+ "error_type": "ValueError",
+ "message": "provider stream failed",
+ }
+ else:
+ assert envelopes[-1]["data"]["delta"]["usage"] == usage.model_dump(mode="json")
+ assert envelopes[-1]["data"]["delta"]["finish_reason"] == "tool_calls"
def test_post_rejects_invalid_body_before_model_resolution() -> None: