Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: normalize json part before deduplicating (#42196)

Yunlu Wenb85102c8e85d
diff --git a/api/core/tools/tool_engine.py b/api/core/tools/tool_engine.py
index d9761eb4dbd..9c747acc2f8 100644
--- a/api/core/tools/tool_engine.py
+++ b/api/core/tools/tool_engine.py
@@ -273,8 +273,15 @@ class ToolEngine:
# Add JSON parts, avoiding duplicates from text parts.
if json_parts:
- existing_parts = set(parts)
- parts.extend(p for p in json_parts if p not in existing_parts)
+
+ def normalize_json(text: str) -> str:
+ try:
+ return json.dumps(json.loads(text), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+ except (ValueError, RecursionError):
+ return text
+
+ existing_parts = {normalize_json(p) for p in parts}
+ parts.extend(p for p in json_parts if normalize_json(p) not in existing_parts)
return "".join(parts)
diff --git a/api/tests/unit_tests/core/tools/test_tool_engine.py b/api/tests/unit_tests/core/tools/test_tool_engine.py
index f688c68cfa6..941dbb22a7d 100644
--- a/api/tests/unit_tests/core/tools/test_tool_engine.py
+++ b/api/tests/unit_tests/core/tools/test_tool_engine.py
@@ -162,6 +162,90 @@ def test_convert_tool_response_to_str_and_extract_binary_messages():
)
[email protected]("json_first", [True, False])
+ ("text", "json_object"),
+ [
+ pytest.param('{"a":1,"b":2}', {"a": 1, "b": 2}, id="compact"),
+ pytest.param(' \r\n{\n "a": 1,\n "b": 2\n}\t', {"a": 1, "b": 2}, id="whitespace"),
+ pytest.param('{"b": 2, "a": 1}', {"a": 1, "b": 2}, id="key-order"),
+ pytest.param(
+ '{"items":[{"b":2,"a":1}],"nested":{"y":2,"x":1}}',
+ {"nested": {"x": 1, "y": 2}, "items": [{"a": 1, "b": 2}]},
+ id="nested-key-order",
+ ),
+ pytest.param(r'{"\u540d\u79f0":"\u4e2d\u6587"}', {"名称": "中文"}, id="unicode-escapes"),
+ pytest.param(r'{"path":"\/a\/b"}', {"path": "/a/b"}, id="slash-escapes"),
+ pytest.param('{"a":1e0}', {"a": 1.0}, id="exponent"),
+ pytest.param(" { } ", {}, id="empty-object"),
+ pytest.param('[ {"b":2,"a":1},null,true ]', [{"a": 1, "b": 2}, None, True], id="array"),
+ ],
+)
+def test_convert_tool_response_deduplicates_equivalent_json(
+ text: str, json_object: dict[str, Any] | list[Any], json_first: bool
+):
+ tool = _build_tool()
+ messages = [
+ ToolInvokeMessage(
+ type=ToolInvokeMessage.MessageType.JSON,
+ message=ToolInvokeMessage.JsonMessage(json_object=json_object),
+ ),
+ tool.create_text_message(text),
+ ]
+ if not json_first:
+ messages.reverse()
+
+ assert ToolEngine.tool_response_to_str(messages) == text
+
+
+ ("text", "json_object", "json_text"),
+ [
+ pytest.param("plain text", {"a": 1}, '{"a": 1}', id="plain-text"),
+ pytest.param("[" * 2000 + "0" + "]" * 2000, {"a": 1}, '{"a": 1}', id="deeply-nested-text"),
+ pytest.param("9" * 5000, {"a": 1}, '{"a": 1}', id="oversized-integer"),
+ pytest.param("{'a': 1}", {"a": 1}, '{"a": 1}', id="invalid-json"),
+ pytest.param('result: {"a":1}', {"a": 1}, '{"a": 1}', id="embedded-json"),
+ pytest.param('{"a":1}{"a":1}', {"a": 1}, '{"a": 1}', id="concatenated-json"),
+ pytest.param('{"a":1}', {"a": 2}, '{"a": 2}', id="different-value"),
+ pytest.param('{"a":1}', {"b": 1}, '{"b": 1}', id="different-key"),
+ pytest.param('{"a":[1,2]}', {"a": [2, 1]}, '{"a": [2, 1]}', id="array-order"),
+ pytest.param('{"a":true}', {"a": 1}, '{"a": 1}', id="boolean-vs-number"),
+ pytest.param('{"a":null}', {"a": False}, '{"a": false}', id="null-vs-boolean"),
+ pytest.param('{"a":"1"}', {"a": 1}, '{"a": 1}', id="string-vs-number"),
+ pytest.param('{"a":"a b"}', {"a": "ab"}, '{"a": "ab"}', id="string-whitespace"),
+ ],
+)
+def test_convert_tool_response_preserves_distinct_json(text: str, json_object: dict[str, Any], json_text: str):
+ tool = _build_tool()
+ messages = [tool.create_json_message(json_object), tool.create_text_message(text)]
+
+ assert ToolEngine.tool_response_to_str(messages) == text + json_text
+
+
[email protected]("suppress_output", [True, False])
+def test_convert_tool_response_preserves_json_only_output(suppress_output: bool):
+ tool = _build_tool()
+ messages = [tool.create_json_message({"b": 2, "a": "中文"}, suppress_output=suppress_output)]
+
+ assert ToolEngine.tool_response_to_str(messages) == ("" if suppress_output else '{"b": 2, "a": "中文"}')
+
+
+def test_convert_tool_response_deduplicates_only_matching_json():
+ tool = _build_tool()
+ text = '{"b":2,"a":1}'
+ messages = [
+ tool.create_json_message({"a": 1, "b": 2}),
+ tool.create_text_message(text),
+ tool.create_text_message("result"),
+ tool.create_text_message(text),
+ tool.create_json_message({"c": 3}),
+ tool.create_json_message({"c": 3}),
+ ]
+
+ assert ToolEngine.tool_response_to_str(messages) == text + "result" + text + '{"c": 3}{"c": 3}'
+
+
@pytest.mark.parametrize("sqlite_session", [(MessageFile,)], indirect=True)
def test_create_message_files_and_invoke_generator(sqlite_engine: Engine, sqlite_session: Session):
binaries = [