Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): combine UTF-16 surrogate pairs in RTF extraction (#44565)

Jaya Chandra Vennam1ba3ef469cdd
diff --git a/api/services/skill_management_service.py b/api/services/skill_management_service.py
index 10a83790fd9..18358d46120 100644
--- a/api/services/skill_management_service.py
+++ b/api/services/skill_management_service.py
@@ -4794,7 +4794,14 @@ class SkillManagementService:
text_parts.append(source[index + 1])
index += 2
- text = re.sub(r"[ \t]+\n", "\n", "".join(text_parts))
+ raw_text = "".join(text_parts)
+ text = re.sub(
+ r"([\uD800-\uDBFF])([\uDC00-\uDFFF])",
+ lambda m: chr(0x10000 + ((ord(m.group(1)) - 0xD800) << 10) + (ord(m.group(2)) - 0xDC00)),
+ raw_text,
+ )
+ text = re.sub(r"[\uD800-\uDFFF]", "\ufffd", text)
+ text = re.sub(r"[ \t]+\n", "\n", text)
text = re.sub(r"\n{3,}", "\n\n", text).strip()
if len(text) > max_chars:
return f"{text[:max_chars]}\n[TRUNCATED]"
diff --git a/api/tests/unit_tests/services/test_skill_management_service.py b/api/tests/unit_tests/services/test_skill_management_service.py
index f7d2ca066ab..ce323a0ec01 100644
--- a/api/tests/unit_tests/services/test_skill_management_service.py
+++ b/api/tests/unit_tests/services/test_skill_management_service.py
@@ -3442,6 +3442,10 @@ def test_pptx_extraction_ignores_unreferenced_external_and_missing_slide_parts()
pytest.param(b"{\\rtf1 Caf\xe9}", "Café", id="latin1-literal-text"),
pytest.param(rb"{\rtf1\ansi caf\'e9}", "café", id="hex-literal-text"),
pytest.param(rb"{\rtf1\ansi \{notes\} \\ path}", r"{notes} \ path", id="escaped-literal-text"),
+ pytest.param(rb"{\rtf1\ansi \u-10179?\u-8704? followed}", "😀 followed", id="surrogate-pair-signed"),
+ pytest.param(rb"{\rtf1\ansi \u55357?\u56832? followed}", "😀 followed", id="surrogate-pair-unsigned"),
+ pytest.param(rb"{\rtf1\ansi \u-10179? alone}", "\ufffd alone", id="unmatched-high-surrogate"),
+ pytest.param(rb"{\rtf1\ansi \u-8704? alone}", "\ufffd alone", id="unmatched-low-surrogate"),
],
)
def test_build_assistant_attachment_context_extracts_rtf_text(payload: bytes, expected: str) -> None:
@@ -3462,6 +3466,14 @@ def test_build_assistant_attachment_context_extracts_rtf_text(payload: bytes, ex
)
assert context == f"--- notes.rtf (application/rtf, {len(payload)} bytes) ---\n{expected}"
+ assert context.encode("utf-8")
+
+
+def test_extract_rtf_text_truncation_preserves_supplementary_characters() -> None:
+ payload = rb"{\rtf1\ansi \u-10179?\u-8704?tail}"
+ extracted = SkillManagementService._extract_rtf_text(payload, max_chars=1)
+ assert extracted == "😀\n[TRUNCATED]"
+ assert extracted.encode("utf-8")
def test_build_assistant_image_contents_encodes_images_for_vision_models() -> None: