Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: honor RTF Unicode fallback lengths in skill attachments (#43789)

LimbC-Caf8a7a43293a
diff --git a/api/services/skill_management_service.py b/api/services/skill_management_service.py
index ee4b325a1f9..a2e3ead7c03 100644
--- a/api/services/skill_management_service.py
+++ b/api/services/skill_management_service.py
@@ -4685,23 +4685,33 @@ class SkillManagementService:
@staticmethod
def _extract_rtf_text(payload: bytes, *, max_chars: int) -> str:
try:
- source = payload.decode("utf-8")
+ payload.decode("utf-8")
+ text_encoding = "utf-8"
except UnicodeDecodeError:
- source = payload.decode("latin-1", errors="replace")
+ text_encoding = "latin-1"
+ # Keep source positions byte-aligned: \bin lengths count bytes, not decoded characters.
+ source = payload.decode("latin-1")
text_parts: list[str] = []
destination_skip_depth: int | None = None
depth = 0
+ unicode_skip = 1
+ unicode_skip_stack: list[int] = []
+ fallback_remaining = 0
index = 0
while index < len(source):
char = source[index]
if char == "{":
+ unicode_skip_stack.append(unicode_skip)
+ fallback_remaining = 0
depth += 1
index += 1
if source[index : index + 1] == "\\" and source[index + 1 : index + 2] == "*":
destination_skip_depth = depth
continue
if char == "}":
+ unicode_skip = unicode_skip_stack.pop() if unicode_skip_stack else 1
+ fallback_remaining = 0
if destination_skip_depth is not None and depth <= destination_skip_depth:
destination_skip_depth = None
depth = max(0, depth - 1)
@@ -4711,7 +4721,17 @@ class SkillManagementService:
index += 1
continue
if char != "\\":
- text_parts.append("\n" if char in "\r\n" else char)
+ if fallback_remaining:
+ if char not in "\r\n\x00":
+ fallback_remaining -= 1
+ else:
+ end = index + 1
+ while end < len(source) and source[end] not in "\\{}":
+ end += 1
+ literal = payload[index:end].decode(text_encoding, errors="replace")
+ text_parts.append(literal.replace("\r", "\n"))
+ index = end
+ continue
index += 1
continue
@@ -4719,7 +4739,15 @@ class SkillManagementService:
if match:
word = match.group(1)
value = match.group(2)
- if word in {"par", "line"}:
+ index += len(match.group(0))
+ if fallback_remaining:
+ # A control word (including its binary payload) counts as one fallback character.
+ fallback_remaining -= 1
+ if word == "bin" and value is not None:
+ index += max(0, int(value))
+ elif word == "uc" and value is not None:
+ unicode_skip = max(0, int(value))
+ elif word in {"par", "line"}:
text_parts.append("\n")
elif word == "tab":
text_parts.append("\t")
@@ -4728,19 +4756,20 @@ class SkillManagementService:
if codepoint < 0:
codepoint += 65536
text_parts.append(chr(codepoint))
- index += len(match.group(0))
- if word == "u" and source[index : index + 2].startswith("\\'"):
- index += 4
- elif word == "u" and index < len(source):
- index += 1
+ fallback_remaining = unicode_skip
continue
if source[index : index + 2] == "\\'":
- with suppress(ValueError):
- text_parts.append(bytes.fromhex(source[index + 2 : index + 4]).decode("latin-1"))
+ if fallback_remaining:
+ fallback_remaining -= 1
+ else:
+ with suppress(ValueError):
+ text_parts.append(bytes.fromhex(source[index + 2 : index + 4]).decode("latin-1"))
index += 4
continue
- if index + 1 < len(source):
+ if fallback_remaining:
+ fallback_remaining -= 1
+ elif index + 1 < len(source):
text_parts.append(source[index + 1])
index += 2
diff --git a/api/tests/unit_tests/services/test_skill_management_service.py b/api/tests/unit_tests/services/test_skill_management_service.py
index 866f42e9a56..abc0a57ffe6 100644
--- a/api/tests/unit_tests/services/test_skill_management_service.py
+++ b/api/tests/unit_tests/services/test_skill_management_service.py
@@ -3374,27 +3374,52 @@ def test_pptx_extraction_ignores_unreferenced_external_and_missing_slide_parts()
assert SkillManagementService._extract_pptx_text(payload, max_chars=1000) == "keep"
-def test_build_assistant_attachment_context_extracts_rtf_text() -> None:
+ ("payload", "expected"),
+ [
+ pytest.param(
+ rb"{\rtf1\ansi Skill Builder\par reads RTF notes.}", "Skill Builder\nreads RTF notes.", id="ascii"
+ ),
+ pytest.param(rb"{\rtf1\ansi \u945? followed}", "α followed", id="default-uc1"),
+ pytest.param(rb"{\rtf1\ansi\uc0\u20320\u22909!}", "你好!", id="uc0-adjacent-unicode"),
+ pytest.param(rb"{\rtf1\ansi\uc2\u945?? followed}", "α followed", id="uc2-plain-fallback"),
+ pytest.param(rb"{\rtf1\ansi\uc2\u945\'3f\'3f followed}", "α followed", id="uc2-hex-fallback"),
+ pytest.param(rb"{\rtf1\ansi\u945\{ followed}", "α followed", id="escaped-brace-fallback"),
+ pytest.param(rb"{\rtf1\ansi\u945\tab followed}", "αfollowed", id="control-word-fallback"),
+ pytest.param(rb"{\rtf1\ansi\uc2\u945??{\uc0\u946!}\u947?? end}", "αβ!γ end", id="group-scope"),
+ pytest.param(rb"{\rtf1\ansi\uc2{\u945?}\u946?? end}", "αβ end", id="group-ends-fallback"),
+ pytest.param(b"{\\rtf1\\ansi\\uc2\\u945?\r\n? followed}", "α followed", id="newline-in-fallback"),
+ pytest.param(rb"{\rtf1\ansi\uc0\u-255!}", "!!", id="signed-unicode"),
+ pytest.param(rb"{\rtf1\ansi\u945\bin3 abc followed}", "α followed", id="binary-fallback"),
+ pytest.param(rb"{\rtf1\ansi{\*\hidden\uc0 discarded}\u945? kept}", "α kept", id="ignored-group"),
+ pytest.param(b"{\\rtf1\\u945\\bin2 \xc3\xa9OK}", "αOK", id="utf8-binary-fallback"),
+ pytest.param(b"{\\rtf1\\u945\\bin2 \xc3\xa9OK{\\*\\hidden \xff}}", "αOK", id="latin1-binary-fallback"),
+ pytest.param(b"{\\rtf1\\u945\\bin4 \xf0\x9f\x98\x80OK}", "αOK", id="utf8-four-byte-binary-fallback"),
+ pytest.param(b"{\\rtf1\\u945\\bin6 \xc3\xa9\xf0\x9f\x98\x80OK}", "αOK", id="utf8-mixed-binary-fallback"),
+ pytest.param("{\\rtf1 Café 中文}".encode(), "Café 中文", id="utf8-literal-text"),
+ pytest.param(b"{\\rtf1 Caf\xe9}", "Café", id="latin1-literal-text"),
+ pytest.param(rb"{\rtf1\ansi caf\'e9}", "café", id="hex-literal-text"),
+ pytest.param(rb"{\rtf1\ansi \{notes\} \\ path}", r"{notes} \ path", id="escaped-literal-text"),
+ ],
+)
+def test_build_assistant_attachment_context_extracts_rtf_text(payload: bytes, expected: str) -> None:
attachment = SkillAssistAttachmentPayload(
tool_file_id="rtf-file-1",
name="notes.rtf",
mime_type="application/rtf",
- size=74,
+ size=len(payload),
)
with patch(
"services.skill_management_service.SkillManagementService._load_assistant_tool_file_bytes",
- return_value=b"{\\rtf1\\ansi Skill Builder\\par reads RTF notes.}",
+ return_value=payload,
):
context = SkillManagementService._build_assistant_attachment_context(
tenant_id=TENANT,
attachments=[attachment],
)
- assert "--- notes.rtf" in context
- assert "Skill Builder" in context
- assert "reads RTF notes." in context
- assert "Binary attachment available" not in context
+ assert context == f"--- notes.rtf (application/rtf, {len(payload)} bytes) ---\n{expected}"
def test_build_assistant_image_contents_encodes_images_for_vision_models() -> None: