RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(markdown): delimit a link destination that holds a space or a parenthesis (#19747)
Lukas23273576ed06
diff --git a/common/markdown_utils.py b/common/markdown_utils.pyindex c16905b025..4931db2f84 100644--- a/common/markdown_utils.py+++ b/common/markdown_utils.py@@ -25,6 +25,10 @@ from markdownify import MarkdownConverter# consuming the escape that is added._TABLE_CELL_PIPE = re.compile(r"(?<!\\)(\\*)\|")+# A bare Markdown link destination ends at the first whitespace and at an+# unbalanced closing parenthesis, and an angle bracket would close it early.+_NEEDS_ANGLE_BRACKETS = re.compile(r"[\s()<>]")+# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the# surrounding whitespace out of the text and then returns an empty string once nothing is# left. A whitespace-only element therefore disappears together with its whitespace and the@@ -61,6 +65,22 @@ def _escape_table_cell(text: str) -> str:return _TABLE_CELL_PIPE.sub(lambda match: match.group(1) * 2 + r"\|", text)+def _format_destination(url: str) -> str:+ """Render `url` so it survives being read back as a Markdown destination.++ A URL holding a space -- a SharePoint or OneDrive path, say -- or an+ unbalanced parenthesis is cut short when the Markdown is parsed again.+ Angle brackets are what CommonMark provides for the case, and they leave+ the URL itself byte for byte as it was, rather than re-encoding characters+ the server may be reading.+ """+ if not url or not _NEEDS_ANGLE_BRACKETS.search(url):+ return url++ # A `<...>` destination may not contain an unescaped angle bracket.+ return "<{}>".format(url.replace("<", "%3C").replace(">", "%3E"))++class _WhitespacePreservingConverter(MarkdownConverter):"""Same as markdownify's converter, but a whitespace-only inline element keeps its text."""@@ -83,6 +103,20 @@ class _WhitespacePreservingConverter(MarkdownConverter):def convert_th(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:return super().convert_th(el, _escape_table_cell(text), *args, **kwargs)+ def convert_a(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:+ href = el.get("href")+ if href:+ el["href"] = _format_destination(href)++ return super().convert_a(el, text, *args, **kwargs)++ def convert_img(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:+ src = el.get("src")+ if src:+ el["src"] = _format_destination(src)++ return super().convert_img(el, text, *args, **kwargs)+def html_to_markdown(html: str, **options: Any) -> str:"""Convert `html` to Markdown without losing the word boundaries it carries.diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.pyindex e3636002f7..8a81cf4c88 100644--- a/test/unit_test/common/test_markdown_utils.py+++ b/test/unit_test/common/test_markdown_utils.py@@ -153,3 +153,41 @@ def test_a_pipe_in_a_header_cell_stays_inside_the_cell():def test_a_pipe_outside_a_table_is_left_alone():assert html_to_markdown("<p>stdin | stdout</p>").strip() == "stdin | stdout"+++ ("html", "expected"),+ [+ # A SharePoint or OneDrive hyperlink in a .docx routinely holds spaces.+ (+ '<a href="https://company.sharepoint.com/Shared Documents/report.docx">Report</a>',+ "[Report](<https://company.sharepoint.com/Shared Documents/report.docx>)",+ ),+ # A closing parenthesis in a query string closes the destination early.+ ('<a href="https://example.com/s?q=a)b">result</a>', "[result](<https://example.com/s?q=a)b>)"),+ ('<img src="https://example.com/a b.png" alt="pic"/>', ""),+ ],+)+def test_a_destination_that_would_not_survive_a_reparse_is_delimited(html, expected):+ """A bare Markdown destination ends at the first space and at an unbalanced `)`.++ The truncated tail is then read as body text, so the link points somewhere else and the+ remainder of the URL is chunked and embedded as prose.+ """+ assert html_to_markdown(html).strip() == expected+++ ("html", "expected"),+ [+ ('<a href="https://example.com/a/b?x=1&y=2">ok</a>', "[ok](https://example.com/a/b?x=1&y=2)"),+ ('<img src="https://example.com/a.png" alt="pic"/>', ""),+ # A base64 data URI is what the DOCX path inlines images as; it carries no+ # character that needs delimiting, so it has to come through untouched.+ ('<img src="data:image/png;base64,iVBORw0KGgo=" alt="inline"/>', ""),+ # markdownify renders an anchor with no destination as plain text.+ ('<a href="">empty</a>', "empty"),+ ],+)+def test_a_destination_that_needs_nothing_is_left_alone(html, expected):+ assert html_to_markdown(html).strip() == expected