Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(markdown): delimit a link destination that holds a space or a parenthesis (#19747)

Lukas23273576ed06
diff --git a/common/markdown_utils.py b/common/markdown_utils.py
index c16905b025..4931db2f84 100644
--- a/common/markdown_utils.py
+++ b/common/markdown_utils.py
@@ -25,6 +25,10 @@ from markdownify import MarkdownConverter
# consuming the escape that is added.
_TABLE_CELL_PIPE = re.compile(r"(?<!\\)(\\*)\|")
+# A bare Markdown link destination ends at the first whitespace and at an
+# unbalanced closing parenthesis, and an angle bracket would close it early.
+_NEEDS_ANGLE_BRACKETS = re.compile(r"[\s()<>]")
+
# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the
# surrounding whitespace out of the text and then returns an empty string once nothing is
# left. A whitespace-only element therefore disappears together with its whitespace and the
@@ -61,6 +65,22 @@ def _escape_table_cell(text: str) -> str:
return _TABLE_CELL_PIPE.sub(lambda match: match.group(1) * 2 + r"\|", text)
+def _format_destination(url: str) -> str:
+ """Render `url` so it survives being read back as a Markdown destination.
+
+ A URL holding a space -- a SharePoint or OneDrive path, say -- or an
+ unbalanced parenthesis is cut short when the Markdown is parsed again.
+ Angle brackets are what CommonMark provides for the case, and they leave
+ the URL itself byte for byte as it was, rather than re-encoding characters
+ the server may be reading.
+ """
+ if not url or not _NEEDS_ANGLE_BRACKETS.search(url):
+ return url
+
+ # A `<...>` destination may not contain an unescaped angle bracket.
+ return "<{}>".format(url.replace("<", "%3C").replace(">", "%3E"))
+
+
class _WhitespacePreservingConverter(MarkdownConverter):
"""Same as markdownify's converter, but a whitespace-only inline element keeps its text."""
@@ -83,6 +103,20 @@ class _WhitespacePreservingConverter(MarkdownConverter):
def convert_th(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:
return super().convert_th(el, _escape_table_cell(text), *args, **kwargs)
+ def convert_a(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:
+ href = el.get("href")
+ if href:
+ el["href"] = _format_destination(href)
+
+ return super().convert_a(el, text, *args, **kwargs)
+
+ def convert_img(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:
+ src = el.get("src")
+ if src:
+ el["src"] = _format_destination(src)
+
+ return super().convert_img(el, text, *args, **kwargs)
+
def html_to_markdown(html: str, **options: Any) -> str:
"""Convert `html` to Markdown without losing the word boundaries it carries.
diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.py
index e3636002f7..8a81cf4c88 100644
--- a/test/unit_test/common/test_markdown_utils.py
+++ b/test/unit_test/common/test_markdown_utils.py
@@ -153,3 +153,41 @@ def test_a_pipe_in_a_header_cell_stays_inside_the_cell():
def test_a_pipe_outside_a_table_is_left_alone():
assert html_to_markdown("<p>stdin | stdout</p>").strip() == "stdin | stdout"
+
+
+ ("html", "expected"),
+ [
+ # A SharePoint or OneDrive hyperlink in a .docx routinely holds spaces.
+ (
+ '<a href="https://company.sharepoint.com/Shared Documents/report.docx">Report</a>',
+ "[Report](<https://company.sharepoint.com/Shared Documents/report.docx>)",
+ ),
+ # A closing parenthesis in a query string closes the destination early.
+ ('<a href="https://example.com/s?q=a)b">result</a>', "[result](<https://example.com/s?q=a)b>)"),
+ ('<img src="https://example.com/a b.png" alt="pic"/>', "![pic](<https://example.com/a b.png>)"),
+ ],
+)
+def test_a_destination_that_would_not_survive_a_reparse_is_delimited(html, expected):
+ """A bare Markdown destination ends at the first space and at an unbalanced `)`.
+
+ The truncated tail is then read as body text, so the link points somewhere else and the
+ remainder of the URL is chunked and embedded as prose.
+ """
+ assert html_to_markdown(html).strip() == expected
+
+
+ ("html", "expected"),
+ [
+ ('<a href="https://example.com/a/b?x=1&amp;y=2">ok</a>', "[ok](https://example.com/a/b?x=1&y=2)"),
+ ('<img src="https://example.com/a.png" alt="pic"/>', "![pic](https://example.com/a.png)"),
+ # A base64 data URI is what the DOCX path inlines images as; it carries no
+ # character that needs delimiting, so it has to come through untouched.
+ ('<img src="data:image/png;base64,iVBORw0KGgo=" alt="inline"/>', "![inline](data:image/png;base64,iVBORw0KGgo=)"),
+ # markdownify renders an anchor with no destination as plain text.
+ ('<a href="">empty</a>', "empty"),
+ ],
+)
+def test_a_destination_that_needs_nothing_is_left_alone(html, expected):
+ assert html_to_markdown(html).strip() == expected