RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(markdown): keep a pipe in a table cell from adding a column (#19749)
Lukasc18f0dd2b6b9
diff --git a/common/markdown_utils.py b/common/markdown_utils.pyindex 94d1e4e766..554c216742 100644--- a/common/markdown_utils.py+++ b/common/markdown_utils.py@@ -15,10 +15,16 @@#"""HTML to Markdown conversion for the ingestion paths."""+import refrom typing import Anyfrom markdownify import MarkdownConverter+# A run of backslashes that is not itself escaped, followed by the pipe it would+# otherwise escape. Matching the run is what keeps a cell's own backslash from+# consuming the escape that is added.+_TABLE_CELL_PIPE = re.compile(r"(?<!\\)(\\*)\|")+# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the# surrounding whitespace out of the text and then returns an empty string once nothing is# left. A whitespace-only element therefore disappears together with its whitespace and the@@ -45,6 +51,16 @@ _WHITESPACE_ONLY_PRESERVING_TAGS = frozenset()+def _escape_table_cell(text: str) -> str:+ """Escape the pipes in a table cell so the cell cannot add a column.++ A Markdown table row is split on every unescaped pipe, so a cell holding one+ -- a part number, a shell command, a regex alternation -- pushes the rest of+ the row into columns the header does not have.+ """+ return _TABLE_CELL_PIPE.sub(lambda match: match.group(1) * 2 + r"\|", text)++class _WhitespacePreservingConverter(MarkdownConverter):"""Same as markdownify's converter, but a whitespace-only inline element keeps its text."""@@ -61,6 +77,12 @@ class _WhitespacePreservingConverter(MarkdownConverter):return _keep_whitespace_only+ def convert_td(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:+ return super().convert_td(el, _escape_table_cell(text), *args, **kwargs)++ def convert_th(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:+ return super().convert_th(el, _escape_table_cell(text), *args, **kwargs)+def html_to_markdown(html: str, **options: Any) -> str:"""Convert `html` to Markdown without losing the word boundaries it carries.diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.pyindex 0d3c0189ef..6b464f1cd1 100644--- a/test/unit_test/common/test_markdown_utils.py+++ b/test/unit_test/common/test_markdown_utils.py@@ -19,6 +19,8 @@ markdownify drops an inline element that holds nothing but whitespace, and takeswhitespace with it, so two words end up as one in whatever gets chunked and embedded."""+import re+import pytestfrom common.markdown_utils import html_to_markdown@@ -60,3 +62,60 @@ def test_an_element_with_content_converts_as_before(html, expected):def test_markdownify_options_are_forwarded():assert html_to_markdown("<h1>Title</h1>", heading_style="ATX").strip() == "# Title"+++_UNESCAPED_PIPE = re.compile(r"(?<!\\)((?:\\\\)*)\|")+# CommonMark: a backslash escapes ASCII punctuation, and nothing else.+_ESCAPE = re.compile(r"\\([!-/:-@\[-`{-~])")+++def _rows(markdown: str) -> list[list[str]]:+ """Read the markdown table back the way a reader does.++ A row is split on every pipe that is not escaped, and each cell then has its+ backslash escapes resolved, so a cell comes back as the text it held.+ """+ rows = []+ for line in markdown.splitlines():+ line = line.strip()+ if not line.startswith("|"):+ continue+ cells = _UNESCAPED_PIPE.split(line.strip("|"))[::2]+ rows.append([_ESCAPE.sub(r"\1", cell).strip() for cell in cells])+ return rows+++ "cell",+ [+ # A part number, a shell command and a regex alternation all carry one.+ "USB-A|USB-C",+ "grep -E 'a|b'",+ "a||b",+ # A backslash already in the cell must not consume the escape.+ r"a\|b",+ # The controls: neither needs escaping.+ r"C:\path",+ "plain",+ ],+)+def test_a_pipe_in_a_table_cell_stays_inside_the_cell(cell):+ html = f"<table><tr><th>Product</th><th>Spec</th></tr><tr><td>Cable</td><td>{cell}</td></tr></table>"++ rows = _rows(html_to_markdown(html))++ assert rows[0] == ["Product", "Spec"]+ assert rows[-1] == ["Cable", cell]+++def test_a_pipe_in_a_header_cell_stays_inside_the_cell():+ html = "<table><tr><th>Cable a|b</th><th>Spec</th></tr><tr><td>1</td><td>2</td></tr></table>"++ rows = _rows(html_to_markdown(html))++ assert rows[0] == ["Cable a|b", "Spec"]+ assert rows[-1] == ["1", "2"]+++def test_a_pipe_outside_a_table_is_left_alone():+ assert html_to_markdown("<p>stdin | stdout</p>").strip() == "stdin | stdout"