Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(markdown): keep a pipe in a table cell from adding a column (#19749)

Lukasc18f0dd2b6b9
diff --git a/common/markdown_utils.py b/common/markdown_utils.py
index 94d1e4e766..554c216742 100644
--- a/common/markdown_utils.py
+++ b/common/markdown_utils.py
@@ -15,10 +15,16 @@
#
"""HTML to Markdown conversion for the ingestion paths."""
+import re
from typing import Any
from markdownify import MarkdownConverter
+# A run of backslashes that is not itself escaped, followed by the pipe it would
+# otherwise escape. Matching the run is what keeps a cell's own backslash from
+# consuming the escape that is added.
+_TABLE_CELL_PIPE = re.compile(r"(?<!\\)(\\*)\|")
+
# Inline tags whose markdownify conversion runs the text through chomp(), which lifts the
# surrounding whitespace out of the text and then returns an empty string once nothing is
# left. A whitespace-only element therefore disappears together with its whitespace and the
@@ -45,6 +51,16 @@ _WHITESPACE_ONLY_PRESERVING_TAGS = frozenset(
)
+def _escape_table_cell(text: str) -> str:
+ """Escape the pipes in a table cell so the cell cannot add a column.
+
+ A Markdown table row is split on every unescaped pipe, so a cell holding one
+ -- a part number, a shell command, a regex alternation -- pushes the rest of
+ the row into columns the header does not have.
+ """
+ return _TABLE_CELL_PIPE.sub(lambda match: match.group(1) * 2 + r"\|", text)
+
+
class _WhitespacePreservingConverter(MarkdownConverter):
"""Same as markdownify's converter, but a whitespace-only inline element keeps its text."""
@@ -61,6 +77,12 @@ class _WhitespacePreservingConverter(MarkdownConverter):
return _keep_whitespace_only
+ def convert_td(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:
+ return super().convert_td(el, _escape_table_cell(text), *args, **kwargs)
+
+ def convert_th(self, el: Any, text: str, *args: Any, **kwargs: Any) -> str:
+ return super().convert_th(el, _escape_table_cell(text), *args, **kwargs)
+
def html_to_markdown(html: str, **options: Any) -> str:
"""Convert `html` to Markdown without losing the word boundaries it carries.
diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.py
index 0d3c0189ef..6b464f1cd1 100644
--- a/test/unit_test/common/test_markdown_utils.py
+++ b/test/unit_test/common/test_markdown_utils.py
@@ -19,6 +19,8 @@ markdownify drops an inline element that holds nothing but whitespace, and takes
whitespace with it, so two words end up as one in whatever gets chunked and embedded.
"""
+import re
+
import pytest
from common.markdown_utils import html_to_markdown
@@ -60,3 +62,60 @@ def test_an_element_with_content_converts_as_before(html, expected):
def test_markdownify_options_are_forwarded():
assert html_to_markdown("<h1>Title</h1>", heading_style="ATX").strip() == "# Title"
+
+
+_UNESCAPED_PIPE = re.compile(r"(?<!\\)((?:\\\\)*)\|")
+# CommonMark: a backslash escapes ASCII punctuation, and nothing else.
+_ESCAPE = re.compile(r"\\([!-/:-@\[-`{-~])")
+
+
+def _rows(markdown: str) -> list[list[str]]:
+ """Read the markdown table back the way a reader does.
+
+ A row is split on every pipe that is not escaped, and each cell then has its
+ backslash escapes resolved, so a cell comes back as the text it held.
+ """
+ rows = []
+ for line in markdown.splitlines():
+ line = line.strip()
+ if not line.startswith("|"):
+ continue
+ cells = _UNESCAPED_PIPE.split(line.strip("|"))[::2]
+ rows.append([_ESCAPE.sub(r"\1", cell).strip() for cell in cells])
+ return rows
+
+
+ "cell",
+ [
+ # A part number, a shell command and a regex alternation all carry one.
+ "USB-A|USB-C",
+ "grep -E 'a|b'",
+ "a||b",
+ # A backslash already in the cell must not consume the escape.
+ r"a\|b",
+ # The controls: neither needs escaping.
+ r"C:\path",
+ "plain",
+ ],
+)
+def test_a_pipe_in_a_table_cell_stays_inside_the_cell(cell):
+ html = f"<table><tr><th>Product</th><th>Spec</th></tr><tr><td>Cable</td><td>{cell}</td></tr></table>"
+
+ rows = _rows(html_to_markdown(html))
+
+ assert rows[0] == ["Product", "Spec"]
+ assert rows[-1] == ["Cable", cell]
+
+
+def test_a_pipe_in_a_header_cell_stays_inside_the_cell():
+ html = "<table><tr><th>Cable a|b</th><th>Spec</th></tr><tr><td>1</td><td>2</td></tr></table>"
+
+ rows = _rows(html_to_markdown(html))
+
+ assert rows[0] == ["Cable a|b", "Spec"]
+ assert rows[-1] == ["1", "2"]
+
+
+def test_a_pipe_outside_a_table_is_left_alone():
+ assert html_to_markdown("<p>stdin | stdout</p>").strip() == "stdin | stdout"