Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(markdown): read the first row of a header-less table as its header (#19750)

Lukasb247ba009466
diff --git a/common/markdown_utils.py b/common/markdown_utils.py
index 554c216742..c16905b025 100644
--- a/common/markdown_utils.py
+++ b/common/markdown_utils.py
@@ -89,4 +89,11 @@ def html_to_markdown(html: str, **options: Any) -> str:
`options` are markdownify's own, so this is a drop-in for `markdownify(html, ...)`.
"""
+ # A Markdown table cannot start with a body row, so markdownify puts an empty
+ # header above a table that has no <th>, and the column names end up in the
+ # first body row. Word writes exactly such a table -- mammoth emits <thead>
+ # only for a row the author marked as repeating -- and the header is what
+ # gives every value in the table its meaning.
+ options.setdefault("table_infer_header", True)
+
return _WhitespacePreservingConverter(**options).convert(html)
diff --git a/test/unit_test/common/test_markdown_utils.py b/test/unit_test/common/test_markdown_utils.py
index 6b464f1cd1..e3636002f7 100644
--- a/test/unit_test/common/test_markdown_utils.py
+++ b/test/unit_test/common/test_markdown_utils.py
@@ -64,6 +64,40 @@ def test_markdownify_options_are_forwarded():
assert html_to_markdown("<h1>Title</h1>", heading_style="ATX").strip() == "# Title"
+def _table_lines(markdown):
+ return [line.strip() for line in markdown.splitlines() if line.strip().startswith("|")]
+
+
+ "html",
+ [
+ # What mammoth hands over for a Word table: no <thead>, no <th>.
+ "<table><tr><td>Product</td><td>Spec</td></tr><tr><td>Cable</td><td>USB-C</td></tr></table>",
+ # The same table wrapped in a <tbody>, which is what a browser's DOM has.
+ "<table><tbody><tr><td>Product</td><td>Spec</td></tr><tr><td>Cable</td><td>USB-C</td></tr></tbody></table>",
+ ],
+)
+def test_a_table_without_th_uses_its_first_row_as_the_header(html):
+ assert _table_lines(html_to_markdown(html)) == ["| Product | Spec |", "| --- | --- |", "| Cable | USB-C |"]
+
+
+ "html",
+ [
+ "<table><tr><th>Product</th><th>Spec</th></tr><tr><td>Cable</td><td>USB-C</td></tr></table>",
+ "<table><thead><tr><th>Product</th><th>Spec</th></tr></thead><tbody><tr><td>Cable</td><td>USB-C</td></tr></tbody></table>",
+ ],
+)
+def test_a_table_that_marks_its_header_is_unchanged(html):
+ assert _table_lines(html_to_markdown(html)) == ["| Product | Spec |", "| --- | --- |", "| Cable | USB-C |"]
+
+
+def test_the_caller_can_still_ask_for_the_empty_header():
+ html = "<table><tr><td>Product</td><td>Spec</td></tr><tr><td>Cable</td><td>USB-C</td></tr></table>"
+
+ assert _table_lines(html_to_markdown(html, table_infer_header=False))[0] == "| | |"
+
+
_UNESCAPED_PIPE = re.compile(r"(?<!\\)((?:\\\\)*)\|")
# CommonMark: a backslash escapes ASCII punctuation, and nothing else.
_ESCAPE = re.compile(r"\\([!-/:-@\[-`{-~])")