Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): preserve code fences during table normalization (#4494)

fengyue-xveef23a1641b9a
diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.py
index bbf6baff3..bf1eff476 100644
--- a/docreader/parser/markdown_parser.py
+++ b/docreader/parser/markdown_parser.py
@@ -33,6 +33,7 @@ _ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL)
# Cells of a GFM alignment row as matched by the previous ``align_pattern``.
# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies.
_ALIGNMENT_CELL = re.compile(r"^[:-]+$")
+_CODE_FENCE = re.compile(r" {0,3}(`{3,}|~{3,})(.*)$")
class MarkdownTableUtil:
@@ -141,6 +142,38 @@ class MarkdownTableUtil:
return "\n".join(out)
def format_table(self, content: str) -> str:
+ """Normalize tables outside fenced code blocks, preserving code examples."""
+ unified = content.replace("\r\n", "\n").replace("\r", "\n")
+ out: List[str] = []
+ pending: List[str] = []
+ fence: Optional[str] = None
+
+ def flush_pending() -> None:
+ if pending:
+ out.extend(self._format_table_block("\n".join(pending)).split("\n"))
+ pending.clear()
+
+ for line in unified.split("\n"):
+ marker = _CODE_FENCE.fullmatch(line)
+ if fence is not None:
+ out.append(line)
+ if (
+ marker
+ and marker[1][0] == fence[0]
+ and len(marker[1]) >= len(fence)
+ and not marker[2].strip(" \t")
+ ):
+ fence = None
+ elif marker and (marker[1][0] != "`" or "`" not in marker[2]):
+ flush_pending()
+ fence = marker[1]
+ out.append(line)
+ else:
+ pending.append(line)
+ flush_pending()
+ return "\n".join(out)
+
+ def _format_table_block(self, content: str) -> str:
"""Format all Markdown tables in the content.
Line-based and regex-free: only well-formed rows (leading *and*
diff --git a/docreader/tests/test_markdown_table_fences.py b/docreader/tests/test_markdown_table_fences.py
new file mode 100644
index 000000000..62c0c13b5
--- /dev/null
+++ b/docreader/tests/test_markdown_table_fences.py
@@ -0,0 +1,47 @@
+"""Table normalization must preserve code examples in fenced blocks."""
+
+import unittest
+
+from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil
+
+
+class MarkdownTableFenceTest(unittest.TestCase):
+ def test_fenced_rows_are_preserved_and_surrounding_tables_are_formatted(self):
+ for fence, closing in (
+ ("```markdown", "```"),
+ ("~~~text", "~~~"),
+ ("````markdown", "````"),
+ (" ```text", " ```"),
+ ):
+ code = fence + "\n|a|b|\n|---|---|\n|1|2|\n" + closing
+ table = "|Name|Value|\n|---|---|\n|x|1|"
+ formatted_table = "| Name | Value |\n| --- | --- |\n| x | 1 |"
+ with self.subTest(fence=fence):
+ self.assertEqual(
+ MarkdownTableUtil().format_table(table + "\n\n" + code + "\n\n" + table),
+ formatted_table + "\n\n" + code + "\n\n" + formatted_table,
+ )
+
+ def test_shorter_or_different_fence_does_not_close_code(self):
+ raw = "````markdown\n```\n|a|b|\n~~~\n|c|d|\n`````\n"
+ self.assertEqual(MarkdownTableUtil().format_table(raw), raw)
+
+ def test_unclosed_fence_keeps_remaining_rows(self):
+ raw = "~~~example\n|a|b|\n|---|---|\n|1|2|\n"
+ self.assertEqual(MarkdownTableUtil().format_table(raw), raw)
+
+ def test_fence_with_trailing_text_does_not_close_code(self):
+ raw = "```text\n``` not a closer\n|a|b|\n```\n"
+ self.assertEqual(MarkdownTableUtil().format_table(raw), raw)
+
+ def test_backtick_in_info_string_is_not_an_opening_fence(self):
+ raw = "```lang`example\n|a|b|\n|---|---|"
+ self.assertEqual(
+ MarkdownTableUtil().format_table(raw),
+ "```lang`example\n| a | b |\n| --- | --- |",
+ )
+
+ def test_uploaded_markdown_preserves_fenced_table_source(self):
+ raw = "```markdown\n|a|b|\n|---|---|\n|1|2|\n```\n"
+ document = MarkdownParser(file_name="example.md", file_type="md").parse_into_text(raw.encode())
+ self.assertIn(raw.rstrip(), document.content)