Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): preserve code fences during table normalization (#4494)
fengyue-xveef23a1641b9a
diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.pyindex bbf6baff3..bf1eff476 100644--- a/docreader/parser/markdown_parser.py+++ b/docreader/parser/markdown_parser.py@@ -33,6 +33,7 @@ _ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL)# Cells of a GFM alignment row as matched by the previous ``align_pattern``.# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies._ALIGNMENT_CELL = re.compile(r"^[:-]+$")+_CODE_FENCE = re.compile(r" {0,3}(`{3,}|~{3,})(.*)$")class MarkdownTableUtil:@@ -141,6 +142,38 @@ class MarkdownTableUtil:return "\n".join(out)def format_table(self, content: str) -> str:+ """Normalize tables outside fenced code blocks, preserving code examples."""+ unified = content.replace("\r\n", "\n").replace("\r", "\n")+ out: List[str] = []+ pending: List[str] = []+ fence: Optional[str] = None++ def flush_pending() -> None:+ if pending:+ out.extend(self._format_table_block("\n".join(pending)).split("\n"))+ pending.clear()++ for line in unified.split("\n"):+ marker = _CODE_FENCE.fullmatch(line)+ if fence is not None:+ out.append(line)+ if (+ marker+ and marker[1][0] == fence[0]+ and len(marker[1]) >= len(fence)+ and not marker[2].strip(" \t")+ ):+ fence = None+ elif marker and (marker[1][0] != "`" or "`" not in marker[2]):+ flush_pending()+ fence = marker[1]+ out.append(line)+ else:+ pending.append(line)+ flush_pending()+ return "\n".join(out)++ def _format_table_block(self, content: str) -> str:"""Format all Markdown tables in the content.Line-based and regex-free: only well-formed rows (leading *and*diff --git a/docreader/tests/test_markdown_table_fences.py b/docreader/tests/test_markdown_table_fences.pynew file mode 100644index 000000000..62c0c13b5--- /dev/null+++ b/docreader/tests/test_markdown_table_fences.py@@ -0,0 +1,47 @@+"""Table normalization must preserve code examples in fenced blocks."""++import unittest++from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil+++class MarkdownTableFenceTest(unittest.TestCase):+ def test_fenced_rows_are_preserved_and_surrounding_tables_are_formatted(self):+ for fence, closing in (+ ("```markdown", "```"),+ ("~~~text", "~~~"),+ ("````markdown", "````"),+ (" ```text", " ```"),+ ):+ code = fence + "\n|a|b|\n|---|---|\n|1|2|\n" + closing+ table = "|Name|Value|\n|---|---|\n|x|1|"+ formatted_table = "| Name | Value |\n| --- | --- |\n| x | 1 |"+ with self.subTest(fence=fence):+ self.assertEqual(+ MarkdownTableUtil().format_table(table + "\n\n" + code + "\n\n" + table),+ formatted_table + "\n\n" + code + "\n\n" + formatted_table,+ )++ def test_shorter_or_different_fence_does_not_close_code(self):+ raw = "````markdown\n```\n|a|b|\n~~~\n|c|d|\n`````\n"+ self.assertEqual(MarkdownTableUtil().format_table(raw), raw)++ def test_unclosed_fence_keeps_remaining_rows(self):+ raw = "~~~example\n|a|b|\n|---|---|\n|1|2|\n"+ self.assertEqual(MarkdownTableUtil().format_table(raw), raw)++ def test_fence_with_trailing_text_does_not_close_code(self):+ raw = "```text\n``` not a closer\n|a|b|\n```\n"+ self.assertEqual(MarkdownTableUtil().format_table(raw), raw)++ def test_backtick_in_info_string_is_not_an_opening_fence(self):+ raw = "```lang`example\n|a|b|\n|---|---|"+ self.assertEqual(+ MarkdownTableUtil().format_table(raw),+ "```lang`example\n| a | b |\n| --- | --- |",+ )++ def test_uploaded_markdown_preserves_fenced_table_source(self):+ raw = "```markdown\n|a|b|\n|---|---|\n|1|2|\n```\n"+ document = MarkdownParser(file_name="example.md", file_type="md").parse_into_text(raw.encode())+ self.assertIn(raw.rstrip(), document.content)