Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): keep a pipe inside an HTML table cell in that cell (#4433)
Lukas578ca1b5193b
diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.pyindex 937f3c220..bbf6baff3 100644--- a/docreader/parser/markdown_parser.py+++ b/docreader/parser/markdown_parser.py@@ -27,6 +27,9 @@ from docreader.utils import endecodelogger = logging.getLogger(__name__)_SEPARATOR_CELL = re.compile(r"^:?-{3,}:?$")+# A backslash escape, or a bare pipe. Scanning both keeps an escaped pipe+# (``\|``) out of the cell delimiters without a lookbehind, in linear time.+_ESCAPE_OR_PIPE = re.compile(r"\\.|\|", re.DOTALL)# Cells of a GFM alignment row as matched by the previous ``align_pattern``.# Looser than ``_SEPARATOR_CELL``: any non-empty run of ``:``/``-`` qualifies._ALIGNMENT_CELL = re.compile(r"^[:-]+$")@@ -53,11 +56,21 @@ class MarkdownTableUtil:@staticmethoddef _split_row_cells(row_line: str) -> List[str]:- """Split a markdown table row into cells, preserving empty cells."""+ """Split a markdown table row into cells, preserving empty cells.++ As in GFM, a pipe escaped as ``\\|`` is text inside its cell rather+ than a cell delimiter.+ """inner = row_line.strip()if not inner.startswith("|"):return []- parts = inner.split("|")+ parts: List[str] = []+ start = 0+ for match in _ESCAPE_OR_PIPE.finditer(inner):+ if match.group() == "|":+ parts.append(inner[start:match.start()])+ start = match.end()+ parts.append(inner[start:])if parts and parts[0].strip() == "":parts = parts[1:]if parts and parts[-1].strip() == "":diff --git a/docreader/parser/mhtml_parser.py b/docreader/parser/mhtml_parser.pyindex d79ecdd41..113493be0 100644--- a/docreader/parser/mhtml_parser.py+++ b/docreader/parser/mhtml_parser.py@@ -13,7 +13,7 @@ from urllib.parse import unquote, urljoin, urlparseimport uuidfrom typing import Dict-from bs4 import BeautifulSoup+from bs4 import BeautifulSoup, NavigableStringfrom docreader.models.document import Documentfrom docreader.parser.base_parser import BaseParser@@ -289,6 +289,7 @@ class MHTMLParser(BaseParser):self._rewrite_image_sources(soup, image_aliases, base_location)self._unwrap_whitespace_only_inline(soup)text_fallback = soup.get_text(separator="\n", strip=True)+ self._escape_table_cell_pipes(soup)markdown_text = md(str(soup), heading_style="ATX")result = self._normalize_markdown(markdown_text)if not result and text_fallback:@@ -326,6 +327,20 @@ class MHTMLParser(BaseParser):if tag.find(True) is None and not tag.get_text().strip():tag.unwrap()+ @staticmethod+ def _escape_table_cell_pipes(soup: BeautifulSoup) -> None:+ """Escape the ``|`` in table cell text, which markdownify writes as is.++ In the GFM table markdownify writes, a bare pipe in a cell's text ends+ the cell, so the row gains a column and every later cell shifts one+ column right.+ """+ for cell in soup.find_all(["td", "th"]):+ for text in cell.find_all(string=True):+ # Comments and other special strings keep their own type+ if type(text) is NavigableString and "|" in text:+ text.replace_with(text.replace("|", "\\|"))+@staticmethoddef _normalize_markdown(markdown_text: str) -> str:text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")diff --git a/docreader/tests/test_table_cell_pipes.py b/docreader/tests/test_table_cell_pipes.pynew file mode 100644index 000000000..55154bf22--- /dev/null+++ b/docreader/tests/test_table_cell_pipes.py@@ -0,0 +1,64 @@+"""A pipe inside a table cell stays in that cell; GFM escapes it as ``\\|``."""++import unittest++from docreader.parser.html_parser import HTMLParser+from docreader.parser.markdown_parser import MarkdownParser, MarkdownTableUtil+++class TableCellPipeTest(unittest.TestCase):+ def test_format_table_keeps_an_escaped_pipe_in_its_cell(self):+ raw = "|Pattern|Matches|\n|---|---|\n|`a\\|b`|a or b|\n|x \\| y|literal|"++ formatted = MarkdownTableUtil().format_table(raw)++ self.assertEqual(+ formatted,+ "| Pattern | Matches |\n| --- | --- |\n"+ "| `a\\|b` | a or b |\n| x \\| y | literal |",+ )++ def test_format_table_ends_a_cell_at_a_pipe_after_an_escaped_backslash(self):+ # "\\\\" is an escaped backslash, so the pipe after it is a delimiter+ formatted = MarkdownTableUtil().format_table("|a\\\\|b|\n|---|---|")++ self.assertEqual(formatted, "| a\\\\ | b |\n| --- | --- |")++ def test_markdown_upload_keeps_an_escaped_pipe(self):+ content = "| Type | Notes |\n| --- | --- |\n| int \\| None | optional |\n"++ document = MarkdownParser(+ file_name="types.md", file_type="md"+ ).parse_into_text(content.encode())++ self.assertIn("| int \\| None | optional |", document.content)++ def test_html_upload_escapes_a_pipe_in_a_cell(self):+ html = (+ "<table><tr><th>Expr</th><th>Means</th></tr>"+ "<tr><td>a|b</td><td>run <code>ls|wc</code></td></tr></table>"+ )++ document = HTMLParser(+ file_name="pipes.html", file_type="html"+ ).parse_into_text(html.encode())++ self.assertIn("| Expr | Means |", document.content)+ self.assertIn("| a\\|b | run `ls\\|wc` |", document.content)++ def test_html_comment_in_a_cell_stays_hidden(self):+ html = (+ "<table><tr><th>Name</th></tr>"+ "<tr><td>pen<!-- sku|42 --></td></tr></table>"+ )++ document = HTMLParser(+ file_name="comment.html", file_type="html"+ ).parse_into_text(html.encode())++ self.assertIn("| pen |", document.content)+ self.assertNotIn("sku", document.content)+++if __name__ == "__main__":+ unittest.main()