Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): escape Word and PPT Markdown table cells (#2377)
kaiwangleo3b6915f4ac5a
diff --git a/src/agentscope/rag/_parser/_ppt.py b/src/agentscope/rag/_parser/_ppt.pyindex 52bc2b481..8d1011874 100644--- a/src/agentscope/rag/_parser/_ppt.py+++ b/src/agentscope/rag/_parser/_ppt.py@@ -45,7 +45,11 @@ def _extract_table_rows(table: Any) -> list[list[str]]:cells: list[str] = []for cell in row.cells:text = cell.text.strip()- text = text.replace("\r\n", "\n").replace("\r", "\n")+ text = (+ text.replace("\r\n", "\n")+ .replace("\r", "\n")+ .replace("\v", "\n")+ )cells.append(text)rows.append(cells)return rows@@ -133,9 +137,10 @@ class PPTParser(ParserBase):table_format (`Literal["markdown", "json"]`, defaults to``"markdown"``):How to render tables. ``"markdown"`` uses pipe-table- syntax; ``"json"`` emits a JSON array prefixed with a- ``<system-info>`` marker — choose JSON when cells- contain newlines that would corrupt Markdown layout.+ syntax, escaping pipes and rendering cell line breaks+ as ``<br>``; ``"json"`` emits a JSON array prefixed with+ a ``<system-info>`` marker and preserves extracted cell+ strings without Markdown rendering.slide_prefix (`str | None`, defaults to``"<slide index={index}>"``):Prepended to the first text section of each slide.diff --git a/src/agentscope/rag/_parser/_utils.py b/src/agentscope/rag/_parser/_utils.pyindex bff8f45f1..5172d8a61 100644--- a/src/agentscope/rag/_parser/_utils.py+++ b/src/agentscope/rag/_parser/_utils.py@@ -11,8 +11,8 @@ and :class:`PPTParser`:Markdown pipe-table; the default rendering for table content.- :func:`_table_to_json` — render the same shape as a JSON arrayprefixed with a one-line ``<system-info>`` marker; used when the- caller picks ``table_format="json"`` to avoid Markdown's- multi-line-cell ambiguity.+ caller picks ``table_format="json"`` to preserve the extracted cell+ strings instead of Markdown rendering."""import json@@ -47,9 +47,29 @@ def _guess_image_media_type(data: bytes) -> str:return "image/jpeg"+def _format_markdown_table_cell(cell: str) -> str:+ """Escape content for use in a Markdown table cell.++ Args:+ cell (`str`):+ The raw cell content.++ Returns:+ `str`:+ The escaped single-line Markdown cell content.+ """+ normalised = cell.replace("\r\n", "\n").replace("\r", "\n")+ escaped = normalised.replace("\\", "\\\\").replace("|", "\\|")+ return escaped.replace("\n", "<br>")++def _table_to_markdown(table_data: list[list[str]]) -> str:"""Render a 2-D table as a Markdown pipe-table.+ Pipe characters are escaped so they remain part of the cell, and+ line endings within a cell are normalised and rendered as ``<br>``+ tags so each table row stays on one physical Markdown line.+Args:table_data (`list[list[str]]`):The table data; ``table_data[0]`` is the header row.@@ -66,14 +86,18 @@ def _table_to_markdown(table_data: list[list[str]]) -> str:if num_cols == 0:return ""+ header = [_format_markdown_table_cell(cell) for cell in table_data[0]]lines = [- "| " + " | ".join(table_data[0]) + " |",+ "| " + " | ".join(header) + " |","| " + " | ".join(["---"] * num_cols) + " |",]for row in table_data[1:]:# Pad short rows so column counts match the header.padded = list(row) + [""] * max(0, num_cols - len(row))- lines.append("| " + " | ".join(padded[:num_cols]) + " |")+ cells = [+ _format_markdown_table_cell(cell) for cell in padded[:num_cols]+ ]+ lines.append("| " + " | ".join(cells) + " |")return "\n".join(lines) + "\n"diff --git a/src/agentscope/rag/_parser/_word.py b/src/agentscope/rag/_parser/_word.pyindex c230caff8..7287aef6c 100644--- a/src/agentscope/rag/_parser/_word.py+++ b/src/agentscope/rag/_parser/_word.py@@ -83,17 +83,21 @@ def _extract_table_data(table: DocxTable) -> list[list[str]]:"""from docx.oxml.ns import qn+ text_tag = qn("w:t")+ break_tags = {qn("w:br"), qn("w:cr")}table_data: list[list[str]] = []for tr in table._element.findall(qn("w:tr")):row_data: list[str] = []for tc in tr.findall(qn("w:tc")):paragraphs: list[str] = []for p_elem in tc.findall(qn("w:p")):- texts: list[str] = []- for t_elem in p_elem.findall(".//" + qn("w:t")):- if t_elem.text:- texts.append(t_elem.text)- para_text = "".join(texts)+ text_parts: list[str] = []+ for element in p_elem.iter():+ if element.tag == text_tag and element.text:+ text_parts.append(element.text)+ elif element.tag in break_tags:+ text_parts.append("\n")+ para_text = "".join(text_parts)if para_text:paragraphs.append(para_text)row_data.append("\n".join(paragraphs))@@ -211,8 +215,10 @@ class WordParser(ParserBase):table_format (`Literal["markdown", "json"]`, defaults to``"markdown"``):How to render tables. ``"markdown"`` uses pipe-table- syntax; ``"json"`` emits a JSON array prefixed with a- ``<system-info>`` marker.+ syntax, escaping pipes and rendering cell line breaks+ as ``<br>``; ``"json"`` emits a JSON array prefixed with+ a ``<system-info>`` marker and preserves extracted cell+ strings without Markdown rendering.Raises:`ValueError`: If ``table_format`` is not one ofdiff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.pyindex a2eb4d83b..f944c29ec 100644--- a/tests/rag_parser_test.py+++ b/tests/rag_parser_test.py@@ -111,6 +111,31 @@ def _make_pptx_rich() -> bytes:return buffer.getvalue()+def _make_pptx_with_special_table_cells() -> bytes:+ """Build a PPTX table with pipes and a multi-line cell."""+ from pptx import Presentation+ from pptx.util import Inches++ prs = Presentation()+ slide = prs.slides.add_slide(prs.slide_layouts[5])+ table = slide.shapes.add_table(+ rows=2,+ cols=2,+ left=Inches(1),+ top=Inches(1),+ width=Inches(4),+ height=Inches(1),+ ).table+ table.cell(0, 0).text = "A|B"+ table.cell(0, 1).text = r"Path \| label"+ table.cell(1, 0).text = "1|2"+ table.cell(1, 1).text = "Line 1\vLine 2"++ buffer = io.BytesIO()+ prs.save(buffer)+ return buffer.getvalue()++def _make_docx_simple(paragraphs: list[str]) -> bytes:"""Build a DOCX in memory with plain text paragraphs."""from docx import Document as DocxDocument@@ -144,6 +169,22 @@ def _make_docx_with_table() -> bytes:return buffer.getvalue()+def _make_docx_with_special_table_cells() -> bytes:+ """Build a DOCX table with pipes and a multi-line cell."""+ from docx import Document as DocxDocument++ doc = DocxDocument()+ table = doc.add_table(rows=2, cols=2)+ table.cell(0, 0).text = "A|B"+ table.cell(0, 1).text = r"Path \| label"+ table.cell(1, 0).text = "1|2"+ table.cell(1, 1).text = "Line 1\nLine 2"++ buffer = io.BytesIO()+ doc.save(buffer)+ return buffer.getvalue()++def _make_docx_with_image() -> bytes:"""Build a DOCX with a paragraph and an embedded PNG image."""from docx import Document as DocxDocument@@ -525,6 +566,44 @@ class PPTParserTest(IsolatedAsyncioTestCase):],)+ async def test_markdown_table_escapes_special_cells(self) -> None:+ """Pipes and line breaks do not corrupt Markdown table rows."""+ pptx_bytes = _make_pptx_with_special_table_cells()+ parser = PPTParser(+ include_image=False,+ separate_table=True,+ slide_prefix=None,+ slide_suffix=None,+ )+ sections = await parser.parse(pptx_bytes, "special.pptx")++ expected_text = (+ "\n".join(+ [+ r"| A\|B | Path \\\| label |",+ "| --- | --- |",+ r"| 1\|2 | Line 1<br>Line 2 |",+ ],+ )+ + "\n"+ )+ self.assertEqual(+ [section.model_dump() for section in sections],+ [+ {+ "content": {+ "type": "text",+ "text": expected_text,+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "special.pptx",+ "metadata": {"slide": 1},+ },+ ],+ )+async def test_table_separated_when_separate_table_true(self) -> None:"""``separate_table=True`` flushes the running text around thetable."""@@ -1121,6 +1200,39 @@ class WordParserTest(IsolatedAsyncioTestCase):],)+ async def test_markdown_table_escapes_special_cells(self) -> None:+ """Pipes and line breaks do not corrupt Markdown table rows."""+ docx_bytes = _make_docx_with_special_table_cells()+ parser = WordParser(include_image=False, separate_table=True)+ sections = await parser.parse(docx_bytes, "special.docx")++ expected_text = (+ "\n".join(+ [+ r"| A\|B | Path \\\| label |",+ "| --- | --- |",+ r"| 1\|2 | Line 1<br>Line 2 |",+ ],+ )+ + "\n"+ )+ self.assertEqual(+ [section.model_dump() for section in sections],+ [+ {+ "content": {+ "type": "text",+ "text": expected_text,+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": None,+ },+ "source": "special.docx",+ "metadata": {},+ },+ ],+ )+async def test_table_separated(self) -> None:"""``separate_table=True`` isolates the table from surroundingparagraphs."""