Skip to content

Build and run agents you can see, understand and trust.

fix(rag): escape Word and PPT Markdown table cells (#2377)

kaiwangleo3b6915f4ac5a
diff --git a/src/agentscope/rag/_parser/_ppt.py b/src/agentscope/rag/_parser/_ppt.py
index 52bc2b481..8d1011874 100644
--- a/src/agentscope/rag/_parser/_ppt.py
+++ b/src/agentscope/rag/_parser/_ppt.py
@@ -45,7 +45,11 @@ def _extract_table_rows(table: Any) -> list[list[str]]:
cells: list[str] = []
for cell in row.cells:
text = cell.text.strip()
- text = text.replace("\r\n", "\n").replace("\r", "\n")
+ text = (
+ text.replace("\r\n", "\n")
+ .replace("\r", "\n")
+ .replace("\v", "\n")
+ )
cells.append(text)
rows.append(cells)
return rows
@@ -133,9 +137,10 @@ class PPTParser(ParserBase):
table_format (`Literal["markdown", "json"]`, defaults to
``"markdown"``):
How to render tables. ``"markdown"`` uses pipe-table
- syntax; ``"json"`` emits a JSON array prefixed with a
- ``<system-info>`` marker — choose JSON when cells
- contain newlines that would corrupt Markdown layout.
+ syntax, escaping pipes and rendering cell line breaks
+ as ``<br>``; ``"json"`` emits a JSON array prefixed with
+ a ``<system-info>`` marker and preserves extracted cell
+ strings without Markdown rendering.
slide_prefix (`str | None`, defaults to
``"<slide index={index}>"``):
Prepended to the first text section of each slide.
diff --git a/src/agentscope/rag/_parser/_utils.py b/src/agentscope/rag/_parser/_utils.py
index bff8f45f1..5172d8a61 100644
--- a/src/agentscope/rag/_parser/_utils.py
+++ b/src/agentscope/rag/_parser/_utils.py
@@ -11,8 +11,8 @@ and :class:`PPTParser`:
Markdown pipe-table; the default rendering for table content.
- :func:`_table_to_json` — render the same shape as a JSON array
prefixed with a one-line ``<system-info>`` marker; used when the
- caller picks ``table_format="json"`` to avoid Markdown's
- multi-line-cell ambiguity.
+ caller picks ``table_format="json"`` to preserve the extracted cell
+ strings instead of Markdown rendering.
"""
import json
@@ -47,9 +47,29 @@ def _guess_image_media_type(data: bytes) -> str:
return "image/jpeg"
+def _format_markdown_table_cell(cell: str) -> str:
+ """Escape content for use in a Markdown table cell.
+
+ Args:
+ cell (`str`):
+ The raw cell content.
+
+ Returns:
+ `str`:
+ The escaped single-line Markdown cell content.
+ """
+ normalised = cell.replace("\r\n", "\n").replace("\r", "\n")
+ escaped = normalised.replace("\\", "\\\\").replace("|", "\\|")
+ return escaped.replace("\n", "<br>")
+
+
def _table_to_markdown(table_data: list[list[str]]) -> str:
"""Render a 2-D table as a Markdown pipe-table.
+ Pipe characters are escaped so they remain part of the cell, and
+ line endings within a cell are normalised and rendered as ``<br>``
+ tags so each table row stays on one physical Markdown line.
+
Args:
table_data (`list[list[str]]`):
The table data; ``table_data[0]`` is the header row.
@@ -66,14 +86,18 @@ def _table_to_markdown(table_data: list[list[str]]) -> str:
if num_cols == 0:
return ""
+ header = [_format_markdown_table_cell(cell) for cell in table_data[0]]
lines = [
- "| " + " | ".join(table_data[0]) + " |",
+ "| " + " | ".join(header) + " |",
"| " + " | ".join(["---"] * num_cols) + " |",
]
for row in table_data[1:]:
# Pad short rows so column counts match the header.
padded = list(row) + [""] * max(0, num_cols - len(row))
- lines.append("| " + " | ".join(padded[:num_cols]) + " |")
+ cells = [
+ _format_markdown_table_cell(cell) for cell in padded[:num_cols]
+ ]
+ lines.append("| " + " | ".join(cells) + " |")
return "\n".join(lines) + "\n"
diff --git a/src/agentscope/rag/_parser/_word.py b/src/agentscope/rag/_parser/_word.py
index c230caff8..7287aef6c 100644
--- a/src/agentscope/rag/_parser/_word.py
+++ b/src/agentscope/rag/_parser/_word.py
@@ -83,17 +83,21 @@ def _extract_table_data(table: DocxTable) -> list[list[str]]:
"""
from docx.oxml.ns import qn
+ text_tag = qn("w:t")
+ break_tags = {qn("w:br"), qn("w:cr")}
table_data: list[list[str]] = []
for tr in table._element.findall(qn("w:tr")):
row_data: list[str] = []
for tc in tr.findall(qn("w:tc")):
paragraphs: list[str] = []
for p_elem in tc.findall(qn("w:p")):
- texts: list[str] = []
- for t_elem in p_elem.findall(".//" + qn("w:t")):
- if t_elem.text:
- texts.append(t_elem.text)
- para_text = "".join(texts)
+ text_parts: list[str] = []
+ for element in p_elem.iter():
+ if element.tag == text_tag and element.text:
+ text_parts.append(element.text)
+ elif element.tag in break_tags:
+ text_parts.append("\n")
+ para_text = "".join(text_parts)
if para_text:
paragraphs.append(para_text)
row_data.append("\n".join(paragraphs))
@@ -211,8 +215,10 @@ class WordParser(ParserBase):
table_format (`Literal["markdown", "json"]`, defaults to
``"markdown"``):
How to render tables. ``"markdown"`` uses pipe-table
- syntax; ``"json"`` emits a JSON array prefixed with a
- ``<system-info>`` marker.
+ syntax, escaping pipes and rendering cell line breaks
+ as ``<br>``; ``"json"`` emits a JSON array prefixed with
+ a ``<system-info>`` marker and preserves extracted cell
+ strings without Markdown rendering.
Raises:
`ValueError`: If ``table_format`` is not one of
diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.py
index a2eb4d83b..f944c29ec 100644
--- a/tests/rag_parser_test.py
+++ b/tests/rag_parser_test.py
@@ -111,6 +111,31 @@ def _make_pptx_rich() -> bytes:
return buffer.getvalue()
+def _make_pptx_with_special_table_cells() -> bytes:
+ """Build a PPTX table with pipes and a multi-line cell."""
+ from pptx import Presentation
+ from pptx.util import Inches
+
+ prs = Presentation()
+ slide = prs.slides.add_slide(prs.slide_layouts[5])
+ table = slide.shapes.add_table(
+ rows=2,
+ cols=2,
+ left=Inches(1),
+ top=Inches(1),
+ width=Inches(4),
+ height=Inches(1),
+ ).table
+ table.cell(0, 0).text = "A|B"
+ table.cell(0, 1).text = r"Path \| label"
+ table.cell(1, 0).text = "1|2"
+ table.cell(1, 1).text = "Line 1\vLine 2"
+
+ buffer = io.BytesIO()
+ prs.save(buffer)
+ return buffer.getvalue()
+
+
def _make_docx_simple(paragraphs: list[str]) -> bytes:
"""Build a DOCX in memory with plain text paragraphs."""
from docx import Document as DocxDocument
@@ -144,6 +169,22 @@ def _make_docx_with_table() -> bytes:
return buffer.getvalue()
+def _make_docx_with_special_table_cells() -> bytes:
+ """Build a DOCX table with pipes and a multi-line cell."""
+ from docx import Document as DocxDocument
+
+ doc = DocxDocument()
+ table = doc.add_table(rows=2, cols=2)
+ table.cell(0, 0).text = "A|B"
+ table.cell(0, 1).text = r"Path \| label"
+ table.cell(1, 0).text = "1|2"
+ table.cell(1, 1).text = "Line 1\nLine 2"
+
+ buffer = io.BytesIO()
+ doc.save(buffer)
+ return buffer.getvalue()
+
+
def _make_docx_with_image() -> bytes:
"""Build a DOCX with a paragraph and an embedded PNG image."""
from docx import Document as DocxDocument
@@ -525,6 +566,44 @@ class PPTParserTest(IsolatedAsyncioTestCase):
],
)
+ async def test_markdown_table_escapes_special_cells(self) -> None:
+ """Pipes and line breaks do not corrupt Markdown table rows."""
+ pptx_bytes = _make_pptx_with_special_table_cells()
+ parser = PPTParser(
+ include_image=False,
+ separate_table=True,
+ slide_prefix=None,
+ slide_suffix=None,
+ )
+ sections = await parser.parse(pptx_bytes, "special.pptx")
+
+ expected_text = (
+ "\n".join(
+ [
+ r"| A\|B | Path \\\| label |",
+ "| --- | --- |",
+ r"| 1\|2 | Line 1<br>Line 2 |",
+ ],
+ )
+ + "\n"
+ )
+ self.assertEqual(
+ [section.model_dump() for section in sections],
+ [
+ {
+ "content": {
+ "type": "text",
+ "text": expected_text,
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "special.pptx",
+ "metadata": {"slide": 1},
+ },
+ ],
+ )
+
async def test_table_separated_when_separate_table_true(self) -> None:
"""``separate_table=True`` flushes the running text around the
table."""
@@ -1121,6 +1200,39 @@ class WordParserTest(IsolatedAsyncioTestCase):
],
)
+ async def test_markdown_table_escapes_special_cells(self) -> None:
+ """Pipes and line breaks do not corrupt Markdown table rows."""
+ docx_bytes = _make_docx_with_special_table_cells()
+ parser = WordParser(include_image=False, separate_table=True)
+ sections = await parser.parse(docx_bytes, "special.docx")
+
+ expected_text = (
+ "\n".join(
+ [
+ r"| A\|B | Path \\\| label |",
+ "| --- | --- |",
+ r"| 1\|2 | Line 1<br>Line 2 |",
+ ],
+ )
+ + "\n"
+ )
+ self.assertEqual(
+ [section.model_dump() for section in sections],
+ [
+ {
+ "content": {
+ "type": "text",
+ "text": expected_text,
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ "source": "special.docx",
+ "metadata": {},
+ },
+ ],
+ )
+
async def test_table_separated(self) -> None:
"""``separate_table=True`` isolates the table from surrounding
paragraphs."""