Skip to content

Build and run agents you can see, understand and trust.

fix(rag): preserve blank lines between paragraphs in WordParser (#2538)

liugy789c7d8e31bfd0a
diff --git a/src/agentscope/rag/_parser/_word.py b/src/agentscope/rag/_parser/_word.py
index 7287aef6c..de1f05e3c 100644
--- a/src/agentscope/rag/_parser/_word.py
+++ b/src/agentscope/rag/_parser/_word.py
@@ -280,6 +280,8 @@ class WordParser(ParserBase):
text_buffer: list[str] = []
def flush_text() -> None:
+ while text_buffer and text_buffer[-1] == "":
+ text_buffer.pop()
if not text_buffer:
return
sections.append(
@@ -298,6 +300,11 @@ class WordParser(ParserBase):
text = _extract_text_from_paragraph(para)
if text:
text_buffer.append(text)
+ elif text_buffer and not para._element.findall(
+ ".//" + qn("w:r"),
+ ):
+ # No runs at all, i.e. a blank line in Word
+ text_buffer.append("")
if self.include_image:
has_drawing = bool(
diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.py
index e1ddcb099..f5d8c7ae3 100644
--- a/tests/rag_parser_test.py
+++ b/tests/rag_parser_test.py
@@ -1210,6 +1210,78 @@ class WordParserTest(IsolatedAsyncioTestCase):
],
)
+ async def test_empty_paragraph_is_preserved_between_text(self) -> None:
+ """A blank paragraph between text paragraphs remains a blank line."""
+ docx_bytes = _make_docx_simple(
+ ["Paragraph one.", "", "Paragraph two."],
+ )
+ sections = await WordParser(include_image=False).parse(
+ docx_bytes,
+ "demo.docx",
+ )
+
+ self.assertEqual(
+ sections[0].content.text,
+ "Paragraph one.\n\nParagraph two.",
+ )
+
+ async def test_consecutive_empty_paragraphs_are_preserved(self) -> None:
+ """Consecutive blank paragraphs preserve each intervening line."""
+ docx_bytes = _make_docx_simple(
+ ["Paragraph one.", "", "", "Paragraph two."],
+ )
+ sections = await WordParser(include_image=False).parse(
+ docx_bytes,
+ "demo.docx",
+ )
+
+ self.assertEqual(
+ sections[0].content.text,
+ "Paragraph one.\n\n\nParagraph two.",
+ )
+
+ async def test_empty_paragraph_with_bookmark_is_preserved(self) -> None:
+ """Word often leaves bookmarks (e.g. ``_GoBack``) on blank
+ paragraphs, which are still blank lines."""
+ from docx import Document as DocxDocument
+ from docx.oxml import OxmlElement
+ from docx.oxml.ns import qn
+
+ doc = DocxDocument()
+ doc.add_paragraph("Paragraph one.")
+ blank = doc.add_paragraph("")
+ start = OxmlElement("w:bookmarkStart")
+ start.set(qn("w:id"), "0")
+ start.set(qn("w:name"), "_GoBack")
+ end = OxmlElement("w:bookmarkEnd")
+ end.set(qn("w:id"), "0")
+ blank._element.append(start) # pylint: disable=protected-access
+ blank._element.append(end) # pylint: disable=protected-access
+ doc.add_paragraph("Paragraph two.")
+ buffer = io.BytesIO()
+ doc.save(buffer)
+
+ sections = await WordParser(include_image=False).parse(
+ buffer.getvalue(),
+ "demo.docx",
+ )
+
+ self.assertEqual(
+ sections[0].content.text,
+ "Paragraph one.\n\nParagraph two.",
+ )
+
+ async def test_trailing_empty_paragraphs_do_not_add_newlines(self) -> None:
+ """Trailing blank paragraphs do not leave a trailing newline."""
+ docx_bytes = _make_docx_simple(["Paragraph one.", "", ""])
+ sections = await WordParser(include_image=False).parse(
+ docx_bytes,
+ "demo.docx",
+ )
+
+ self.assertEqual(len(sections), 1)
+ self.assertEqual(sections[0].content.text, "Paragraph one.")
+
async def test_table_merges_by_default(self) -> None:
"""``separate_table=False`` merges the table into surrounding
text."""