Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(rag): preserve blank lines between paragraphs in WordParser (#2538)
liugy789c7d8e31bfd0a
diff --git a/src/agentscope/rag/_parser/_word.py b/src/agentscope/rag/_parser/_word.pyindex 7287aef6c..de1f05e3c 100644--- a/src/agentscope/rag/_parser/_word.py+++ b/src/agentscope/rag/_parser/_word.py@@ -280,6 +280,8 @@ class WordParser(ParserBase):text_buffer: list[str] = []def flush_text() -> None:+ while text_buffer and text_buffer[-1] == "":+ text_buffer.pop()if not text_buffer:returnsections.append(@@ -298,6 +300,11 @@ class WordParser(ParserBase):text = _extract_text_from_paragraph(para)if text:text_buffer.append(text)+ elif text_buffer and not para._element.findall(+ ".//" + qn("w:r"),+ ):+ # No runs at all, i.e. a blank line in Word+ text_buffer.append("")if self.include_image:has_drawing = bool(diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.pyindex e1ddcb099..f5d8c7ae3 100644--- a/tests/rag_parser_test.py+++ b/tests/rag_parser_test.py@@ -1210,6 +1210,78 @@ class WordParserTest(IsolatedAsyncioTestCase):],)+ async def test_empty_paragraph_is_preserved_between_text(self) -> None:+ """A blank paragraph between text paragraphs remains a blank line."""+ docx_bytes = _make_docx_simple(+ ["Paragraph one.", "", "Paragraph two."],+ )+ sections = await WordParser(include_image=False).parse(+ docx_bytes,+ "demo.docx",+ )++ self.assertEqual(+ sections[0].content.text,+ "Paragraph one.\n\nParagraph two.",+ )++ async def test_consecutive_empty_paragraphs_are_preserved(self) -> None:+ """Consecutive blank paragraphs preserve each intervening line."""+ docx_bytes = _make_docx_simple(+ ["Paragraph one.", "", "", "Paragraph two."],+ )+ sections = await WordParser(include_image=False).parse(+ docx_bytes,+ "demo.docx",+ )++ self.assertEqual(+ sections[0].content.text,+ "Paragraph one.\n\n\nParagraph two.",+ )++ async def test_empty_paragraph_with_bookmark_is_preserved(self) -> None:+ """Word often leaves bookmarks (e.g. ``_GoBack``) on blank+ paragraphs, which are still blank lines."""+ from docx import Document as DocxDocument+ from docx.oxml import OxmlElement+ from docx.oxml.ns import qn++ doc = DocxDocument()+ doc.add_paragraph("Paragraph one.")+ blank = doc.add_paragraph("")+ start = OxmlElement("w:bookmarkStart")+ start.set(qn("w:id"), "0")+ start.set(qn("w:name"), "_GoBack")+ end = OxmlElement("w:bookmarkEnd")+ end.set(qn("w:id"), "0")+ blank._element.append(start) # pylint: disable=protected-access+ blank._element.append(end) # pylint: disable=protected-access+ doc.add_paragraph("Paragraph two.")+ buffer = io.BytesIO()+ doc.save(buffer)++ sections = await WordParser(include_image=False).parse(+ buffer.getvalue(),+ "demo.docx",+ )++ self.assertEqual(+ sections[0].content.text,+ "Paragraph one.\n\nParagraph two.",+ )++ async def test_trailing_empty_paragraphs_do_not_add_newlines(self) -> None:+ """Trailing blank paragraphs do not leave a trailing newline."""+ docx_bytes = _make_docx_simple(["Paragraph one.", "", ""])+ sections = await WordParser(include_image=False).parse(+ docx_bytes,+ "demo.docx",+ )++ self.assertEqual(len(sections), 1)+ self.assertEqual(sections[0].content.text, "Paragraph one.")+async def test_table_merges_by_default(self) -> None:"""``separate_table=False`` merges the table into surroundingtext."""