Skip to content

Build and run agents you can see, understand and trust.

fix(rag): keep blank Word paragraphs that still carry a run (#3401)

Starchaser1024ce04210e2c1d
diff --git a/src/agentscope/rag/_parser/_word.py b/src/agentscope/rag/_parser/_word.py
index 4fe63b3e4..4222d6e31 100644
--- a/src/agentscope/rag/_parser/_word.py
+++ b/src/agentscope/rag/_parser/_word.py
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
from docx.text.paragraph import Paragraph as DocxParagraph
_VML_NS = "{urn:schemas-microsoft-com:vml}"
+_MATH_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/math}"
_MC_ALTERNATE_CONTENT = (
"{http://schemas.openxmlformats.org/markup-compatibility/2006}"
"AlternateContent"
@@ -105,6 +106,46 @@ def _extract_text_from_paragraph(para: DocxParagraph) -> str:
return "\n".join(block.strip() for block in blocks if block.strip())
+def _is_blank_paragraph(para: DocxParagraph) -> bool:
+ """Return whether Word renders *para* as a blank line.
+
+ A blank line has no text and no picture. It may still hold a run:
+ typing a space before Enter leaves ``<w:t> </w:t>``, and Shift+Enter
+ on an empty line leaves a run holding only ``<w:br/>``. A run that
+ carries anything else (field instructions, a symbol, an equation) is
+ not blank even when the extractor finds no text in it.
+ """
+ from docx.oxml.ns import qn
+
+ root = para._element
+ for tag in ("w:drawing", "w:pict", "w:object"):
+ if root.findall(".//" + qn(tag)):
+ return False
+ if root.findall(".//" + _MATH_NS + "oMath"):
+ return False
+
+ blank_run_tags = {
+ qn(tag)
+ for tag in (
+ "w:rPr",
+ "w:br",
+ "w:cr",
+ "w:tab",
+ "w:lastRenderedPageBreak",
+ "w:softHyphen",
+ )
+ }
+ text_tag = qn("w:t")
+ for run in root.findall(".//" + qn("w:r")):
+ for child in run:
+ if child.tag == text_tag:
+ if (child.text or "").strip():
+ return False
+ elif child.tag not in blank_run_tags:
+ return False
+ return True
+
+
def _extract_table_data(table: DocxTable) -> list[list[str]]:
"""Extract table data from a python-docx Table, preserving line breaks
within cells.
@@ -378,10 +419,9 @@ class WordParser(ParserBase):
text = _extract_text_from_paragraph(para)
if text:
text_buffer.append(text)
- elif text_buffer and not para._element.findall(
- ".//" + qn("w:r"),
- ):
- # No runs at all, i.e. a blank line in Word
+ elif text_buffer and _is_blank_paragraph(para):
+ # A blank line in Word: an empty <w:p>, a run holding
+ # only spaces, or a run holding only <w:br/>/<w:cr/>.
text_buffer.append("")
if self.include_image:
diff --git a/tests/rag_parser_test.py b/tests/rag_parser_test.py
index 53ea9c6f6..ec557f186 100644
--- a/tests/rag_parser_test.py
+++ b/tests/rag_parser_test.py
@@ -2878,6 +2878,59 @@ class WordParserTest(IsolatedAsyncioTestCase):
class WordParserParagraphTest(IsolatedAsyncioTestCase):
"""Regression coverage for paragraph and text-box extraction."""
+ async def test_blank_paragraph_with_a_run_is_preserved(self) -> None:
+ """A blank-looking paragraph whose only run holds a space or a line
+ break is still a blank line."""
+ from docx import Document as DocxDocument
+
+ doc = DocxDocument()
+ doc.add_paragraph("Paragraph one.")
+ doc.add_paragraph(" ") # a run that holds only a space
+ break_only = doc.add_paragraph() # a run that holds only <w:br/>
+ break_only.add_run().add_break()
+ doc.add_paragraph("Paragraph two.")
+ buffer = io.BytesIO()
+ doc.save(buffer)
+
+ sections = await WordParser(include_image=False).parse(
+ buffer.getvalue(),
+ "demo.docx",
+ )
+
+ self.assertEqual(
+ sections[0].content.text,
+ "Paragraph one.\n\n\nParagraph two.",
+ )
+
+ async def test_paragraph_with_a_field_code_is_not_a_blank_line(
+ self,
+ ) -> None:
+ """A paragraph holding field instructions is not a blank line, even
+ though the extractor returns no text for it."""
+ from docx import Document as DocxDocument
+ from docx.oxml import OxmlElement
+
+ doc = DocxDocument()
+ doc.add_paragraph("Paragraph one.")
+ field_run = doc.add_paragraph().add_run()
+ instr = OxmlElement("w:instrText")
+ instr.text = " PAGE "
+ # pylint: disable-next=protected-access
+ field_run._element.append(instr)
+ doc.add_paragraph("Paragraph two.")
+ buffer = io.BytesIO()
+ doc.save(buffer)
+
+ sections = await WordParser(include_image=False).parse(
+ buffer.getvalue(),
+ "demo.docx",
+ )
+
+ self.assertEqual(
+ sections[0].content.text,
+ "Paragraph one.\nParagraph two.",
+ )
+
async def test_text_boxes_are_separate_and_read_once(self) -> None:
"""Read modern text boxes once, with or without a VML fallback."""
for name, template in (