Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): keep DOCX table content in parsed text (#2898)

LuckVd81142dfd17b2
diff --git a/docreader/parser/docx_parser.py b/docreader/parser/docx_parser.py
index bd713f805..f9820069f 100644
--- a/docreader/parser/docx_parser.py
+++ b/docreader/parser/docx_parser.py
@@ -39,6 +39,7 @@ from docx.image.exceptions import (
UnexpectedEndOfFileError,
UnrecognizedImageError,
)
+from docx.oxml.ns import qn
from PIL import Image
from docreader.config import CONFIG
@@ -48,6 +49,41 @@ from docreader.utils import endecode
logger = logging.getLogger(__name__)
+_W_P = qn("w:p")
+_W_TBL = qn("w:tbl")
+
+
+def _gfm_cell_text(text: Optional[str]) -> str:
+ """Collapse whitespace and escape pipes so a cell stays one GFM column."""
+ return re.sub(r"\s+", " ", text or "").strip().replace("|", "\\|")
+
+
+def table_to_gfm_markdown(table: Any) -> str:
+ """Render a python-docx Table as GFM Markdown.
+
+ ``row.cells`` already expands merged regions to one entry per grid slot
+ (repeating the same Cell object). Emitting every slot keeps column
+ alignment; comparing cell text would collapse adjacent equal/empty cells.
+ """
+ rows: List[List[str]] = []
+ for row in table.rows:
+ cells = [_gfm_cell_text(cell.text) for cell in row.cells]
+ if cells:
+ rows.append(cells)
+ if not rows:
+ return ""
+ width = max(len(row) for row in rows)
+ if width == 0:
+ return ""
+ padded = [row + [""] * (width - len(row)) for row in rows]
+ lines = [
+ "| " + " | ".join(padded[0]) + " |",
+ "| " + " | ".join("---" for _ in padded[0]) + " |",
+ ]
+ for row in padded[1:]:
+ lines.append("| " + " | ".join(row) + " |")
+ return "\n".join(lines)
+
class ImageData:
"""Represents a processed image of document content"""
@@ -142,7 +178,7 @@ class DocxParser(BaseParser):
enable_multimodal=True,
upload_file=_inline_upload,
)
- all_lines, tables = docx_processor(
+ all_lines, ordered_text = docx_processor(
binary=content,
max_workers=max_workers,
to_page=self.max_pages,
@@ -150,7 +186,7 @@ class DocxParser(BaseParser):
processing_time = time.time() - start_time
logger.info(
f"Docx processing completed in {processing_time:.2f}s, "
- f"extracted {len(all_lines)} sections and {len(tables)} tables"
+ f"extracted {len(all_lines)} sections"
)
logger.info("Processing document sections")
@@ -187,7 +223,11 @@ class DocxParser(BaseParser):
f"Section processing completed in {section_processing_time:.2f}s"
)
logger.info("Combining all text parts")
- text = "\n\n".join([part for part in text_parts if part])
+ # Prefer body-order markdown (paragraphs interleaved with tables).
+ # Fall back to page-section text if composition produced nothing.
+ text = (ordered_text or "").strip()
+ if not text:
+ text = "\n\n".join([part for part in text_parts if part])
# Check if the generated text is empty
if not text:
@@ -481,7 +521,7 @@ class Docx:
from_page: int = 0,
to_page: int = 100000,
max_workers: Optional[int] = None,
- ) -> Tuple[List[LineData], List[Any]]:
+ ) -> Tuple[List[LineData], str]:
"""
Process DOCX document, supporting concurrent processing of each page
@@ -492,7 +532,8 @@ class Docx:
max_workers: Maximum number of workers, default to None (system decides)
Returns:
- tuple: (List of LineData objects with document content, List of tables)
+ tuple: (List of LineData objects with document content,
+ body-order markdown with tables interleaved)
"""
logger.info("Processing DOCX document")
@@ -503,7 +544,7 @@ class Docx:
# Load document
self.doc = self._load_document(binary)
if not self.doc:
- return [], []
+ return [], ""
# Identify page structure
self.para_page_mapping = self._identify_page_paragraph_mapping(to_page)
@@ -517,7 +558,7 @@ class Docx:
)
if not pages_to_process:
logger.warning("No pages to process after applying page limits!")
- return [], []
+ return [], ""
# Initialize shared resources
self._init_shared_resources()
@@ -531,17 +572,17 @@ class Docx:
max_workers,
)
- # Process tables
- tbls = self._process_tables()
+ ordered_text = self._compose_ordered_markdown(pages_to_process)
# Clean up document resources
self.doc = None
logger.info(
f"Document processing complete, "
- f"extracted {len(self.all_lines)} text sections and {len(tbls)} tables"
+ f"extracted {len(self.all_lines)} text sections, "
+ f"{len(ordered_text)} characters of body-order markdown"
)
- return self.all_lines, tbls
+ return self.all_lines, ordered_text
def _load_document(self, binary):
"""Load document
@@ -1027,60 +1068,52 @@ class Docx:
except Exception as e:
logger.error(f"Failed to remove temporary file: {str(e)}")
- def _process_tables(self):
- """Process tables in the document
+ def _compose_ordered_markdown(self, pages_to_process: List[int]) -> str:
+ """Walk body paragraphs and tables in document order and emit GFM.
- Returns:
- list: List of tables
+ Paragraph processing is page-limited; tables are included only when they
+ sit within that same paragraph range so max_pages does not leak later
+ tables into the parsed text.
"""
- tbls = []
- table_count = len(self.doc.tables)
- if table_count > 0:
- logger.info(f"Processing {table_count} tables")
- for tb_idx, tb in enumerate(self.doc.tables):
- if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume
- logger.info(f"Processing table {tb_idx + 1}/{table_count}")
-
- # Optimize: Check if table is empty
- if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows):
- logger.info(f"Skipping empty table {tb_idx + 1}")
- continue
-
- table_html = self._convert_table_to_html(tb)
- # Still using tuple format for tables as they are handled differently
- tbls.append(((None, table_html), ""))
-
- return tbls
-
- def _convert_table_to_html(self, table):
- """Convert table to HTML
-
- Args:
- table: Table object
-
- Returns:
- str: HTML formatted table
- """
- html = "<table>"
- for r in table.rows:
- html += "<tr>"
- i = 0
- while i < len(r.cells):
- span = 1
- c = r.cells[i]
- for j in range(i + 1, len(r.cells)):
- if c.text == r.cells[j].text:
- span += 1
- i = j
- i += 1
- html += (
- f"<td>{c.text}</td>"
- if span == 1
- else f"<td colspan='{span}'>{c.text}</td>"
- )
- html += "</tr>"
- html += "</table>"
- return html
+ allowed = set()
+ for page in pages_to_process:
+ allowed.update(self.para_page_mapping.get(page, []))
+ min_allowed = min(allowed) if allowed else 0
+ max_allowed = max(allowed) if allowed else -1
+
+ parts: List[str] = []
+ para_i = 0
+ tbl_i = 0
+ paragraphs = self.doc.paragraphs
+ tables = self.doc.tables
+
+ for child in self.doc.element.body:
+ if child.tag == _W_P:
+ if para_i in allowed and para_i < len(paragraphs):
+ text = (paragraphs[para_i].text or "").strip()
+ if text:
+ parts.append(text)
+ para_i += 1
+ continue
+ if child.tag != _W_TBL:
+ continue
+ include_table = False
+ if not allowed:
+ include_table = True
+ elif min_allowed <= para_i <= max_allowed + 1:
+ include_table = True
+ if include_table and tbl_i < len(tables):
+ table = tables[tbl_i]
+ if not (
+ len(table.rows) == 0
+ or all(len(row.cells) == 0 for row in table.rows)
+ ):
+ markdown = table_to_gfm_markdown(table)
+ if markdown:
+ parts.append(markdown)
+ tbl_i += 1
+
+ return "\n\n".join(parts)
def _safe_concat_images(self, images):
"""Safely concatenate image lists
diff --git a/docreader/tests/test_docx_tables.py b/docreader/tests/test_docx_tables.py
new file mode 100644
index 000000000..b3338d4fe
--- /dev/null
+++ b/docreader/tests/test_docx_tables.py
@@ -0,0 +1,213 @@
+import importlib.util
+import io
+import sys
+import types
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from unittest.mock import patch
+
+from docx import Document as WordDocument
+
+
+def _load_docx_parser():
+ """Load docx_parser without triggering the heavy package __init__.
+
+ ``docreader/parser/__init__.py`` imports doc_parser -> textract, a heavy
+ dependency unrelated to DOCX parsing. Registering the packages as bare
+ namespaces lets us import only the modules docx_parser actually needs.
+ """
+ root = Path(__file__).resolve().parents[2]
+ docreader_pkg = types.ModuleType("docreader")
+ docreader_pkg.__path__ = [str(root / "docreader")]
+ sys.modules.setdefault("docreader", docreader_pkg)
+ parser_pkg = types.ModuleType("docreader.parser")
+ parser_pkg.__path__ = [str(root / "docreader" / "parser")]
+ sys.modules["docreader.parser"] = parser_pkg
+
+ spec = importlib.util.spec_from_file_location(
+ "docreader.parser.docx_parser", root / "docreader" / "parser" / "docx_parser.py"
+ )
+ module = importlib.util.module_from_spec(spec)
+ sys.modules["docreader.parser.docx_parser"] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+docx_parser = _load_docx_parser()
+DocxParser = docx_parser.DocxParser
+table_to_gfm_markdown = docx_parser.table_to_gfm_markdown
+
+
+def _parse(content):
+ """Parse a DOCX through the real Docx processor.
+
+ The production path uses a ProcessPoolExecutor backed by a multiprocessing
+ Manager. Some CI sandboxes forbid POSIX semaphores, so run the same page
+ task pool on threads and swap the manager for a plain list - parse
+ behavior is identical, only the execution backend changes.
+ """
+
+ class _FakeManager:
+ def __enter__(self):
+ self._items = []
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def list(self):
+ return self._items
+
+ with patch.object(docx_parser, "Manager", _FakeManager), patch.object(
+ docx_parser, "ProcessPoolExecutor", ThreadPoolExecutor
+ ):
+ return DocxParser(max_pages=100).parse_into_text(content)
+
+
+def _docx_bytes(build):
+ doc = WordDocument()
+ build(doc)
+ buf = io.BytesIO()
+ doc.save(buf)
+ return buf.getvalue()
+
+
+class DocxTableContentTest(unittest.TestCase):
+ """Regression test: DOCX tables must be kept in the parsed text.
+
+ Docx.__call__ used to return tables separately from the text lines, and
+ parse_into_text dropped them, silently losing all table content.
+ """
+
+ def _docx_with_table(self, cells):
+ def build(doc):
+ doc.add_paragraph("Introduction paragraph")
+ table = doc.add_table(rows=len(cells), cols=len(cells[0]))
+ for r, row in enumerate(cells):
+ for c, value in enumerate(row):
+ table.cell(r, c).text = value
+
+ return _docx_bytes(build)
+
+ def test_table_content_is_kept_in_parsed_text(self):
+ content = self._docx_with_table(
+ [["City", "Population"], ["Beijing", "21.5M"]]
+ )
+ document = _parse(content)
+ self.assertIn("Introduction paragraph", document.content)
+ for cell in ("City", "Population", "Beijing", "21.5M"):
+ self.assertIn(cell, document.content)
+ self.assertIn("| City | Population |", document.content)
+ self.assertIn("| --- | --- |", document.content)
+
+ def test_table_only_document_is_not_empty(self):
+ def build(doc):
+ table = doc.add_table(rows=2, cols=1)
+ table.cell(0, 0).text = "Header"
+ table.cell(1, 0).text = "Value"
+
+ document = _parse(_docx_bytes(build))
+ self.assertIn("Header", document.content)
+ self.assertIn("Value", document.content)
+
+ def test_pipe_in_cell_does_not_break_table(self):
+ def build(doc):
+ table = doc.add_table(rows=2, cols=1)
+ table.cell(0, 0).text = "A|B"
+ table.cell(1, 0).text = "C"
+
+ document = _parse(_docx_bytes(build))
+ self.assertIn(r"A\|B", document.content)
+
+ def test_empty_adjacent_cells_keep_columns(self):
+ def build(doc):
+ table = doc.add_table(rows=2, cols=3)
+ table.cell(0, 0).text = "A"
+ table.cell(0, 1).text = "B"
+ table.cell(0, 2).text = "C"
+ table.cell(1, 0).text = "1"
+ table.cell(1, 1).text = ""
+ table.cell(1, 2).text = ""
+
+ document = _parse(_docx_bytes(build))
+ self.assertIn("| A | B | C |", document.content)
+ self.assertIn("| 1 | | |", document.content)
+
+ def test_equal_adjacent_cells_are_not_collapsed(self):
+ """Adjacent independent cells with the same text must stay separate.
+
+ Regression for the #2634 control row: ``相同值 | 相同值 | 独立值``.
+ """
+
+ def build(doc):
+ table = doc.add_table(rows=2, cols=3)
+ table.cell(0, 0).text = "相同值"
+ table.cell(0, 1).text = "相同值"
+ table.cell(0, 2).text = "独立值"
+ table.cell(1, 0).text = "x"
+ table.cell(1, 1).text = "y"
+ table.cell(1, 2).text = "z"
+
+ document = _parse(_docx_bytes(build))
+ self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content)
+ self.assertIn("| x | y | z |", document.content)
+
+ def test_horizontal_merge_keeps_grid_width(self):
+ def build(doc):
+ table = doc.add_table(rows=2, cols=3)
+ table.cell(0, 0).merge(table.cell(0, 1)).text = "merged"
+ table.cell(0, 2).text = "right"
+ table.cell(1, 0).text = "a"
+ table.cell(1, 1).text = "b"
+ table.cell(1, 2).text = "c"
+
+ document = _parse(_docx_bytes(build))
+ self.assertIn("| merged | merged | right |", document.content)
+ self.assertIn("| a | b | c |", document.content)
+
+ def test_table_stays_between_paragraphs(self):
+ def build(doc):
+ doc.add_paragraph("before table")
+ table = doc.add_table(rows=2, cols=1)
+ table.cell(0, 0).text = "H"
+ table.cell(1, 0).text = "V"
+ doc.add_paragraph("after table")
+
+ document = _parse(_docx_bytes(build))
+ before = document.content.index("before table")
+ header = document.content.index("| H |")
+ after = document.content.index("after table")
+ self.assertLess(before, header)
+ self.assertLess(header, after)
+
+ def test_html_like_cell_text_is_literal(self):
+ def build(doc):
+ table = doc.add_table(rows=1, cols=2)
+ table.cell(0, 0).text = "a < b"
+ table.cell(0, 1).text = "</td><td>injected"
+
+ document = _parse(_docx_bytes(build))
+ self.assertIn("| a < b | </td><td>injected |", document.content)
+
+ def test_table_to_gfm_markdown_helper(self):
+ doc = WordDocument()
+ table = doc.add_table(rows=2, cols=2)
+ table.cell(0, 0).text = "A"
+ table.cell(0, 1).text = "B"
+ table.cell(1, 0).text = "C"
+ table.cell(1, 1).text = "D"
+ self.assertEqual(
+ table_to_gfm_markdown(table),
+ "| A | B |\n| --- | --- |\n| C | D |",
+ )
+
+ def test_table_to_gfm_markdown_skips_empty(self):
+ class _Empty:
+ rows = []
+
+ self.assertEqual(table_to_gfm_markdown(_Empty()), "")
+
+
+if __name__ == "__main__":
+ unittest.main()