Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): keep DOCX table content in parsed text (#2898)
LuckVd81142dfd17b2
diff --git a/docreader/parser/docx_parser.py b/docreader/parser/docx_parser.pyindex bd713f805..f9820069f 100644--- a/docreader/parser/docx_parser.py+++ b/docreader/parser/docx_parser.py@@ -39,6 +39,7 @@ from docx.image.exceptions import (UnexpectedEndOfFileError,UnrecognizedImageError,)+from docx.oxml.ns import qnfrom PIL import Imagefrom docreader.config import CONFIG@@ -48,6 +49,41 @@ from docreader.utils import endecodelogger = logging.getLogger(__name__)+_W_P = qn("w:p")+_W_TBL = qn("w:tbl")+++def _gfm_cell_text(text: Optional[str]) -> str:+ """Collapse whitespace and escape pipes so a cell stays one GFM column."""+ return re.sub(r"\s+", " ", text or "").strip().replace("|", "\\|")+++def table_to_gfm_markdown(table: Any) -> str:+ """Render a python-docx Table as GFM Markdown.++ ``row.cells`` already expands merged regions to one entry per grid slot+ (repeating the same Cell object). Emitting every slot keeps column+ alignment; comparing cell text would collapse adjacent equal/empty cells.+ """+ rows: List[List[str]] = []+ for row in table.rows:+ cells = [_gfm_cell_text(cell.text) for cell in row.cells]+ if cells:+ rows.append(cells)+ if not rows:+ return ""+ width = max(len(row) for row in rows)+ if width == 0:+ return ""+ padded = [row + [""] * (width - len(row)) for row in rows]+ lines = [+ "| " + " | ".join(padded[0]) + " |",+ "| " + " | ".join("---" for _ in padded[0]) + " |",+ ]+ for row in padded[1:]:+ lines.append("| " + " | ".join(row) + " |")+ return "\n".join(lines)+class ImageData:"""Represents a processed image of document content"""@@ -142,7 +178,7 @@ class DocxParser(BaseParser):enable_multimodal=True,upload_file=_inline_upload,)- all_lines, tables = docx_processor(+ all_lines, ordered_text = docx_processor(binary=content,max_workers=max_workers,to_page=self.max_pages,@@ -150,7 +186,7 @@ class DocxParser(BaseParser):processing_time = time.time() - start_timelogger.info(f"Docx processing completed in {processing_time:.2f}s, "- f"extracted {len(all_lines)} sections and {len(tables)} tables"+ f"extracted {len(all_lines)} sections")logger.info("Processing document sections")@@ -187,7 +223,11 @@ class DocxParser(BaseParser):f"Section processing completed in {section_processing_time:.2f}s")logger.info("Combining all text parts")- text = "\n\n".join([part for part in text_parts if part])+ # Prefer body-order markdown (paragraphs interleaved with tables).+ # Fall back to page-section text if composition produced nothing.+ text = (ordered_text or "").strip()+ if not text:+ text = "\n\n".join([part for part in text_parts if part])# Check if the generated text is emptyif not text:@@ -481,7 +521,7 @@ class Docx:from_page: int = 0,to_page: int = 100000,max_workers: Optional[int] = None,- ) -> Tuple[List[LineData], List[Any]]:+ ) -> Tuple[List[LineData], str]:"""Process DOCX document, supporting concurrent processing of each page@@ -492,7 +532,8 @@ class Docx:max_workers: Maximum number of workers, default to None (system decides)Returns:- tuple: (List of LineData objects with document content, List of tables)+ tuple: (List of LineData objects with document content,+ body-order markdown with tables interleaved)"""logger.info("Processing DOCX document")@@ -503,7 +544,7 @@ class Docx:# Load documentself.doc = self._load_document(binary)if not self.doc:- return [], []+ return [], ""# Identify page structureself.para_page_mapping = self._identify_page_paragraph_mapping(to_page)@@ -517,7 +558,7 @@ class Docx:)if not pages_to_process:logger.warning("No pages to process after applying page limits!")- return [], []+ return [], ""# Initialize shared resourcesself._init_shared_resources()@@ -531,17 +572,17 @@ class Docx:max_workers,)- # Process tables- tbls = self._process_tables()+ ordered_text = self._compose_ordered_markdown(pages_to_process)# Clean up document resourcesself.doc = Nonelogger.info(f"Document processing complete, "- f"extracted {len(self.all_lines)} text sections and {len(tbls)} tables"+ f"extracted {len(self.all_lines)} text sections, "+ f"{len(ordered_text)} characters of body-order markdown")- return self.all_lines, tbls+ return self.all_lines, ordered_textdef _load_document(self, binary):"""Load document@@ -1027,60 +1068,52 @@ class Docx:except Exception as e:logger.error(f"Failed to remove temporary file: {str(e)}")- def _process_tables(self):- """Process tables in the document+ def _compose_ordered_markdown(self, pages_to_process: List[int]) -> str:+ """Walk body paragraphs and tables in document order and emit GFM.- Returns:- list: List of tables+ Paragraph processing is page-limited; tables are included only when they+ sit within that same paragraph range so max_pages does not leak later+ tables into the parsed text."""- tbls = []- table_count = len(self.doc.tables)- if table_count > 0:- logger.info(f"Processing {table_count} tables")- for tb_idx, tb in enumerate(self.doc.tables):- if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume- logger.info(f"Processing table {tb_idx + 1}/{table_count}")-- # Optimize: Check if table is empty- if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows):- logger.info(f"Skipping empty table {tb_idx + 1}")- continue-- table_html = self._convert_table_to_html(tb)- # Still using tuple format for tables as they are handled differently- tbls.append(((None, table_html), ""))-- return tbls-- def _convert_table_to_html(self, table):- """Convert table to HTML-- Args:- table: Table object-- Returns:- str: HTML formatted table- """- html = "<table>"- for r in table.rows:- html += "<tr>"- i = 0- while i < len(r.cells):- span = 1- c = r.cells[i]- for j in range(i + 1, len(r.cells)):- if c.text == r.cells[j].text:- span += 1- i = j- i += 1- html += (- f"<td>{c.text}</td>"- if span == 1- else f"<td colspan='{span}'>{c.text}</td>"- )- html += "</tr>"- html += "</table>"- return html+ allowed = set()+ for page in pages_to_process:+ allowed.update(self.para_page_mapping.get(page, []))+ min_allowed = min(allowed) if allowed else 0+ max_allowed = max(allowed) if allowed else -1++ parts: List[str] = []+ para_i = 0+ tbl_i = 0+ paragraphs = self.doc.paragraphs+ tables = self.doc.tables++ for child in self.doc.element.body:+ if child.tag == _W_P:+ if para_i in allowed and para_i < len(paragraphs):+ text = (paragraphs[para_i].text or "").strip()+ if text:+ parts.append(text)+ para_i += 1+ continue+ if child.tag != _W_TBL:+ continue+ include_table = False+ if not allowed:+ include_table = True+ elif min_allowed <= para_i <= max_allowed + 1:+ include_table = True+ if include_table and tbl_i < len(tables):+ table = tables[tbl_i]+ if not (+ len(table.rows) == 0+ or all(len(row.cells) == 0 for row in table.rows)+ ):+ markdown = table_to_gfm_markdown(table)+ if markdown:+ parts.append(markdown)+ tbl_i += 1++ return "\n\n".join(parts)def _safe_concat_images(self, images):"""Safely concatenate image listsdiff --git a/docreader/tests/test_docx_tables.py b/docreader/tests/test_docx_tables.pynew file mode 100644index 000000000..b3338d4fe--- /dev/null+++ b/docreader/tests/test_docx_tables.py@@ -0,0 +1,213 @@+import importlib.util+import io+import sys+import types+import unittest+from concurrent.futures import ThreadPoolExecutor+from pathlib import Path+from unittest.mock import patch++from docx import Document as WordDocument+++def _load_docx_parser():+ """Load docx_parser without triggering the heavy package __init__.++ ``docreader/parser/__init__.py`` imports doc_parser -> textract, a heavy+ dependency unrelated to DOCX parsing. Registering the packages as bare+ namespaces lets us import only the modules docx_parser actually needs.+ """+ root = Path(__file__).resolve().parents[2]+ docreader_pkg = types.ModuleType("docreader")+ docreader_pkg.__path__ = [str(root / "docreader")]+ sys.modules.setdefault("docreader", docreader_pkg)+ parser_pkg = types.ModuleType("docreader.parser")+ parser_pkg.__path__ = [str(root / "docreader" / "parser")]+ sys.modules["docreader.parser"] = parser_pkg++ spec = importlib.util.spec_from_file_location(+ "docreader.parser.docx_parser", root / "docreader" / "parser" / "docx_parser.py"+ )+ module = importlib.util.module_from_spec(spec)+ sys.modules["docreader.parser.docx_parser"] = module+ spec.loader.exec_module(module)+ return module+++docx_parser = _load_docx_parser()+DocxParser = docx_parser.DocxParser+table_to_gfm_markdown = docx_parser.table_to_gfm_markdown+++def _parse(content):+ """Parse a DOCX through the real Docx processor.++ The production path uses a ProcessPoolExecutor backed by a multiprocessing+ Manager. Some CI sandboxes forbid POSIX semaphores, so run the same page+ task pool on threads and swap the manager for a plain list - parse+ behavior is identical, only the execution backend changes.+ """++ class _FakeManager:+ def __enter__(self):+ self._items = []+ return self++ def __exit__(self, *exc):+ return False++ def list(self):+ return self._items++ with patch.object(docx_parser, "Manager", _FakeManager), patch.object(+ docx_parser, "ProcessPoolExecutor", ThreadPoolExecutor+ ):+ return DocxParser(max_pages=100).parse_into_text(content)+++def _docx_bytes(build):+ doc = WordDocument()+ build(doc)+ buf = io.BytesIO()+ doc.save(buf)+ return buf.getvalue()+++class DocxTableContentTest(unittest.TestCase):+ """Regression test: DOCX tables must be kept in the parsed text.++ Docx.__call__ used to return tables separately from the text lines, and+ parse_into_text dropped them, silently losing all table content.+ """++ def _docx_with_table(self, cells):+ def build(doc):+ doc.add_paragraph("Introduction paragraph")+ table = doc.add_table(rows=len(cells), cols=len(cells[0]))+ for r, row in enumerate(cells):+ for c, value in enumerate(row):+ table.cell(r, c).text = value++ return _docx_bytes(build)++ def test_table_content_is_kept_in_parsed_text(self):+ content = self._docx_with_table(+ [["City", "Population"], ["Beijing", "21.5M"]]+ )+ document = _parse(content)+ self.assertIn("Introduction paragraph", document.content)+ for cell in ("City", "Population", "Beijing", "21.5M"):+ self.assertIn(cell, document.content)+ self.assertIn("| City | Population |", document.content)+ self.assertIn("| --- | --- |", document.content)++ def test_table_only_document_is_not_empty(self):+ def build(doc):+ table = doc.add_table(rows=2, cols=1)+ table.cell(0, 0).text = "Header"+ table.cell(1, 0).text = "Value"++ document = _parse(_docx_bytes(build))+ self.assertIn("Header", document.content)+ self.assertIn("Value", document.content)++ def test_pipe_in_cell_does_not_break_table(self):+ def build(doc):+ table = doc.add_table(rows=2, cols=1)+ table.cell(0, 0).text = "A|B"+ table.cell(1, 0).text = "C"++ document = _parse(_docx_bytes(build))+ self.assertIn(r"A\|B", document.content)++ def test_empty_adjacent_cells_keep_columns(self):+ def build(doc):+ table = doc.add_table(rows=2, cols=3)+ table.cell(0, 0).text = "A"+ table.cell(0, 1).text = "B"+ table.cell(0, 2).text = "C"+ table.cell(1, 0).text = "1"+ table.cell(1, 1).text = ""+ table.cell(1, 2).text = ""++ document = _parse(_docx_bytes(build))+ self.assertIn("| A | B | C |", document.content)+ self.assertIn("| 1 | | |", document.content)++ def test_equal_adjacent_cells_are_not_collapsed(self):+ """Adjacent independent cells with the same text must stay separate.++ Regression for the #2634 control row: ``相同值 | 相同值 | 独立值``.+ """++ def build(doc):+ table = doc.add_table(rows=2, cols=3)+ table.cell(0, 0).text = "相同值"+ table.cell(0, 1).text = "相同值"+ table.cell(0, 2).text = "独立值"+ table.cell(1, 0).text = "x"+ table.cell(1, 1).text = "y"+ table.cell(1, 2).text = "z"++ document = _parse(_docx_bytes(build))+ self.assertIn("| 相同值 | 相同值 | 独立值 |", document.content)+ self.assertIn("| x | y | z |", document.content)++ def test_horizontal_merge_keeps_grid_width(self):+ def build(doc):+ table = doc.add_table(rows=2, cols=3)+ table.cell(0, 0).merge(table.cell(0, 1)).text = "merged"+ table.cell(0, 2).text = "right"+ table.cell(1, 0).text = "a"+ table.cell(1, 1).text = "b"+ table.cell(1, 2).text = "c"++ document = _parse(_docx_bytes(build))+ self.assertIn("| merged | merged | right |", document.content)+ self.assertIn("| a | b | c |", document.content)++ def test_table_stays_between_paragraphs(self):+ def build(doc):+ doc.add_paragraph("before table")+ table = doc.add_table(rows=2, cols=1)+ table.cell(0, 0).text = "H"+ table.cell(1, 0).text = "V"+ doc.add_paragraph("after table")++ document = _parse(_docx_bytes(build))+ before = document.content.index("before table")+ header = document.content.index("| H |")+ after = document.content.index("after table")+ self.assertLess(before, header)+ self.assertLess(header, after)++ def test_html_like_cell_text_is_literal(self):+ def build(doc):+ table = doc.add_table(rows=1, cols=2)+ table.cell(0, 0).text = "a < b"+ table.cell(0, 1).text = "</td><td>injected"++ document = _parse(_docx_bytes(build))+ self.assertIn("| a < b | </td><td>injected |", document.content)++ def test_table_to_gfm_markdown_helper(self):+ doc = WordDocument()+ table = doc.add_table(rows=2, cols=2)+ table.cell(0, 0).text = "A"+ table.cell(0, 1).text = "B"+ table.cell(1, 0).text = "C"+ table.cell(1, 1).text = "D"+ self.assertEqual(+ table_to_gfm_markdown(table),+ "| A | B |\n| --- | --- |\n| C | D |",+ )++ def test_table_to_gfm_markdown_skips_empty(self):+ class _Empty:+ rows = []++ self.assertEqual(table_to_gfm_markdown(_Empty()), "")+++if __name__ == "__main__":+ unittest.main()