Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638)
Fei.Chena7ac21ce769f
diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.pyindex 48e37819f..bd20acd29 100644--- a/docreader/parser/excel_parser.py+++ b/docreader/parser/excel_parser.py@@ -21,7 +21,7 @@ from docreader.parser.excel_convert import (normalize_excel_bytes,)from docreader.parser.xlsx_merge import fill_merged_cells_xlsx-from docreader.parser.xlsx_repair import repair_xlsx_bytes+from docreader.parser.xlsx_repair import repair_xlsx_bytes, sanitize_xlsx_styleslogger = logging.getLogger(__name__)@@ -205,7 +205,17 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:repaired = repair_xlsx_bytes(data)if repaired is not None:data = repaired- return fill_merged_cells_xlsx(data)+ try:+ return fill_merged_cells_xlsx(data)+ except TypeError:+ # A non-conforming styles.xml (empty/malformed fills) makes openpyxl's+ # load_workbook raise before pandas ever sees the file (#3637).+ # sanitize_xlsx_styles returns None when the fills are clean, in which+ # case the TypeError has a different cause and must propagate.+ sanitized = sanitize_xlsx_styles(data)+ if sanitized is None:+ raise+ return fill_merged_cells_xlsx(sanitized)def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:diff --git a/docreader/parser/markitdown_parser.py b/docreader/parser/markitdown_parser.pyindex f2a28915d..b5934333a 100644--- a/docreader/parser/markitdown_parser.py+++ b/docreader/parser/markitdown_parser.py@@ -15,6 +15,7 @@ from docreader.parser.pptx_media import (attach_pptx_media_to_markdown,markdown_needs_pptx_media_attach,)+from docreader.parser.xlsx_repair import sanitize_xlsx_styleslogger = logging.getLogger(__name__)@@ -46,6 +47,14 @@ class StdMarkitdownParser(BaseParser):ft = "pptx"elif ft == "docx":content = fill_vertical_merged_cells_docx(content)+ elif ft in ("xlsx", "xlsm"):+ # MarkItDown reads spreadsheets through pandas/openpyxl, so the+ # same non-conforming styles.xml fills that break the builtin+ # ExcelParser break this engine too (#3637). The sanitize pass+ # scans only styles.xml and returns None for clean workbooks.+ sanitized = sanitize_xlsx_styles(content)+ if sanitized is not None:+ content = sanitizedif ext and not ext.startswith("."):ext = "." + extdiff --git a/docreader/parser/xlsx_repair.py b/docreader/parser/xlsx_repair.pyindex 9087d61a9..c320b436e 100644--- a/docreader/parser/xlsx_repair.py+++ b/docreader/parser/xlsx_repair.py@@ -3,10 +3,13 @@from __future__ import annotationsimport io+import loggingimport reimport zipfilefrom typing import Callable, Dict, Iterable, Set+logger = logging.getLogger(__name__)+SST_PART = "xl/sharedStrings.xml"_SST_OVERRIDE_RE = re.compile(r'<Override[^>]*PartName="[^"]*sharedStrings\.xml"[^>]*/>',@@ -124,3 +127,81 @@ def _rewrite_zip(for name, data in files.items():zout.writestr(name, data)return out.getvalue()+++# The one stylesheet openpyxl reads (openpyxl.xml.constants.ARC_STYLE). Match+# it exactly: a suffix match also hits xl/richData/richStyles.xml (Excel 365+# in-cell images / data types), which carries no fills.+STYLES_PART = "xl/styles.xml"+_REPLACEMENT_FILL = b'<fill><patternFill patternType="none"/></fill>'+_FILLS_BLOCK_RE = re.compile(rb"<fills\b[^>]*>.*?</fills>", re.S)+# Case-insensitive so wrong-case <Fill>…</Fill> entries are caught too.+_FILL_ELEMENT_RE = re.compile(rb"<fill\s*/>|<fill\s*>.*?</fill\s*>", re.S | re.I)+# A fill openpyxl can deserialize starts (case-sensitively) with a+# patternFill or gradientFill child — everything else, including the+# self-closing <fill/>, is rejected by its sequence descriptor.+_WELL_FORMED_FILL_RE = re.compile(rb"<fill\s*>\s*<(?:patternFill|gradientFill)[\s/>]")+++def sanitize_xlsx_styles(content: bytes) -> bytes | None:+ """Rewrite styles.xml fills that openpyxl cannot deserialize.++ Some exporters (cloud-doc systems, LabView, Acumatica ERP) write ``<fill>``+ elements with no child, or a child that is neither patternFill nor+ gradientFill. openpyxl's strict sequence descriptor raises+ ``TypeError: expected <class 'openpyxl.styles.fills.Fill'>`` on them and+ treats the workbook as non-conformant, so both parse engines die on a+ file Excel itself opens fine (#3637). Replacing each malformed fill in+ place with a no-op patternFill keeps the fill list length — and therefore+ every cellXf fillId reference — stable. Returns None when nothing needs+ fixing, so callers pay one styles.xml scan on the happy path.++ Never raises: on any internal failure the original bytes flow on+ unchanged, so a caller recovering from its own error keeps that error's+ cause instead of trading it for a sanitizer failure.+ """+ try:+ return _sanitize_xlsx_styles(content)+ except Exception:+ logger.warning(+ "XLSX styles sanitize failed; passing original bytes through",+ exc_info=True,+ )+ return None+++def _sanitize_xlsx_styles(content: bytes) -> bytes | None:+ if not zipfile.is_zipfile(io.BytesIO(content)):+ return None++ with zipfile.ZipFile(io.BytesIO(content)) as zin:+ if STYLES_PART not in zin.namelist():+ return None+ styles = zin.read(STYLES_PART)++ block = _FILLS_BLOCK_RE.search(styles)+ if block is None:+ return None++ fixed_count = 0++ def _fix(match: "re.Match[bytes]") -> bytes:+ nonlocal fixed_count+ if _WELL_FORMED_FILL_RE.match(match.group(0)):+ return match.group(0)+ fixed_count += 1+ return _REPLACEMENT_FILL++ patched_block = _FILL_ELEMENT_RE.sub(_fix, block.group(0))+ if fixed_count == 0:+ return None++ # Doubles as the recurrence meter for #3637: how often real uploads+ # carry non-conforming fills, and how many per file.+ logger.info(+ "Sanitized %d non-conforming fill(s) in XLSX styles.xml before parse",+ fixed_count,+ )++ patched = styles[: block.start()] + patched_block + styles[block.end():]+ return _rewrite_zip(zin, lambda files: {**files, STYLES_PART: patched})diff --git a/docreader/tests/test_xlsx_styles_repair.py b/docreader/tests/test_xlsx_styles_repair.pynew file mode 100644index 000000000..6d637ec33--- /dev/null+++ b/docreader/tests/test_xlsx_styles_repair.py@@ -0,0 +1,216 @@+"""Regression tests for non-conforming styles.xml fills (#3637).++Real-world exporters (cloud-doc systems, LabView, Acumatica ERP) write+``<fill>`` elements openpyxl cannot deserialize. The fixtures below replace+the first fill of an openpyxl-written workbook at the zip level, which+reproduces the production failure without an original file.+"""++import io+import os+import re+import subprocess+import sys+import tempfile+import unittest+import zipfile+from unittest.mock import patch++import openpyxl+import pandas as pd++from docreader.parser import xlsx_repair+from docreader.parser.excel_parser import _prepare_xlsx_bytes+from docreader.parser.markitdown_parser import StdMarkitdownParser+from docreader.parser.xlsx_merge import fill_merged_cells_xlsx+from docreader.parser.xlsx_repair import sanitize_xlsx_styles++CRASHING_FILLS = {+ "empty-self-closing": b"<fill/>",+ "capitalized": b"<Fill/>",+ "text-content": b"<fill>oops</fill>",+}++# openpyxl happens to tolerate an unknown fill child, but it is still+# non-conforming; the sanitizer normalizes it like the crashing forms.+TOLERATED_FILL = b"<fill><Foo/></fill>"+++def _workbook_bytes() -> bytes:+ wb = openpyxl.Workbook()+ ws = wb.active+ ws["A1"] = "hello"+ ws["B2"] = 42+ ws["C3"] = "world"+ ws.merge_cells("C3:D3")+ bio = io.BytesIO()+ wb.save(bio)+ return bio.getvalue()+++def _with_first_fill_replaced(workbook: bytes, fragment: bytes) -> bytes:+ out = io.BytesIO()+ with zipfile.ZipFile(io.BytesIO(workbook)) as zin, zipfile.ZipFile(+ out, "w", zipfile.ZIP_DEFLATED+ ) as zout:+ for item in zin.infolist():+ data = zin.read(item.filename)+ if item.filename == "xl/styles.xml":+ data = re.sub(+ b"<fill>.*?</fill>", fragment, data, count=1, flags=re.S+ )+ zout.writestr(item, data)+ return out.getvalue()+++def _with_rich_styles_part(workbook: bytes) -> bytes:+ """Add the xl/richData/richStyles.xml part Excel 365 writes for in-cell+ images / data types; its name also ends in ``styles.xml``."""+ out = io.BytesIO()+ with zipfile.ZipFile(io.BytesIO(workbook)) as zin, zipfile.ZipFile(+ out, "w", zipfile.ZIP_DEFLATED+ ) as zout:+ for item in zin.infolist():+ zout.writestr(item, zin.read(item.filename))+ zout.writestr(+ "xl/richData/richStyles.xml",+ b'<richStyleSheet xmlns="http://schemas.microsoft.com/office/'+ b'spreadsheetml/2017/richdata2"/>',+ )+ return out.getvalue()+++class TestSanitizeXlsxStyles(unittest.TestCase):+ def test_clean_workbook_is_returned_unchanged(self):+ self.assertIsNone(sanitize_xlsx_styles(_workbook_bytes()))++ def test_non_zip_input_is_ignored(self):+ self.assertIsNone(sanitize_xlsx_styles(b"not a zip"))++ def test_repair_is_logged_with_count(self):+ fixture = _with_first_fill_replaced(+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]+ )+ with self.assertLogs("docreader.parser.xlsx_repair", level="INFO") as logs:+ sanitize_xlsx_styles(fixture)+ self.assertTrue(+ any("Sanitized 1 non-conforming fill" in line for line in logs.output),+ logs.output,+ )++ def test_internal_failure_returns_none_instead_of_raising(self):+ fixture = _with_first_fill_replaced(+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]+ )+ with patch(+ "docreader.parser.xlsx_repair._rewrite_zip", side_effect=OSError("boom")+ ):+ with self.assertLogs("docreader.parser.xlsx_repair", level="WARNING"):+ result = sanitize_xlsx_styles(fixture)+ self.assertIsNone(result)++ def test_each_malformed_form_is_repaired_in_place(self):+ for name, fragment in {**CRASHING_FILLS, "tolerated-unknown-child": TOLERATED_FILL}.items():+ with self.subTest(form=name):+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)++ sanitized = sanitize_xlsx_styles(fixture)+ self.assertIsNotNone(sanitized)++ wb = openpyxl.load_workbook(io.BytesIO(sanitized), data_only=True)+ ws = wb.active+ self.assertEqual(ws["A1"].value, "hello")+ self.assertEqual(ws["B2"].value, 42)++ with zipfile.ZipFile(io.BytesIO(sanitized)) as z:+ styles = z.read("xl/styles.xml")+ count = re.search(rb'<fills count="(\d+)"', styles)+ num_fills = len(re.findall(rb"<fill[ />]", styles))+ self.assertIsNotNone(count)+ self.assertEqual(int(count.group(1)), num_fills)++ def test_workbook_stylesheet_is_picked_over_rich_styles_part(self):+ fixture = _with_rich_styles_part(+ _with_first_fill_replaced(+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]+ )+ )+ sanitized = sanitize_xlsx_styles(fixture)+ self.assertIsNotNone(sanitized)+ openpyxl.load_workbook(io.BytesIO(sanitized), data_only=True)++ def test_stylesheet_choice_does_not_depend_on_hash_seed(self):+ # Zip member names are str, so any set-ordered lookup changes with+ # PYTHONHASHSEED; run the sanitizer under several seeds.+ fixture = _with_rich_styles_part(+ _with_first_fill_replaced(+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]+ )+ )+ repo_root = os.path.dirname(+ os.path.dirname(os.path.dirname(os.path.abspath(xlsx_repair.__file__)))+ )+ script = (+ "import sys\n"+ "from docreader.parser.xlsx_repair import sanitize_xlsx_styles\n"+ "data = open(sys.argv[1], 'rb').read()\n"+ "sys.exit(0 if sanitize_xlsx_styles(data) is not None else 1)\n"+ )+ with tempfile.TemporaryDirectory() as tmpdir:+ path = os.path.join(tmpdir, "rich.xlsx")+ with open(path, "wb") as handle:+ handle.write(fixture)+ for seed in range(8):+ with self.subTest(seed=seed):+ env = {+ **os.environ,+ "PYTHONHASHSEED": str(seed),+ "PYTHONPATH": os.pathsep.join(+ filter(None, [repo_root, os.environ.get("PYTHONPATH")])+ ),+ }+ result = subprocess.run(+ [sys.executable, "-c", script, path], env=env+ )+ self.assertEqual(result.returncode, 0)++ def test_crashing_forms_raise_without_the_fix(self):+ for name, fragment in CRASHING_FILLS.items():+ with self.subTest(form=name):+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)+ with self.assertRaises(TypeError):+ openpyxl.load_workbook(io.BytesIO(fixture), data_only=True)+++class TestBuiltinChain(unittest.TestCase):+ def test_prepare_xlsx_bytes_recovers_and_keeps_merge_fill(self):+ for name, fragment in CRASHING_FILLS.items():+ with self.subTest(form=name):+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)+ with self.assertRaises(TypeError):+ fill_merged_cells_xlsx(fixture) # the unfixed failure++ prepared = _prepare_xlsx_bytes(fixture)+ excel = pd.ExcelFile(io.BytesIO(prepared), engine="openpyxl")+ self.assertTrue(excel.sheet_names)++ wb = openpyxl.load_workbook(io.BytesIO(prepared), data_only=True)+ ws = wb.active+ # the merge-cell prefill still applied on the sanitized bytes+ self.assertEqual(ws["C3"].value, "world")+ self.assertEqual(ws["D3"].value, "world")+++class TestMarkitdownChain(unittest.TestCase):+ def test_markitdown_engine_parses_sanitized_workbook(self):+ for name, fragment in {**CRASHING_FILLS, "tolerated-unknown-child": TOLERATED_FILL}.items():+ with self.subTest(form=name):+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)+ parser = StdMarkitdownParser(file_type="xlsx")+ doc = parser.parse_into_text(fixture)+ self.assertIn("hello", doc.content)+ self.assertIn("world", doc.content)+++if __name__ == "__main__":+ unittest.main()