Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): sanitize non-conforming styles.xml fills before openpyxl loads (#3637) (#3638)

Fei.Chena7ac21ce769f
diff --git a/docreader/parser/excel_parser.py b/docreader/parser/excel_parser.py
index 48e37819f..bd20acd29 100644
--- a/docreader/parser/excel_parser.py
+++ b/docreader/parser/excel_parser.py
@@ -21,7 +21,7 @@ from docreader.parser.excel_convert import (
normalize_excel_bytes,
)
from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
-from docreader.parser.xlsx_repair import repair_xlsx_bytes
+from docreader.parser.xlsx_repair import repair_xlsx_bytes, sanitize_xlsx_styles
logger = logging.getLogger(__name__)
@@ -205,7 +205,17 @@ def _prepare_xlsx_bytes(data: bytes) -> bytes:
repaired = repair_xlsx_bytes(data)
if repaired is not None:
data = repaired
- return fill_merged_cells_xlsx(data)
+ try:
+ return fill_merged_cells_xlsx(data)
+ except TypeError:
+ # A non-conforming styles.xml (empty/malformed fills) makes openpyxl's
+ # load_workbook raise before pandas ever sees the file (#3637).
+ # sanitize_xlsx_styles returns None when the fills are clean, in which
+ # case the TypeError has a different cause and must propagate.
+ sanitized = sanitize_xlsx_styles(data)
+ if sanitized is None:
+ raise
+ return fill_merged_cells_xlsx(sanitized)
def _open_excel_file(content: bytes, file_type: str | None = None) -> pd.ExcelFile:
diff --git a/docreader/parser/markitdown_parser.py b/docreader/parser/markitdown_parser.py
index f2a28915d..b5934333a 100644
--- a/docreader/parser/markitdown_parser.py
+++ b/docreader/parser/markitdown_parser.py
@@ -15,6 +15,7 @@ from docreader.parser.pptx_media import (
attach_pptx_media_to_markdown,
markdown_needs_pptx_media_attach,
)
+from docreader.parser.xlsx_repair import sanitize_xlsx_styles
logger = logging.getLogger(__name__)
@@ -46,6 +47,14 @@ class StdMarkitdownParser(BaseParser):
ft = "pptx"
elif ft == "docx":
content = fill_vertical_merged_cells_docx(content)
+ elif ft in ("xlsx", "xlsm"):
+ # MarkItDown reads spreadsheets through pandas/openpyxl, so the
+ # same non-conforming styles.xml fills that break the builtin
+ # ExcelParser break this engine too (#3637). The sanitize pass
+ # scans only styles.xml and returns None for clean workbooks.
+ sanitized = sanitize_xlsx_styles(content)
+ if sanitized is not None:
+ content = sanitized
if ext and not ext.startswith("."):
ext = "." + ext
diff --git a/docreader/parser/xlsx_repair.py b/docreader/parser/xlsx_repair.py
index 9087d61a9..c320b436e 100644
--- a/docreader/parser/xlsx_repair.py
+++ b/docreader/parser/xlsx_repair.py
@@ -3,10 +3,13 @@
from __future__ import annotations
import io
+import logging
import re
import zipfile
from typing import Callable, Dict, Iterable, Set
+logger = logging.getLogger(__name__)
+
SST_PART = "xl/sharedStrings.xml"
_SST_OVERRIDE_RE = re.compile(
r'<Override[^>]*PartName="[^"]*sharedStrings\.xml"[^>]*/>',
@@ -124,3 +127,81 @@ def _rewrite_zip(
for name, data in files.items():
zout.writestr(name, data)
return out.getvalue()
+
+
+# The one stylesheet openpyxl reads (openpyxl.xml.constants.ARC_STYLE). Match
+# it exactly: a suffix match also hits xl/richData/richStyles.xml (Excel 365
+# in-cell images / data types), which carries no fills.
+STYLES_PART = "xl/styles.xml"
+_REPLACEMENT_FILL = b'<fill><patternFill patternType="none"/></fill>'
+_FILLS_BLOCK_RE = re.compile(rb"<fills\b[^>]*>.*?</fills>", re.S)
+# Case-insensitive so wrong-case <Fill>…</Fill> entries are caught too.
+_FILL_ELEMENT_RE = re.compile(rb"<fill\s*/>|<fill\s*>.*?</fill\s*>", re.S | re.I)
+# A fill openpyxl can deserialize starts (case-sensitively) with a
+# patternFill or gradientFill child — everything else, including the
+# self-closing <fill/>, is rejected by its sequence descriptor.
+_WELL_FORMED_FILL_RE = re.compile(rb"<fill\s*>\s*<(?:patternFill|gradientFill)[\s/>]")
+
+
+def sanitize_xlsx_styles(content: bytes) -> bytes | None:
+ """Rewrite styles.xml fills that openpyxl cannot deserialize.
+
+ Some exporters (cloud-doc systems, LabView, Acumatica ERP) write ``<fill>``
+ elements with no child, or a child that is neither patternFill nor
+ gradientFill. openpyxl's strict sequence descriptor raises
+ ``TypeError: expected <class 'openpyxl.styles.fills.Fill'>`` on them and
+ treats the workbook as non-conformant, so both parse engines die on a
+ file Excel itself opens fine (#3637). Replacing each malformed fill in
+ place with a no-op patternFill keeps the fill list length — and therefore
+ every cellXf fillId reference — stable. Returns None when nothing needs
+ fixing, so callers pay one styles.xml scan on the happy path.
+
+ Never raises: on any internal failure the original bytes flow on
+ unchanged, so a caller recovering from its own error keeps that error's
+ cause instead of trading it for a sanitizer failure.
+ """
+ try:
+ return _sanitize_xlsx_styles(content)
+ except Exception:
+ logger.warning(
+ "XLSX styles sanitize failed; passing original bytes through",
+ exc_info=True,
+ )
+ return None
+
+
+def _sanitize_xlsx_styles(content: bytes) -> bytes | None:
+ if not zipfile.is_zipfile(io.BytesIO(content)):
+ return None
+
+ with zipfile.ZipFile(io.BytesIO(content)) as zin:
+ if STYLES_PART not in zin.namelist():
+ return None
+ styles = zin.read(STYLES_PART)
+
+ block = _FILLS_BLOCK_RE.search(styles)
+ if block is None:
+ return None
+
+ fixed_count = 0
+
+ def _fix(match: "re.Match[bytes]") -> bytes:
+ nonlocal fixed_count
+ if _WELL_FORMED_FILL_RE.match(match.group(0)):
+ return match.group(0)
+ fixed_count += 1
+ return _REPLACEMENT_FILL
+
+ patched_block = _FILL_ELEMENT_RE.sub(_fix, block.group(0))
+ if fixed_count == 0:
+ return None
+
+ # Doubles as the recurrence meter for #3637: how often real uploads
+ # carry non-conforming fills, and how many per file.
+ logger.info(
+ "Sanitized %d non-conforming fill(s) in XLSX styles.xml before parse",
+ fixed_count,
+ )
+
+ patched = styles[: block.start()] + patched_block + styles[block.end():]
+ return _rewrite_zip(zin, lambda files: {**files, STYLES_PART: patched})
diff --git a/docreader/tests/test_xlsx_styles_repair.py b/docreader/tests/test_xlsx_styles_repair.py
new file mode 100644
index 000000000..6d637ec33
--- /dev/null
+++ b/docreader/tests/test_xlsx_styles_repair.py
@@ -0,0 +1,216 @@
+"""Regression tests for non-conforming styles.xml fills (#3637).
+
+Real-world exporters (cloud-doc systems, LabView, Acumatica ERP) write
+``<fill>`` elements openpyxl cannot deserialize. The fixtures below replace
+the first fill of an openpyxl-written workbook at the zip level, which
+reproduces the production failure without an original file.
+"""
+
+import io
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+import zipfile
+from unittest.mock import patch
+
+import openpyxl
+import pandas as pd
+
+from docreader.parser import xlsx_repair
+from docreader.parser.excel_parser import _prepare_xlsx_bytes
+from docreader.parser.markitdown_parser import StdMarkitdownParser
+from docreader.parser.xlsx_merge import fill_merged_cells_xlsx
+from docreader.parser.xlsx_repair import sanitize_xlsx_styles
+
+CRASHING_FILLS = {
+ "empty-self-closing": b"<fill/>",
+ "capitalized": b"<Fill/>",
+ "text-content": b"<fill>oops</fill>",
+}
+
+# openpyxl happens to tolerate an unknown fill child, but it is still
+# non-conforming; the sanitizer normalizes it like the crashing forms.
+TOLERATED_FILL = b"<fill><Foo/></fill>"
+
+
+def _workbook_bytes() -> bytes:
+ wb = openpyxl.Workbook()
+ ws = wb.active
+ ws["A1"] = "hello"
+ ws["B2"] = 42
+ ws["C3"] = "world"
+ ws.merge_cells("C3:D3")
+ bio = io.BytesIO()
+ wb.save(bio)
+ return bio.getvalue()
+
+
+def _with_first_fill_replaced(workbook: bytes, fragment: bytes) -> bytes:
+ out = io.BytesIO()
+ with zipfile.ZipFile(io.BytesIO(workbook)) as zin, zipfile.ZipFile(
+ out, "w", zipfile.ZIP_DEFLATED
+ ) as zout:
+ for item in zin.infolist():
+ data = zin.read(item.filename)
+ if item.filename == "xl/styles.xml":
+ data = re.sub(
+ b"<fill>.*?</fill>", fragment, data, count=1, flags=re.S
+ )
+ zout.writestr(item, data)
+ return out.getvalue()
+
+
+def _with_rich_styles_part(workbook: bytes) -> bytes:
+ """Add the xl/richData/richStyles.xml part Excel 365 writes for in-cell
+ images / data types; its name also ends in ``styles.xml``."""
+ out = io.BytesIO()
+ with zipfile.ZipFile(io.BytesIO(workbook)) as zin, zipfile.ZipFile(
+ out, "w", zipfile.ZIP_DEFLATED
+ ) as zout:
+ for item in zin.infolist():
+ zout.writestr(item, zin.read(item.filename))
+ zout.writestr(
+ "xl/richData/richStyles.xml",
+ b'<richStyleSheet xmlns="http://schemas.microsoft.com/office/'
+ b'spreadsheetml/2017/richdata2"/>',
+ )
+ return out.getvalue()
+
+
+class TestSanitizeXlsxStyles(unittest.TestCase):
+ def test_clean_workbook_is_returned_unchanged(self):
+ self.assertIsNone(sanitize_xlsx_styles(_workbook_bytes()))
+
+ def test_non_zip_input_is_ignored(self):
+ self.assertIsNone(sanitize_xlsx_styles(b"not a zip"))
+
+ def test_repair_is_logged_with_count(self):
+ fixture = _with_first_fill_replaced(
+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]
+ )
+ with self.assertLogs("docreader.parser.xlsx_repair", level="INFO") as logs:
+ sanitize_xlsx_styles(fixture)
+ self.assertTrue(
+ any("Sanitized 1 non-conforming fill" in line for line in logs.output),
+ logs.output,
+ )
+
+ def test_internal_failure_returns_none_instead_of_raising(self):
+ fixture = _with_first_fill_replaced(
+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]
+ )
+ with patch(
+ "docreader.parser.xlsx_repair._rewrite_zip", side_effect=OSError("boom")
+ ):
+ with self.assertLogs("docreader.parser.xlsx_repair", level="WARNING"):
+ result = sanitize_xlsx_styles(fixture)
+ self.assertIsNone(result)
+
+ def test_each_malformed_form_is_repaired_in_place(self):
+ for name, fragment in {**CRASHING_FILLS, "tolerated-unknown-child": TOLERATED_FILL}.items():
+ with self.subTest(form=name):
+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)
+
+ sanitized = sanitize_xlsx_styles(fixture)
+ self.assertIsNotNone(sanitized)
+
+ wb = openpyxl.load_workbook(io.BytesIO(sanitized), data_only=True)
+ ws = wb.active
+ self.assertEqual(ws["A1"].value, "hello")
+ self.assertEqual(ws["B2"].value, 42)
+
+ with zipfile.ZipFile(io.BytesIO(sanitized)) as z:
+ styles = z.read("xl/styles.xml")
+ count = re.search(rb'<fills count="(\d+)"', styles)
+ num_fills = len(re.findall(rb"<fill[ />]", styles))
+ self.assertIsNotNone(count)
+ self.assertEqual(int(count.group(1)), num_fills)
+
+ def test_workbook_stylesheet_is_picked_over_rich_styles_part(self):
+ fixture = _with_rich_styles_part(
+ _with_first_fill_replaced(
+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]
+ )
+ )
+ sanitized = sanitize_xlsx_styles(fixture)
+ self.assertIsNotNone(sanitized)
+ openpyxl.load_workbook(io.BytesIO(sanitized), data_only=True)
+
+ def test_stylesheet_choice_does_not_depend_on_hash_seed(self):
+ # Zip member names are str, so any set-ordered lookup changes with
+ # PYTHONHASHSEED; run the sanitizer under several seeds.
+ fixture = _with_rich_styles_part(
+ _with_first_fill_replaced(
+ _workbook_bytes(), CRASHING_FILLS["empty-self-closing"]
+ )
+ )
+ repo_root = os.path.dirname(
+ os.path.dirname(os.path.dirname(os.path.abspath(xlsx_repair.__file__)))
+ )
+ script = (
+ "import sys\n"
+ "from docreader.parser.xlsx_repair import sanitize_xlsx_styles\n"
+ "data = open(sys.argv[1], 'rb').read()\n"
+ "sys.exit(0 if sanitize_xlsx_styles(data) is not None else 1)\n"
+ )
+ with tempfile.TemporaryDirectory() as tmpdir:
+ path = os.path.join(tmpdir, "rich.xlsx")
+ with open(path, "wb") as handle:
+ handle.write(fixture)
+ for seed in range(8):
+ with self.subTest(seed=seed):
+ env = {
+ **os.environ,
+ "PYTHONHASHSEED": str(seed),
+ "PYTHONPATH": os.pathsep.join(
+ filter(None, [repo_root, os.environ.get("PYTHONPATH")])
+ ),
+ }
+ result = subprocess.run(
+ [sys.executable, "-c", script, path], env=env
+ )
+ self.assertEqual(result.returncode, 0)
+
+ def test_crashing_forms_raise_without_the_fix(self):
+ for name, fragment in CRASHING_FILLS.items():
+ with self.subTest(form=name):
+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)
+ with self.assertRaises(TypeError):
+ openpyxl.load_workbook(io.BytesIO(fixture), data_only=True)
+
+
+class TestBuiltinChain(unittest.TestCase):
+ def test_prepare_xlsx_bytes_recovers_and_keeps_merge_fill(self):
+ for name, fragment in CRASHING_FILLS.items():
+ with self.subTest(form=name):
+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)
+ with self.assertRaises(TypeError):
+ fill_merged_cells_xlsx(fixture) # the unfixed failure
+
+ prepared = _prepare_xlsx_bytes(fixture)
+ excel = pd.ExcelFile(io.BytesIO(prepared), engine="openpyxl")
+ self.assertTrue(excel.sheet_names)
+
+ wb = openpyxl.load_workbook(io.BytesIO(prepared), data_only=True)
+ ws = wb.active
+ # the merge-cell prefill still applied on the sanitized bytes
+ self.assertEqual(ws["C3"].value, "world")
+ self.assertEqual(ws["D3"].value, "world")
+
+
+class TestMarkitdownChain(unittest.TestCase):
+ def test_markitdown_engine_parses_sanitized_workbook(self):
+ for name, fragment in {**CRASHING_FILLS, "tolerated-unknown-child": TOLERATED_FILL}.items():
+ with self.subTest(form=name):
+ fixture = _with_first_fill_replaced(_workbook_bytes(), fragment)
+ parser = StdMarkitdownParser(file_type="xlsx")
+ doc = parser.parse_into_text(fixture)
+ self.assertIn("hello", doc.content)
+ self.assertIn("world", doc.content)
+
+
+if __name__ == "__main__":
+ unittest.main()