Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(co-writer): keep a placeholder row and warn when DOCX table row parsing fails (#1700)

evan188199-tech65d9b8f557d1
diff --git a/deeptutor/co_writer/docx_converter.py b/deeptutor/co_writer/docx_converter.py
index aa93ebaf3..4dd78dbb7 100644
--- a/deeptutor/co_writer/docx_converter.py
+++ b/deeptutor/co_writer/docx_converter.py
@@ -9,10 +9,13 @@ from __future__ import annotations
from collections import deque
from io import BytesIO
+import logging
import re
from typing import Any, Iterator
import zipfile
+logger = logging.getLogger(__name__)
+
_OOXML_MAGIC = b"PK\x03\x04"
_OLE_MAGIC = b"\xd0\xcf\x11\xe0"
_DOCX_MAX_MEMBERS = 4096
@@ -60,6 +63,8 @@ _MD_QUOTE_RE = re.compile(r"^>\s?(.*)$")
_MD_HR_RE = re.compile(r"^(-{3,}|\*{3,}|_{3,})$")
_MD_TABLE_SEP_RE = re.compile(r"^\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")
_MD_FENCE_RE = re.compile(r"^```")
+
+_UNPARSEABLE_ROW_CELL = "(unparseable row)"
_INLINE_RE = re.compile(
r"\[(?P<link_text>[^\]\n]*)\]\((?P<link_href>[^)\s]*)\)"
r"|`(?P<code>[^`\n]+)`"
@@ -120,7 +125,14 @@ def docx_to_markdown(data: bytes, filename: str = "document.docx") -> str:
docx = _docx_module()
document = docx.Document(BytesIO(data))
numbering = _numbering_formats(document)
- blocks = [_block_to_markdown(block, numbering) for block in _iter_blocks(document)]
+ tables_seen = 0
+ blocks = []
+ for block in _iter_blocks(document):
+ table_ordinal = None
+ if type(block).__name__ == "Table":
+ table_ordinal = tables_seen
+ tables_seen += 1
+ blocks.append(_block_to_markdown(block, numbering, table_ordinal))
markdown = _join_blocks(blocks)
except DocxConversionError:
raise
@@ -223,9 +235,13 @@ def _iter_blocks(document: Any) -> Iterator[Any]:
yield docx.table.Table(child, document)
-def _block_to_markdown(block: Any, numbering: dict[tuple[str, str], str]) -> str:
+def _block_to_markdown(
+ block: Any,
+ numbering: dict[tuple[str, str], str],
+ table_ordinal: int | None = None,
+) -> str:
if type(block).__name__ == "Table":
- return _table_to_markdown(block)
+ return _table_to_markdown(block, table_ordinal)
return _paragraph_to_markdown(block, numbering)
@@ -348,12 +364,21 @@ def _run_to_markdown(run: Any) -> str:
return out
-def _table_to_markdown(table: Any) -> str:
+def _table_to_markdown(table: Any, table_ordinal: int | None = None) -> str:
rows: list[list[str]] = []
- for row in table.rows:
+ for row_index, row in enumerate(table.rows):
try:
cells = [_cell_text(cell) for cell in row.cells]
- except Exception:
+ except Exception as exc:
+ where = f"row #{row_index + 1}"
+ if table_ordinal is not None:
+ where = f"table #{table_ordinal + 1} {where}"
+ logger.warning(
+ "DOCX %s could not be parsed; kept a placeholder row (%s)",
+ where,
+ exc,
+ )
+ rows.append([_UNPARSEABLE_ROW_CELL])
continue
if any(cell.strip() for cell in cells):
rows.append(cells)
diff --git a/tests/api/test_co_writer.py b/tests/api/test_co_writer.py
index b435a907b..5a90cfcda 100644
--- a/tests/api/test_co_writer.py
+++ b/tests/api/test_co_writer.py
@@ -1,6 +1,7 @@
"""Co-Writer backend tests: doc id validation, storage CRUD, history limits."""
from io import BytesIO
+import logging
from pathlib import Path
from types import SimpleNamespace
import zipfile
@@ -20,9 +21,10 @@ _dt_config.load_config_with_main = lambda *_a, **_k: {
from deeptutor.api.routers import co_writer as co_writer_router
from deeptutor.api.routers.co_writer import _validate_doc_id
-from deeptutor.co_writer import edit_agent
+from deeptutor.co_writer import docx_converter, edit_agent
from deeptutor.co_writer.docx_converter import (
DocxConversionError,
+ _table_to_markdown,
docx_to_markdown,
markdown_to_docx,
)
@@ -373,6 +375,84 @@ def test_docx_to_markdown_rejects_suspicious_zip(monkeypatch):
docx_to_markdown(buf.getvalue(), "bomb.docx")
+class _TextRow:
+ def __init__(self, texts):
+ self._cells = [SimpleNamespace(text=text) for text in texts]
+
+ @property
+ def cells(self):
+ return self._cells
+
+
+class _BadRow:
+ """Row whose ``cells`` access fails, as with malformed merged cells."""
+
+ @property
+ def cells(self):
+ raise ValueError("simulated merged-cell failure")
+
+
+class _FakeTable:
+ def __init__(self, rows):
+ self.rows = rows
+
+
+def test_table_to_markdown_warns_and_keeps_placeholder_for_unparseable_row(caplog):
+ table = _FakeTable(
+ [
+ _TextRow(["Metric", "Value"]),
+ _BadRow(),
+ _TextRow(["Revenue", "1.2M"]),
+ ]
+ )
+ with caplog.at_level(logging.WARNING, logger="deeptutor.co_writer.docx_converter"):
+ markdown = _table_to_markdown(table, table_ordinal=0)
+
+ assert "| Metric | Value |" in markdown
+ assert "| Revenue | 1.2M |" in markdown
+ assert "(unparseable row)" in markdown
+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+ assert warnings, "expected a warning for the unparseable row"
+ assert "table #1" in warnings[0].getMessage()
+ assert "row #2" in warnings[0].getMessage()
+ assert "simulated merged-cell failure" in warnings[0].getMessage()
+
+
+def test_docx_to_markdown_bad_table_row_warns_and_keeps_placeholder(monkeypatch, caplog):
+ document = DocxDocument()
+ table = document.add_table(rows=3, cols=2)
+ table.rows[0].cells[0].text = "Metric"
+ table.rows[0].cells[1].text = "Value"
+ table.rows[1].cells[0].text = "Revenue"
+ table.rows[1].cells[1].text = "1.2M"
+ table.rows[2].cells[0].text = "Profit"
+ table.rows[2].cells[1].text = "0.3M"
+
+ real_cell_text = docx_converter._cell_text
+
+ def flaky_cell_text(cell):
+ if (cell.text or "") == "Revenue":
+ raise ValueError("simulated cell failure")
+ return real_cell_text(cell)
+
+ monkeypatch.setattr(docx_converter, "_cell_text", flaky_cell_text)
+ with caplog.at_level(logging.WARNING, logger="deeptutor.co_writer.docx_converter"):
+ markdown = docx_to_markdown(_docx_bytes_from_document(document), "broken.docx")
+
+ assert "| Metric | Value |" in markdown
+ assert "| Profit | 0.3M |" in markdown
+ assert "Revenue" not in markdown
+ assert "(unparseable row)" in markdown
+ matching = [
+ r
+ for r in caplog.records
+ if r.levelno == logging.WARNING
+ and "table #1" in r.getMessage()
+ and "row #2" in r.getMessage()
+ ]
+ assert matching, "expected a warning naming table #1 row #2"
+
+
def _client(tmp_path, monkeypatch) -> TestClient:
storage = CoWriterStorage(path_service=_StubPathService(tmp_path))
monkeypatch.setattr(co_writer_router, "get_co_writer_storage", lambda: storage)