DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(co-writer): keep a placeholder row and warn when DOCX table row parsing fails (#1700)
evan188199-tech65d9b8f557d1
diff --git a/deeptutor/co_writer/docx_converter.py b/deeptutor/co_writer/docx_converter.pyindex aa93ebaf3..4dd78dbb7 100644--- a/deeptutor/co_writer/docx_converter.py+++ b/deeptutor/co_writer/docx_converter.py@@ -9,10 +9,13 @@ from __future__ import annotationsfrom collections import dequefrom io import BytesIO+import loggingimport refrom typing import Any, Iteratorimport zipfile+logger = logging.getLogger(__name__)+_OOXML_MAGIC = b"PK\x03\x04"_OLE_MAGIC = b"\xd0\xcf\x11\xe0"_DOCX_MAX_MEMBERS = 4096@@ -60,6 +63,8 @@ _MD_QUOTE_RE = re.compile(r"^>\s?(.*)$")_MD_HR_RE = re.compile(r"^(-{3,}|\*{3,}|_{3,})$")_MD_TABLE_SEP_RE = re.compile(r"^\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*$")_MD_FENCE_RE = re.compile(r"^```")++_UNPARSEABLE_ROW_CELL = "(unparseable row)"_INLINE_RE = re.compile(r"\[(?P<link_text>[^\]\n]*)\]\((?P<link_href>[^)\s]*)\)"r"|`(?P<code>[^`\n]+)`"@@ -120,7 +125,14 @@ def docx_to_markdown(data: bytes, filename: str = "document.docx") -> str:docx = _docx_module()document = docx.Document(BytesIO(data))numbering = _numbering_formats(document)- blocks = [_block_to_markdown(block, numbering) for block in _iter_blocks(document)]+ tables_seen = 0+ blocks = []+ for block in _iter_blocks(document):+ table_ordinal = None+ if type(block).__name__ == "Table":+ table_ordinal = tables_seen+ tables_seen += 1+ blocks.append(_block_to_markdown(block, numbering, table_ordinal))markdown = _join_blocks(blocks)except DocxConversionError:raise@@ -223,9 +235,13 @@ def _iter_blocks(document: Any) -> Iterator[Any]:yield docx.table.Table(child, document)-def _block_to_markdown(block: Any, numbering: dict[tuple[str, str], str]) -> str:+def _block_to_markdown(+ block: Any,+ numbering: dict[tuple[str, str], str],+ table_ordinal: int | None = None,+) -> str:if type(block).__name__ == "Table":- return _table_to_markdown(block)+ return _table_to_markdown(block, table_ordinal)return _paragraph_to_markdown(block, numbering)@@ -348,12 +364,21 @@ def _run_to_markdown(run: Any) -> str:return out-def _table_to_markdown(table: Any) -> str:+def _table_to_markdown(table: Any, table_ordinal: int | None = None) -> str:rows: list[list[str]] = []- for row in table.rows:+ for row_index, row in enumerate(table.rows):try:cells = [_cell_text(cell) for cell in row.cells]- except Exception:+ except Exception as exc:+ where = f"row #{row_index + 1}"+ if table_ordinal is not None:+ where = f"table #{table_ordinal + 1} {where}"+ logger.warning(+ "DOCX %s could not be parsed; kept a placeholder row (%s)",+ where,+ exc,+ )+ rows.append([_UNPARSEABLE_ROW_CELL])continueif any(cell.strip() for cell in cells):rows.append(cells)diff --git a/tests/api/test_co_writer.py b/tests/api/test_co_writer.pyindex b435a907b..5a90cfcda 100644--- a/tests/api/test_co_writer.py+++ b/tests/api/test_co_writer.py@@ -1,6 +1,7 @@"""Co-Writer backend tests: doc id validation, storage CRUD, history limits."""from io import BytesIO+import loggingfrom pathlib import Pathfrom types import SimpleNamespaceimport zipfile@@ -20,9 +21,10 @@ _dt_config.load_config_with_main = lambda *_a, **_k: {from deeptutor.api.routers import co_writer as co_writer_routerfrom deeptutor.api.routers.co_writer import _validate_doc_id-from deeptutor.co_writer import edit_agent+from deeptutor.co_writer import docx_converter, edit_agentfrom deeptutor.co_writer.docx_converter import (DocxConversionError,+ _table_to_markdown,docx_to_markdown,markdown_to_docx,)@@ -373,6 +375,84 @@ def test_docx_to_markdown_rejects_suspicious_zip(monkeypatch):docx_to_markdown(buf.getvalue(), "bomb.docx")+class _TextRow:+ def __init__(self, texts):+ self._cells = [SimpleNamespace(text=text) for text in texts]++ @property+ def cells(self):+ return self._cells+++class _BadRow:+ """Row whose ``cells`` access fails, as with malformed merged cells."""++ @property+ def cells(self):+ raise ValueError("simulated merged-cell failure")+++class _FakeTable:+ def __init__(self, rows):+ self.rows = rows+++def test_table_to_markdown_warns_and_keeps_placeholder_for_unparseable_row(caplog):+ table = _FakeTable(+ [+ _TextRow(["Metric", "Value"]),+ _BadRow(),+ _TextRow(["Revenue", "1.2M"]),+ ]+ )+ with caplog.at_level(logging.WARNING, logger="deeptutor.co_writer.docx_converter"):+ markdown = _table_to_markdown(table, table_ordinal=0)++ assert "| Metric | Value |" in markdown+ assert "| Revenue | 1.2M |" in markdown+ assert "(unparseable row)" in markdown+ warnings = [r for r in caplog.records if r.levelno == logging.WARNING]+ assert warnings, "expected a warning for the unparseable row"+ assert "table #1" in warnings[0].getMessage()+ assert "row #2" in warnings[0].getMessage()+ assert "simulated merged-cell failure" in warnings[0].getMessage()+++def test_docx_to_markdown_bad_table_row_warns_and_keeps_placeholder(monkeypatch, caplog):+ document = DocxDocument()+ table = document.add_table(rows=3, cols=2)+ table.rows[0].cells[0].text = "Metric"+ table.rows[0].cells[1].text = "Value"+ table.rows[1].cells[0].text = "Revenue"+ table.rows[1].cells[1].text = "1.2M"+ table.rows[2].cells[0].text = "Profit"+ table.rows[2].cells[1].text = "0.3M"++ real_cell_text = docx_converter._cell_text++ def flaky_cell_text(cell):+ if (cell.text or "") == "Revenue":+ raise ValueError("simulated cell failure")+ return real_cell_text(cell)++ monkeypatch.setattr(docx_converter, "_cell_text", flaky_cell_text)+ with caplog.at_level(logging.WARNING, logger="deeptutor.co_writer.docx_converter"):+ markdown = docx_to_markdown(_docx_bytes_from_document(document), "broken.docx")++ assert "| Metric | Value |" in markdown+ assert "| Profit | 0.3M |" in markdown+ assert "Revenue" not in markdown+ assert "(unparseable row)" in markdown+ matching = [+ r+ for r in caplog.records+ if r.levelno == logging.WARNING+ and "table #1" in r.getMessage()+ and "row #2" in r.getMessage()+ ]+ assert matching, "expected a warning naming table #1 row #2"++def _client(tmp_path, monkeypatch) -> TestClient:storage = CoWriterStorage(path_service=_StubPathService(tmp_path))monkeypatch.setattr(co_writer_router, "get_co_writer_storage", lambda: storage)