DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: sanitize XML text before rendering every DOCX block
Bingxi Zhao (Frank)f41f9cfe74dc
diff --git a/deeptutor/co_writer/docx_converter.py b/deeptutor/co_writer/docx_converter.pyindex a5af7a877..01908232a 100644--- a/deeptutor/co_writer/docx_converter.py+++ b/deeptutor/co_writer/docx_converter.py@@ -27,6 +27,7 @@ _MAX_MARKDOWN_CHARS = 600_000_HYPERLINK_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink"_CODE_FONTS = {"consolas", "courier new", "courier", "menlo", "monaco", "cascadia mono"}_CODE_FONT = "Consolas"+_XML_INVALID_TEXT = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]")_XML_SPACE = "{http://www.w3.org/XML/1998/namespace}space"_HEADING_STYLES = {@@ -458,7 +459,8 @@ def markdown_to_docx(content: str, title: str = "") -> bytes:del title # any heading already lives in the markdown bodydocx = _docx_module()document = docx.Document()- lines = (content or "").replace("\r\n", "\n").replace("\r", "\n").split("\n")+ content = _XML_INVALID_TEXT.sub("", content or "")+ lines = content.replace("\r\n", "\n").replace("\r", "\n").split("\n")index = 0while index < len(lines):line = lines[index]@@ -627,7 +629,7 @@ def _add_inline_runs(paragraph: Any, text: str) -> None:# A literal NUL in the input collides with the escape sentinel ``_MASK``# and would make ``_unmask`` pop a saved char that was never stored. NUL is# not valid XML text content anyway, so drop it before masking.- text = text.replace(_MASK, "")+ text = _XML_INVALID_TEXT.sub("", text)masked, saved = _mask_escapes(text)cursor = 0for match in _INLINE_RE.finditer(masked):diff --git a/tests/co_writer/test_docx_inline_nul.py b/tests/co_writer/test_docx_inline_nul.pyindex ff505d417..c5ced0952 100644--- a/tests/co_writer/test_docx_inline_nul.py+++ b/tests/co_writer/test_docx_inline_nul.py@@ -9,8 +9,11 @@ the whole export instead of treating the NUL as ordinary text.from __future__ import annotations+from io import BytesIOfrom unittest.mock import MagicMock+import pytest+from deeptutor.co_writer.docx_converter import _add_inline_runs@@ -44,3 +47,27 @@ def test_nul_next_to_real_escape_is_handled() -> None:rendered = "".join(paragraph.texts)assert rendered == "a*bc"+++ "markdown",+ [+ "hello{bad}world",+ "# hello{bad}world",+ "```\nhello{bad}world\n```",+ "| Column |\n| --- |\n| hello{bad}world |",+ ],+)[email protected]("bad", ["\x00", "\x0b", "\x1f", "\ud800", "\ufffe"])+def test_export_sanitizes_xml_text_in_every_markdown_block(markdown, bad):+ docx = pytest.importorskip("docx")+ from deeptutor.co_writer.docx_converter import markdown_to_docx++ data = markdown_to_docx(markdown.format(bad=bad))+ document = docx.Document(BytesIO(data))+ text = "\n".join(p.text for p in document.paragraphs)+ text += "\n".join(+ cell.text for table in document.tables for row in table.rows for cell in row.cells+ )+ assert "helloworld" in text+ assert bad not in text