Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: sanitize XML text before rendering every DOCX block

Bingxi Zhao (Frank)f41f9cfe74dc
diff --git a/deeptutor/co_writer/docx_converter.py b/deeptutor/co_writer/docx_converter.py
index a5af7a877..01908232a 100644
--- a/deeptutor/co_writer/docx_converter.py
+++ b/deeptutor/co_writer/docx_converter.py
@@ -27,6 +27,7 @@ _MAX_MARKDOWN_CHARS = 600_000
_HYPERLINK_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink"
_CODE_FONTS = {"consolas", "courier new", "courier", "menlo", "monaco", "cascadia mono"}
_CODE_FONT = "Consolas"
+_XML_INVALID_TEXT = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]")
_XML_SPACE = "{http://www.w3.org/XML/1998/namespace}space"
_HEADING_STYLES = {
@@ -458,7 +459,8 @@ def markdown_to_docx(content: str, title: str = "") -> bytes:
del title # any heading already lives in the markdown body
docx = _docx_module()
document = docx.Document()
- lines = (content or "").replace("\r\n", "\n").replace("\r", "\n").split("\n")
+ content = _XML_INVALID_TEXT.sub("", content or "")
+ lines = content.replace("\r\n", "\n").replace("\r", "\n").split("\n")
index = 0
while index < len(lines):
line = lines[index]
@@ -627,7 +629,7 @@ def _add_inline_runs(paragraph: Any, text: str) -> None:
# A literal NUL in the input collides with the escape sentinel ``_MASK``
# and would make ``_unmask`` pop a saved char that was never stored. NUL is
# not valid XML text content anyway, so drop it before masking.
- text = text.replace(_MASK, "")
+ text = _XML_INVALID_TEXT.sub("", text)
masked, saved = _mask_escapes(text)
cursor = 0
for match in _INLINE_RE.finditer(masked):
diff --git a/tests/co_writer/test_docx_inline_nul.py b/tests/co_writer/test_docx_inline_nul.py
index ff505d417..c5ced0952 100644
--- a/tests/co_writer/test_docx_inline_nul.py
+++ b/tests/co_writer/test_docx_inline_nul.py
@@ -9,8 +9,11 @@ the whole export instead of treating the NUL as ordinary text.
from __future__ import annotations
+from io import BytesIO
from unittest.mock import MagicMock
+import pytest
+
from deeptutor.co_writer.docx_converter import _add_inline_runs
@@ -44,3 +47,27 @@ def test_nul_next_to_real_escape_is_handled() -> None:
rendered = "".join(paragraph.texts)
assert rendered == "a*bc"
+
+
+ "markdown",
+ [
+ "hello{bad}world",
+ "# hello{bad}world",
+ "```\nhello{bad}world\n```",
+ "| Column |\n| --- |\n| hello{bad}world |",
+ ],
+)
[email protected]("bad", ["\x00", "\x0b", "\x1f", "\ud800", "\ufffe"])
+def test_export_sanitizes_xml_text_in_every_markdown_block(markdown, bad):
+ docx = pytest.importorskip("docx")
+ from deeptutor.co_writer.docx_converter import markdown_to_docx
+
+ data = markdown_to_docx(markdown.format(bad=bad))
+ document = docx.Document(BytesIO(data))
+ text = "\n".join(p.text for p in document.paragraphs)
+ text += "\n".join(
+ cell.text for table in document.tables for row in table.rows for cell in row.cells
+ )
+ assert "helloworld" in text
+ assert bad not in text