Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(reading): close a Markdown fence only on a matching bare run

Hui6dd4542a0bb1
diff --git a/deeptutor/reading/extract.py b/deeptutor/reading/extract.py
index 28fe28e4b..b5f5acec5 100644
--- a/deeptutor/reading/extract.py
+++ b/deeptutor/reading/extract.py
@@ -59,7 +59,7 @@ SECTION_HARD_CHARS = 4200
_SLIDE_SEPARATOR = re.compile(r"^--- Slide \d+ ---$", re.MULTILINE)
# Title candidates: a markdown heading, or the first non-trivial line.
_MD_HEADING = re.compile(r"^\s{0,3}(?P<marks>#{1,6})\s+(?P<title>.+?)\s*#*\s*$")
-_MD_FENCE = re.compile(r"^\s{0,3}(?P<marker>`{3,}|~{3,})")
+_MD_FENCE = re.compile(r"^\s{0,3}(?P<marker>`{3,}|~{3,})(?P<info>.*)$")
# Formats whose original bytes the browser can render faithfully next to the
# extracted text. Only PDF today; adding one means teaching the reader pane to
@@ -448,18 +448,30 @@ def split_markdown_by_headings(
return (), ()
boundaries: list[tuple[int, int, str]] = []
+ # The run that opened the current fenced block ("```", "~~~~", ...). As in
+ # CommonMark, only a bare run of the same character that is at least as long
+ # closes it, so a ```` fence can show a ``` example and "```py" inside a
+ # block is content, not a closing fence.
fence_marker = ""
offset = 0
for line in normalised.splitlines(keepends=True):
- fence = _MD_FENCE.match(line)
+ fence = _MD_FENCE.match(line.rstrip("\n"))
if fence:
- marker = fence.group("marker")
+ marker, info = fence.group("marker"), fence.group("info")
if not fence_marker:
- fence_marker = marker[0]
- elif marker[0] == fence_marker:
+ # A backtick fence's info string cannot contain a backtick.
+ if not (marker[0] == "`" and "`" in info):
+ fence_marker = marker
+ offset += len(line)
+ continue
+ elif (
+ marker[0] == fence_marker[0]
+ and len(marker) >= len(fence_marker)
+ and not info.strip()
+ ):
fence_marker = ""
- offset += len(line)
- continue
+ offset += len(line)
+ continue
if not fence_marker:
heading = _MD_HEADING.match(line.rstrip("\n"))
if heading:
diff --git a/tests/reading/test_ingestion.py b/tests/reading/test_ingestion.py
index 86d55f373..b44b1ed3d 100644
--- a/tests/reading/test_ingestion.py
+++ b/tests/reading/test_ingestion.py
@@ -379,6 +379,31 @@ def test_heading_sections_keep_long_continuations_under_the_same_title() -> None
)
+ "block",
+ [
+ pytest.param("````md\n```py\n# Not a heading\n```\n````", id="longer-fence-around-example"),
+ pytest.param("```bash\n```py\n# Not a heading\n```", id="fence-with-info-does-not-close"),
+ pytest.param("~~~\n# Not a heading\n~~~", id="tilde-fence"),
+ ],
+)
+def test_heading_splitter_ignores_headings_inside_fenced_code(block: str) -> None:
+ markdown = f"# Guide\n\nIntro.\n\n{block}\n\n## Next\n\nDone."
+
+ units, outline = split_markdown_by_headings(markdown)
+
+ assert [row.title for row in outline] == ["Guide", "Next"]
+ assert "# Not a heading" in units[0]
+
+
+def test_heading_splitter_does_not_open_a_fence_on_inline_backticks() -> None:
+ markdown = "# Guide\n\n```inline``` code.\n\n## Next\n\nDone."
+
+ _units, outline = split_markdown_by_headings(markdown)
+
+ assert [row.title for row in outline] == ["Guide", "Next"]
+
+
def test_heading_splitter_falls_back_when_only_the_page_title_exists() -> None:
markdown = "# Article title\n\nA flat article paragraph.\n\nAnother paragraph."