DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(reading): close a Markdown fence only on a matching bare run
Hui6dd4542a0bb1
diff --git a/deeptutor/reading/extract.py b/deeptutor/reading/extract.pyindex 28fe28e4b..b5f5acec5 100644--- a/deeptutor/reading/extract.py+++ b/deeptutor/reading/extract.py@@ -59,7 +59,7 @@ SECTION_HARD_CHARS = 4200_SLIDE_SEPARATOR = re.compile(r"^--- Slide \d+ ---$", re.MULTILINE)# Title candidates: a markdown heading, or the first non-trivial line._MD_HEADING = re.compile(r"^\s{0,3}(?P<marks>#{1,6})\s+(?P<title>.+?)\s*#*\s*$")-_MD_FENCE = re.compile(r"^\s{0,3}(?P<marker>`{3,}|~{3,})")+_MD_FENCE = re.compile(r"^\s{0,3}(?P<marker>`{3,}|~{3,})(?P<info>.*)$")# Formats whose original bytes the browser can render faithfully next to the# extracted text. Only PDF today; adding one means teaching the reader pane to@@ -448,18 +448,30 @@ def split_markdown_by_headings(return (), ()boundaries: list[tuple[int, int, str]] = []+ # The run that opened the current fenced block ("```", "~~~~", ...). As in+ # CommonMark, only a bare run of the same character that is at least as long+ # closes it, so a ```` fence can show a ``` example and "```py" inside a+ # block is content, not a closing fence.fence_marker = ""offset = 0for line in normalised.splitlines(keepends=True):- fence = _MD_FENCE.match(line)+ fence = _MD_FENCE.match(line.rstrip("\n"))if fence:- marker = fence.group("marker")+ marker, info = fence.group("marker"), fence.group("info")if not fence_marker:- fence_marker = marker[0]- elif marker[0] == fence_marker:+ # A backtick fence's info string cannot contain a backtick.+ if not (marker[0] == "`" and "`" in info):+ fence_marker = marker+ offset += len(line)+ continue+ elif (+ marker[0] == fence_marker[0]+ and len(marker) >= len(fence_marker)+ and not info.strip()+ ):fence_marker = ""- offset += len(line)- continue+ offset += len(line)+ continueif not fence_marker:heading = _MD_HEADING.match(line.rstrip("\n"))if heading:diff --git a/tests/reading/test_ingestion.py b/tests/reading/test_ingestion.pyindex 86d55f373..b44b1ed3d 100644--- a/tests/reading/test_ingestion.py+++ b/tests/reading/test_ingestion.py@@ -379,6 +379,31 @@ def test_heading_sections_keep_long_continuations_under_the_same_title() -> None)+ "block",+ [+ pytest.param("````md\n```py\n# Not a heading\n```\n````", id="longer-fence-around-example"),+ pytest.param("```bash\n```py\n# Not a heading\n```", id="fence-with-info-does-not-close"),+ pytest.param("~~~\n# Not a heading\n~~~", id="tilde-fence"),+ ],+)+def test_heading_splitter_ignores_headings_inside_fenced_code(block: str) -> None:+ markdown = f"# Guide\n\nIntro.\n\n{block}\n\n## Next\n\nDone."++ units, outline = split_markdown_by_headings(markdown)++ assert [row.title for row in outline] == ["Guide", "Next"]+ assert "# Not a heading" in units[0]+++def test_heading_splitter_does_not_open_a_fence_on_inline_backticks() -> None:+ markdown = "# Guide\n\n```inline``` code.\n\n## Next\n\nDone."++ _units, outline = split_markdown_by_headings(markdown)++ assert [row.title for row in outline] == ["Guide", "Next"]++def test_heading_splitter_falls_back_when_only_the_page_title_exists() -> None:markdown = "# Article title\n\nA flat article paragraph.\n\nAnother paragraph."