Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: derive uploaded markdown outlines from headings

NoxsMedia4738ead45178
diff --git a/deeptutor/reading/extract.py b/deeptutor/reading/extract.py
index 0d41ca4c8..28fe28e4b 100644
--- a/deeptutor/reading/extract.py
+++ b/deeptutor/reading/extract.py
@@ -14,7 +14,9 @@ Per-format strategy, and why:
raw view.
* **PPTX** — the shared extractor already emits ``--- Slide N ---`` separators,
so we split on those instead of re-implementing python-pptx handling.
-* **everything else** (EPUB, DOCX, XLSX, TXT, MD, code, …) — the shared
+* **Markdown** — the shared extractor's text, cut at usable ATX headings and
+ then at paragraph boundaries for long sections.
+* **everything else** (EPUB, DOCX, XLSX, TXT, code, …) — the shared
extractor's plain text, cut into fixed-size *sections* on paragraph
boundaries.
@@ -151,6 +153,9 @@ def extract_material(path: str | Path, *, data: bytes | None = None) -> Extracti
extraction = _extract_epub(data, source.name)
elif suffix == ".pptx":
extraction = _extract_slides(source)
+ elif suffix in {".md", ".markdown"}:
+ units, outline = split_markdown_by_headings(_shared_extract(source))
+ extraction = Extraction(units=units, unit="section", extractor="text", outline=outline)
else:
extraction = _extract_sections(source)
diff --git a/tests/reading/test_engine.py b/tests/reading/test_engine.py
index 271ff833f..dcfbf86f7 100644
--- a/tests/reading/test_engine.py
+++ b/tests/reading/test_engine.py
@@ -168,7 +168,7 @@ def test_pdf_without_bookmarks_does_not_invent_contents(tmp_path: Path) -> None:
def test_text_file_is_cut_into_sections_on_paragraph_boundaries(tmp_path: Path) -> None:
paragraph = "Dense prose about attention mechanisms. " * 30 # ~1.2k chars
- path = tmp_path / "notes.md"
+ path = tmp_path / "notes.txt"
path.write_text("\n\n".join([paragraph] * 8), encoding="utf-8")
extraction = extract_material(path)
@@ -180,6 +180,54 @@ def test_text_file_is_cut_into_sections_on_paragraph_boundaries(tmp_path: Path)
assert all(unit.startswith("Dense prose") for unit in extraction.units)
[email protected]("suffix", [".md", ".markdown"])
+def test_uploaded_markdown_uses_headings_for_stored_outline(
+ store: ReadingStore, tmp_path: Path, suffix: str
+) -> None:
+ prose = "Ordinary prose at a section boundary. " * 95
+ path = tmp_path / f"notes{suffix}"
+ path.write_text(
+ "# Opening\n\nIntroduction.\n\n"
+ "## Long section\n\n" + prose + "\n\n```md\n# Not a heading\n```\n\n"
+ "### Finish\n\nConclusion.",
+ encoding="utf-8",
+ )
+
+ manifest = store.ingest(path)
+ outline = store.outline(manifest.material_id)
+
+ assert manifest.unit == "section"
+ assert [row.title for row in outline] == [
+ "Opening",
+ "Long section",
+ "Long section",
+ "Finish",
+ ]
+ assert [row.level for row in outline] == [1, 2, 2, 3]
+ assert [row.locator for row in outline] == [1, 2, 3, 4]
+ assert all(row.synthesised is False for row in outline)
+ assert all("Ordinary prose" not in row.title for row in outline)
+ assert all(len(store.unit_text(manifest.material_id, row.locator)) <= 4200 for row in outline)
+ assert "# Not a heading" in "\n".join(
+ store.unit_text(manifest.material_id, row.locator) for row in outline
+ )
+
+
[email protected]("markdown", ["No headings.\n\nMore prose.", "# Only\n\nMore prose."])
+def test_uploaded_markdown_with_fewer_than_two_headings_keeps_flat_fallback(
+ store: ReadingStore, tmp_path: Path, markdown: str
+) -> None:
+ path = tmp_path / "flat.md"
+ path.write_text(markdown, encoding="utf-8")
+
+ extraction = extract_material(path)
+ manifest = store.ingest(path)
+
+ assert extraction.units == split_into_sections(markdown)
+ assert extraction.outline == ()
+ assert all(row.synthesised is True for row in store.outline(manifest.material_id))
+
+
def test_epub_preserves_spine_units_source_hrefs_and_nested_outline(tmp_path: Path) -> None:
extraction = extract_material(_write_epub(tmp_path / "book.epub"))