DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: derive uploaded markdown outlines from headings
NoxsMedia4738ead45178
diff --git a/deeptutor/reading/extract.py b/deeptutor/reading/extract.pyindex 0d41ca4c8..28fe28e4b 100644--- a/deeptutor/reading/extract.py+++ b/deeptutor/reading/extract.py@@ -14,7 +14,9 @@ Per-format strategy, and why:raw view.* **PPTX** — the shared extractor already emits ``--- Slide N ---`` separators,so we split on those instead of re-implementing python-pptx handling.-* **everything else** (EPUB, DOCX, XLSX, TXT, MD, code, …) — the shared+* **Markdown** — the shared extractor's text, cut at usable ATX headings and+ then at paragraph boundaries for long sections.+* **everything else** (EPUB, DOCX, XLSX, TXT, code, …) — the sharedextractor's plain text, cut into fixed-size *sections* on paragraphboundaries.@@ -151,6 +153,9 @@ def extract_material(path: str | Path, *, data: bytes | None = None) -> Extractiextraction = _extract_epub(data, source.name)elif suffix == ".pptx":extraction = _extract_slides(source)+ elif suffix in {".md", ".markdown"}:+ units, outline = split_markdown_by_headings(_shared_extract(source))+ extraction = Extraction(units=units, unit="section", extractor="text", outline=outline)else:extraction = _extract_sections(source)diff --git a/tests/reading/test_engine.py b/tests/reading/test_engine.pyindex 271ff833f..dcfbf86f7 100644--- a/tests/reading/test_engine.py+++ b/tests/reading/test_engine.py@@ -168,7 +168,7 @@ def test_pdf_without_bookmarks_does_not_invent_contents(tmp_path: Path) -> None:def test_text_file_is_cut_into_sections_on_paragraph_boundaries(tmp_path: Path) -> None:paragraph = "Dense prose about attention mechanisms. " * 30 # ~1.2k chars- path = tmp_path / "notes.md"+ path = tmp_path / "notes.txt"path.write_text("\n\n".join([paragraph] * 8), encoding="utf-8")extraction = extract_material(path)@@ -180,6 +180,54 @@ def test_text_file_is_cut_into_sections_on_paragraph_boundaries(tmp_path: Path)assert all(unit.startswith("Dense prose") for unit in extraction.units)[email protected]("suffix", [".md", ".markdown"])+def test_uploaded_markdown_uses_headings_for_stored_outline(+ store: ReadingStore, tmp_path: Path, suffix: str+) -> None:+ prose = "Ordinary prose at a section boundary. " * 95+ path = tmp_path / f"notes{suffix}"+ path.write_text(+ "# Opening\n\nIntroduction.\n\n"+ "## Long section\n\n" + prose + "\n\n```md\n# Not a heading\n```\n\n"+ "### Finish\n\nConclusion.",+ encoding="utf-8",+ )++ manifest = store.ingest(path)+ outline = store.outline(manifest.material_id)++ assert manifest.unit == "section"+ assert [row.title for row in outline] == [+ "Opening",+ "Long section",+ "Long section",+ "Finish",+ ]+ assert [row.level for row in outline] == [1, 2, 2, 3]+ assert [row.locator for row in outline] == [1, 2, 3, 4]+ assert all(row.synthesised is False for row in outline)+ assert all("Ordinary prose" not in row.title for row in outline)+ assert all(len(store.unit_text(manifest.material_id, row.locator)) <= 4200 for row in outline)+ assert "# Not a heading" in "\n".join(+ store.unit_text(manifest.material_id, row.locator) for row in outline+ )++[email protected]("markdown", ["No headings.\n\nMore prose.", "# Only\n\nMore prose."])+def test_uploaded_markdown_with_fewer_than_two_headings_keeps_flat_fallback(+ store: ReadingStore, tmp_path: Path, markdown: str+) -> None:+ path = tmp_path / "flat.md"+ path.write_text(markdown, encoding="utf-8")++ extraction = extract_material(path)+ manifest = store.ingest(path)++ assert extraction.units == split_into_sections(markdown)+ assert extraction.outline == ()+ assert all(row.synthesised is True for row in store.outline(manifest.material_id))++def test_epub_preserves_spine_units_source_hrefs_and_nested_outline(tmp_path: Path) -> None:extraction = extract_material(_write_epub(tmp_path / "book.epub"))