Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
feat(extractor): Markdown/AsciiDoc heading detection + multilingual ToC (#44)
Shubham Srivastavac5c7fc59a42a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6758f35..8341a36 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Added+- Structure detection now recognizes Markdown/AsciiDoc ATX headings (`#`, `==`)+ as chapters when no numeric "Chapter N" headings are present, fixing a+ zero-chapter result for `.md`/`.adoc` sources. Headings inside fenced code+ blocks are ignored.+- Table-of-contents detection extended to Chinese, Japanese, French, German,+ Italian, and Dutch.+### Security- **CI security scanning** — added CodeQL (Python, security-and-quality + weeklyschedule), Bandit (gates on HIGH severity; reports MEDIUM+ informationally), anddiff --git a/scripts/extractor/utils.py b/scripts/extractor/utils.pyindex f9747ec..e6d140e 100644--- a/scripts/extractor/utils.py+++ b/scripts/extractor/utils.py@@ -85,6 +85,70 @@ _CN_NUM_CLASS = "〇零一二两三四五六七八九十百千"_CN_CHAPTER = re.compile(rf"^\s*第\s*([0-9{_CN_NUM_CLASS}]+)\s*[章回卷节篇讲]")_MD_CN_HEADING = re.compile(rf"^#{{1,6}}\s+第?\s*([{_CN_NUM_CLASS}]+)\s*[·、.::章回卷节篇讲]")+# Table-of-contents header lines across common languages. Anchored to a whole+# line (^\s*X\s*$) so an inline "the contents of this chapter" never matches.+_TOC_HEADERS = (+ "table of contents", "contents", "índice", "sumário", # EN / ES / PT+ "目录", "目錄", "目次", # Chinese / Japanese+ "table des matières", # French+ "inhaltsverzeichnis", # German+ "indice", "sommario", # Italian (no accent — distinct from índice above)+ "inhoudsopgave", # Dutch+)+_TOC_PATTERN = re.compile(+ r"^\s*(?:" + "|".join(re.escape(h) for h in _TOC_HEADERS) + r")\s*$",+ re.IGNORECASE | re.MULTILINE,+)++# ATX-style heading: "# Title", "## Section", AsciiDoc "= Title", "== Section".+# The required space after the marker distinguishes an AsciiDoc "== X" from a+# reStructuredText underline "=====" (no space) — the latter is intentionally+# ignored (RST underline headings are out of scope).+_ATX_HEADING = re.compile(r"^(#{1,6}|={1,6})\s+(.+?)\s*#*$")+++def _structural_chapter_count(text: str) -> int:+ """Count chapter-like ATX headings in Markdown/AsciiDoc sources.++ Groups distinct (case-normalized) heading titles by depth and returns the+ count at the shallowest depth with >= 2 distinct headings — this selects the+ real chapter level in the common "# Book Title / ## Chapter" layout where the+ top level appears once. Headings inside fenced code blocks are skipped so a+ "# comment" in a code sample is not mistaken for a chapter. Headings whose+ title starts with an ASCII digit (e.g. "## 5 Setup") are excluded — they+ are numeric-list-style, not chapters.+ """+ levels: dict[int, set[str]] = {}+ in_fence = False+ for line in text.splitlines():+ s = line.strip()+ if s.startswith("```") or s.startswith("~~~"):+ in_fence = not in_fence+ continue+ if in_fence:+ continue+ m = _ATX_HEADING.match(s)+ if not m:+ continue+ title = m.group(2).strip().lower()+ if not title:+ continue+ # Skip headings whose title starts with a bare ASCII digit ("## 5 Setup").+ # These are numeric list-item style headings, not chapter headings; they+ # were intentionally excluded from numeric detection and must stay excluded.+ if title[0].isdigit():+ continue+ levels.setdefault(len(m.group(1)), set()).add(title)+ if not levels:+ return 0+ for depth in sorted(levels):+ if len(levels[depth]) >= 2:+ return len(levels[depth])+ # No level has >= 2 distinct headings: a thin doc (e.g. one heading per+ # level). Count them all — this path runs only as a fallback when numeric+ # chapter detection already found zero, so it cannot inflate real books.+ return sum(len(titles) for titles in levels.values())+def _cn_numeral_to_int(s: str) -> int | None:"""Parse a Chinese (or ASCII-digit) chapter numeral into an int (1..999)."""@@ -171,14 +235,15 @@ def detect_structure(text: str) -> dict:if num is not None:numbers.add(num)headings.append(line.strip())- chapters_detected = len(numbers)-- # Look for ToC indicators in the first ~30k chars- toc_pattern = re.compile(- r"^\s*(?:table of contents|contents|índice|sumário)\s*$",- re.IGNORECASE | re.MULTILINE,+ numeric_count = len(numbers)+ # Fall back to structural (Markdown/AsciiDoc) headings only when no numeric+ # "Chapter N" headings were found, so books with real chapters are unaffected.+ chapters_detected = (+ numeric_count if numeric_count > 0 else _structural_chapter_count(text))- has_toc = bool(toc_pattern.search(text[:30000]))++ # Look for ToC indicators in the first ~30k chars (multilingual; see _TOC_PATTERN)+ has_toc = bool(_TOC_PATTERN.search(text[:30000]))return {"chapters_detected": chapters_detected,diff --git a/tests/test_extractor.py b/tests/test_extractor.pyindex b137cbf..d748314 100644--- a/tests/test_extractor.py+++ b/tests/test_extractor.py@@ -553,6 +553,38 @@ class TestDetectStructure:result = detect_structure(text)assert result["has_toc"] is False+ def test_toc_chinese(self):+ assert detect_structure("目录\n第一章 开始\n第二章 进阶\n")["has_toc"] is True++ def test_toc_japanese(self):+ assert detect_structure("目次\n本文")["has_toc"] is True++ def test_toc_french(self):+ assert detect_structure("Table des matières\n1 Intro")["has_toc"] is True++ def test_toc_german(self):+ assert detect_structure("Inhaltsverzeichnis\n1 Einleitung")["has_toc"] is True++ def test_toc_italian(self):+ assert detect_structure("Indice\n1 Introduzione")["has_toc"] is True++ def test_toc_dutch(self):+ assert detect_structure("Inhoudsopgave\n1 Inleiding")["has_toc"] is True++ def test_toc_spanish_accented(self):+ assert detect_structure("Índice\n1 Introducción")["has_toc"] is True++ def test_toc_traditional_chinese(self):+ assert detect_structure("目錄\n第一章")["has_toc"] is True++ def test_toc_italian_sommario(self):+ assert detect_structure("Sommario\n1 Introduzione")["has_toc"] is True++ def test_toc_inline_word_is_not_toc(self):+ # "contents"/"index" mid-sentence must not be mistaken for a ToC header+ text = "The contents of this chapter are varied and the index is long.\n"+ assert detect_structure(text)["has_toc"] is False+def test_numbered_list_items_are_not_chapters(self):# The AI-Engineering failure: numbered list items were counted as chapters.text = (@@ -632,6 +664,44 @@ class TestDetectStructure:# A bare Arabic-numeral Markdown heading is NOT a chapter (unchanged).assert detect_structure("## 5 Setup\n## 6 Teardown\n")["chapters_detected"] == 0+ def test_markdown_atx_chapters(self):+ text = "# Book Title\n\n## Introduction\nbody\n\n## Getting Started\nbody\n\n## Advanced\nbody\n"+ assert detect_structure(text)["chapters_detected"] == 3++ def test_markdown_all_h1_chapters(self):+ text = "# Chapter One\ntext\n# Chapter Two\ntext\n# Chapter Three\ntext\n"+ assert detect_structure(text)["chapters_detected"] == 3++ def test_asciidoc_section_headings(self):+ text = "= Doc Title\n\n== First Section\nbody\n\n== Second Section\nbody\n"+ assert detect_structure(text)["chapters_detected"] == 2++ def test_asciidoc_deeper_levels(self):+ # AsciiDoc levels 3-6 (=== .. ======) are also recognized.+ text = "=== Alpha\nbody\n=== Beta\nbody\n=== Gamma\nbody\n"+ assert detect_structure(text)["chapters_detected"] == 3++ def test_markdown_prefixed_chapter_word(self):+ # "## Chapter 1:" is not caught by the numeric scan (line starts with '#'),+ # so the structural fallback must count it.+ text = "## Chapter 1: Intro\nbody\n## Chapter 2: Models\nbody\n"+ assert detect_structure(text)["chapters_detected"] == 2++ def test_headings_inside_code_fence_are_ignored(self):+ text = "# Real A\n\n```python\n# a comment\n# another comment\n```\n\n# Real B\n"+ assert detect_structure(text)["chapters_detected"] == 2++ def test_plain_prose_has_no_structural_chapters(self):+ # Regression guard: no headings -> still 0, unchanged behavior+ text = "Just paragraphs of prose.\nMore prose here.\n"+ assert detect_structure(text)["chapters_detected"] == 0++ def test_numeric_chapters_win_over_markdown_subsections(self):+ # A book with real "Chapter N" headings must report the numeric count,+ # not the count of markdown subsection headings.+ text = "Chapter 1: Intro\n## sub a\n## sub b\n## sub c\nChapter 2: Next\n"+ assert detect_structure(text)["chapters_detected"] == 2+def test_chinese_numeral_parsing(self):assert _cn_numeral_to_int("一") == 1assert _cn_numeral_to_int("十") == 10