Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

feat(extractor): Markdown/AsciiDoc heading detection + multilingual ToC (#44)

Shubham Srivastavac5c7fc59a42a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6758f35..8341a36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+- Structure detection now recognizes Markdown/AsciiDoc ATX headings (`#`, `==`)
+ as chapters when no numeric "Chapter N" headings are present, fixing a
+ zero-chapter result for `.md`/`.adoc` sources. Headings inside fenced code
+ blocks are ignored.
+- Table-of-contents detection extended to Chinese, Japanese, French, German,
+ Italian, and Dutch.
+
### Security
- **CI security scanning** — added CodeQL (Python, security-and-quality + weekly
schedule), Bandit (gates on HIGH severity; reports MEDIUM+ informationally), and
diff --git a/scripts/extractor/utils.py b/scripts/extractor/utils.py
index f9747ec..e6d140e 100644
--- a/scripts/extractor/utils.py
+++ b/scripts/extractor/utils.py
@@ -85,6 +85,70 @@ _CN_NUM_CLASS = "〇零一二两三四五六七八九十百千"
_CN_CHAPTER = re.compile(rf"^\s*第\s*([0-9{_CN_NUM_CLASS}]+)\s*[章回卷节篇讲]")
_MD_CN_HEADING = re.compile(rf"^#{{1,6}}\s+第?\s*([{_CN_NUM_CLASS}]+)\s*[·、.::章回卷节篇讲]")
+# Table-of-contents header lines across common languages. Anchored to a whole
+# line (^\s*X\s*$) so an inline "the contents of this chapter" never matches.
+_TOC_HEADERS = (
+ "table of contents", "contents", "índice", "sumário", # EN / ES / PT
+ "目录", "目錄", "目次", # Chinese / Japanese
+ "table des matières", # French
+ "inhaltsverzeichnis", # German
+ "indice", "sommario", # Italian (no accent — distinct from índice above)
+ "inhoudsopgave", # Dutch
+)
+_TOC_PATTERN = re.compile(
+ r"^\s*(?:" + "|".join(re.escape(h) for h in _TOC_HEADERS) + r")\s*$",
+ re.IGNORECASE | re.MULTILINE,
+)
+
+# ATX-style heading: "# Title", "## Section", AsciiDoc "= Title", "== Section".
+# The required space after the marker distinguishes an AsciiDoc "== X" from a
+# reStructuredText underline "=====" (no space) — the latter is intentionally
+# ignored (RST underline headings are out of scope).
+_ATX_HEADING = re.compile(r"^(#{1,6}|={1,6})\s+(.+?)\s*#*$")
+
+
+def _structural_chapter_count(text: str) -> int:
+ """Count chapter-like ATX headings in Markdown/AsciiDoc sources.
+
+ Groups distinct (case-normalized) heading titles by depth and returns the
+ count at the shallowest depth with >= 2 distinct headings — this selects the
+ real chapter level in the common "# Book Title / ## Chapter" layout where the
+ top level appears once. Headings inside fenced code blocks are skipped so a
+ "# comment" in a code sample is not mistaken for a chapter. Headings whose
+ title starts with an ASCII digit (e.g. "## 5 Setup") are excluded — they
+ are numeric-list-style, not chapters.
+ """
+ levels: dict[int, set[str]] = {}
+ in_fence = False
+ for line in text.splitlines():
+ s = line.strip()
+ if s.startswith("```") or s.startswith("~~~"):
+ in_fence = not in_fence
+ continue
+ if in_fence:
+ continue
+ m = _ATX_HEADING.match(s)
+ if not m:
+ continue
+ title = m.group(2).strip().lower()
+ if not title:
+ continue
+ # Skip headings whose title starts with a bare ASCII digit ("## 5 Setup").
+ # These are numeric list-item style headings, not chapter headings; they
+ # were intentionally excluded from numeric detection and must stay excluded.
+ if title[0].isdigit():
+ continue
+ levels.setdefault(len(m.group(1)), set()).add(title)
+ if not levels:
+ return 0
+ for depth in sorted(levels):
+ if len(levels[depth]) >= 2:
+ return len(levels[depth])
+ # No level has >= 2 distinct headings: a thin doc (e.g. one heading per
+ # level). Count them all — this path runs only as a fallback when numeric
+ # chapter detection already found zero, so it cannot inflate real books.
+ return sum(len(titles) for titles in levels.values())
+
def _cn_numeral_to_int(s: str) -> int | None:
"""Parse a Chinese (or ASCII-digit) chapter numeral into an int (1..999)."""
@@ -171,14 +235,15 @@ def detect_structure(text: str) -> dict:
if num is not None:
numbers.add(num)
headings.append(line.strip())
- chapters_detected = len(numbers)
-
- # Look for ToC indicators in the first ~30k chars
- toc_pattern = re.compile(
- r"^\s*(?:table of contents|contents|índice|sumário)\s*$",
- re.IGNORECASE | re.MULTILINE,
+ numeric_count = len(numbers)
+ # Fall back to structural (Markdown/AsciiDoc) headings only when no numeric
+ # "Chapter N" headings were found, so books with real chapters are unaffected.
+ chapters_detected = (
+ numeric_count if numeric_count > 0 else _structural_chapter_count(text)
)
- has_toc = bool(toc_pattern.search(text[:30000]))
+
+ # Look for ToC indicators in the first ~30k chars (multilingual; see _TOC_PATTERN)
+ has_toc = bool(_TOC_PATTERN.search(text[:30000]))
return {
"chapters_detected": chapters_detected,
diff --git a/tests/test_extractor.py b/tests/test_extractor.py
index b137cbf..d748314 100644
--- a/tests/test_extractor.py
+++ b/tests/test_extractor.py
@@ -553,6 +553,38 @@ class TestDetectStructure:
result = detect_structure(text)
assert result["has_toc"] is False
+ def test_toc_chinese(self):
+ assert detect_structure("目录\n第一章 开始\n第二章 进阶\n")["has_toc"] is True
+
+ def test_toc_japanese(self):
+ assert detect_structure("目次\n本文")["has_toc"] is True
+
+ def test_toc_french(self):
+ assert detect_structure("Table des matières\n1 Intro")["has_toc"] is True
+
+ def test_toc_german(self):
+ assert detect_structure("Inhaltsverzeichnis\n1 Einleitung")["has_toc"] is True
+
+ def test_toc_italian(self):
+ assert detect_structure("Indice\n1 Introduzione")["has_toc"] is True
+
+ def test_toc_dutch(self):
+ assert detect_structure("Inhoudsopgave\n1 Inleiding")["has_toc"] is True
+
+ def test_toc_spanish_accented(self):
+ assert detect_structure("Índice\n1 Introducción")["has_toc"] is True
+
+ def test_toc_traditional_chinese(self):
+ assert detect_structure("目錄\n第一章")["has_toc"] is True
+
+ def test_toc_italian_sommario(self):
+ assert detect_structure("Sommario\n1 Introduzione")["has_toc"] is True
+
+ def test_toc_inline_word_is_not_toc(self):
+ # "contents"/"index" mid-sentence must not be mistaken for a ToC header
+ text = "The contents of this chapter are varied and the index is long.\n"
+ assert detect_structure(text)["has_toc"] is False
+
def test_numbered_list_items_are_not_chapters(self):
# The AI-Engineering failure: numbered list items were counted as chapters.
text = (
@@ -632,6 +664,44 @@ class TestDetectStructure:
# A bare Arabic-numeral Markdown heading is NOT a chapter (unchanged).
assert detect_structure("## 5 Setup\n## 6 Teardown\n")["chapters_detected"] == 0
+ def test_markdown_atx_chapters(self):
+ text = "# Book Title\n\n## Introduction\nbody\n\n## Getting Started\nbody\n\n## Advanced\nbody\n"
+ assert detect_structure(text)["chapters_detected"] == 3
+
+ def test_markdown_all_h1_chapters(self):
+ text = "# Chapter One\ntext\n# Chapter Two\ntext\n# Chapter Three\ntext\n"
+ assert detect_structure(text)["chapters_detected"] == 3
+
+ def test_asciidoc_section_headings(self):
+ text = "= Doc Title\n\n== First Section\nbody\n\n== Second Section\nbody\n"
+ assert detect_structure(text)["chapters_detected"] == 2
+
+ def test_asciidoc_deeper_levels(self):
+ # AsciiDoc levels 3-6 (=== .. ======) are also recognized.
+ text = "=== Alpha\nbody\n=== Beta\nbody\n=== Gamma\nbody\n"
+ assert detect_structure(text)["chapters_detected"] == 3
+
+ def test_markdown_prefixed_chapter_word(self):
+ # "## Chapter 1:" is not caught by the numeric scan (line starts with '#'),
+ # so the structural fallback must count it.
+ text = "## Chapter 1: Intro\nbody\n## Chapter 2: Models\nbody\n"
+ assert detect_structure(text)["chapters_detected"] == 2
+
+ def test_headings_inside_code_fence_are_ignored(self):
+ text = "# Real A\n\n```python\n# a comment\n# another comment\n```\n\n# Real B\n"
+ assert detect_structure(text)["chapters_detected"] == 2
+
+ def test_plain_prose_has_no_structural_chapters(self):
+ # Regression guard: no headings -> still 0, unchanged behavior
+ text = "Just paragraphs of prose.\nMore prose here.\n"
+ assert detect_structure(text)["chapters_detected"] == 0
+
+ def test_numeric_chapters_win_over_markdown_subsections(self):
+ # A book with real "Chapter N" headings must report the numeric count,
+ # not the count of markdown subsection headings.
+ text = "Chapter 1: Intro\n## sub a\n## sub b\n## sub c\nChapter 2: Next\n"
+ assert detect_structure(text)["chapters_detected"] == 2
+
def test_chinese_numeral_parsing(self):
assert _cn_numeral_to_int("一") == 1
assert _cn_numeral_to_int("十") == 10