Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

feat: improve Persian chapter detection (#139)

Hamed Farazia08859f69706
diff --git a/book_to_skill/utils.py b/book_to_skill/utils.py
index efeca4f..e0674cf 100644
--- a/book_to_skill/utils.py
+++ b/book_to_skill/utils.py
@@ -173,6 +173,88 @@ _KO_CHAPTER = re.compile(
r"^\s*(?:#{1,6}\s+)?제\s*([0-9]+)\s*[장편절관](?:\s*의\s*[0-9]+)?(?:\s*$|[.:\-]|\s+\S)"
)
+# Persian chapter headings: "فصل ۱", "فصل اول", "بخش ۲: مفاهیم",
+# "فصل بیست و یکم", "فصل سی و چهارمخداحافظ…" (PDF glue on long forms).
+# Labels are فصل / بخش. Digits may be ASCII, Persian (U+06F0–U+06F9), or
+# Arabic-Indic (U+0660–U+0669); int() parses all three. Word numerals use a
+# small ordinal map (1–34) with longest-prefix matching so compounds
+# ("بیست و یکم") and teens ("یازدهم") stay maintainable. Markdown "#" prefixes
+# are handled by `_chapter_number`'s second pass (Issue #91).
+#
+# Trailing rules (Persian has no letter case for a Latin-style `_HEADING_TAIL`):
+# - digits: EOL / punctuation / spaced title (Korean-style);
+# - short word ordinals 1–10: require a separator (space, punct, ZWNJ, or EOL)
+# so "فصل اولویت‌ها" / "فصل اولیه" are not read as chapter 1;
+# - teens and compounds: also allow a glued title letter — PDF extractors
+# often drop the space, and a long ordinal is not a plausible word prefix.
+_FA_DIGITS = "۰-۹٠-٩" # Persian then Arabic-Indic
+_FA_ONES = (
+ "اول", "دوم", "سوم", "چهارم", "پنجم", "ششم", "هفتم", "هشتم", "نهم", "دهم",
+)
+# Ones used after "بیست و" / "سی و" (یکم, not اول).
+_FA_COMPOUND_ONES = (
+ "یکم", "دوم", "سوم", "چهارم", "پنجم", "ششم", "هفتم", "هشتم", "نهم",
+)
+_FA_TEENS = (
+ "یازدهم", "دوازدهم", "سیزدهم", "چهاردهم", "پانزدهم",
+ "شانزدهم", "هفدهم", "هجدهم", "نوزدهم",
+)
+_FA_ONES_SET = frozenset(_FA_ONES)
+# After a short (1–10) word ordinal: end, whitespace, punctuation, or ZWNJ.
+_FA_SHORT_ORDINAL_TAIL = re.compile(r"^(?:$|\s|[.:\-—–:]|\u200c)")
+
+
+def _fa_ordinal_map() -> dict[str, int]:
+ """Persian chapter ordinals 1–34, including common spelling variants."""
+ m: dict[str, int] = {}
+ for i, w in enumerate(_FA_ONES, 1):
+ m[w] = i
+ for i, w in enumerate(_FA_TEENS, 11):
+ m[w] = i
+ m["هیجدهم"] = 18 # common alternate spelling of هجدهم
+ # Fused "بیستم" only — "بیست ام" / "بیست‌ام" are not common spellings
+ # (unlike "سی ام" / "سی‌ام" for 30), so they stay unmapped on purpose.
+ m["بیستم"] = 20
+ m["سی ام"] = 30
+ m["سی‌ام"] = 30 # ZWNJ spelling common in Persian typography
+ for i, w in enumerate(_FA_COMPOUND_ONES, 1):
+ m[f"بیست و {w}"] = 20 + i
+ m[f"سی و {w}"] = 30 + i
+ return m
+
+
+_FA_ORDINALS = _fa_ordinal_map()
+# Longest first so "چهاردهم" wins over "چهارم", "بیست و یکم" over nothing shorter.
+_FA_ORDINAL_KEYS = sorted(_FA_ORDINALS, key=len, reverse=True)
+_FA_LABEL_REST = re.compile(r"^\s*(?:فصل|بخش)\s+(.*)$")
+_FA_DIGIT_HEAD = re.compile(rf"^([0-9{_FA_DIGITS}]+)(.*)$")
+# Digit form: same idea as the Korean trailing guard (no Latin case to lean on).
+_FA_DIGIT_TAIL = re.compile(r"^(?:\s*$|[.:\-—–:]|\s+\S)")
+
+
+def _fa_chapter_number(s: str) -> int | None:
+ """Return a Persian chapter number (1–99 digits / 1–34 words) or None."""
+ m = _FA_LABEL_REST.match(s)
+ if not m:
+ return None
+ rest = m.group(1)
+ dm = _FA_DIGIT_HEAD.match(rest)
+ if dm:
+ n = int(dm.group(1))
+ if 1 <= n <= 99 and _FA_DIGIT_TAIL.match(dm.group(2)) is not None:
+ return n
+ return None
+ for key in _FA_ORDINAL_KEYS:
+ if not rest.startswith(key):
+ continue
+ tail = rest[len(key):]
+ # Short 1–10 ordinals need a separator; teens/compounds may be PDF-glued.
+ if key in _FA_ONES_SET and _FA_SHORT_ORDINAL_TAIL.match(tail) is None:
+ return None
+ return _FA_ORDINALS[key]
+ return None
+
+
# Table-of-contents header lines across common languages. Anchored to a whole
# line (^\s*X\s*$) so an inline "the contents of this chapter" never matches.
_TOC_HEADERS = (
@@ -411,6 +493,9 @@ def _match_chapter_number(line: str) -> int | None:
km = _KO_CHAPTER.match(s)
if km:
return int(km.group(1))
+ fa = _fa_chapter_number(s)
+ if fa is not None:
+ return fa
return None
@@ -420,9 +505,11 @@ def _chapter_number(line: str) -> int | None:
Handles Arabic ("Chapter 5", "Capítulo 5: ..."), Roman-numeral
("I: Loomings", "## i. introduction", "II. The Carpet-Bag"),
Chinese ("第三章 …", "## 一 · …", "## 第一讲"), Thai ("บทที่ 3",
- "## บทที่ ๑"), and Korean ("제1장 총칙", "## 제4장 근로시간과 휴식")
- heading styles — each optionally preceded by a Markdown/AsciiDoc heading
- marker ("## Chapter 1" is a chapter heading just like "Chapter 1").
+ "## บทที่ ๑"), Korean ("제1장 총칙", "## 제4장 근로시간과 휴식"), and
+ Persian ("فصل ۱", "فصل اول", "فصل بیست و یکم", "بخش ۲: مفاهیم",
+ "## فصل ۱: مقدمه", PDF-glued "فصل سی و چهارمخداحافظ…") heading styles — each
+ optionally preceded by a Markdown/AsciiDoc heading marker
+ ("## Chapter 1" is a chapter heading just like "Chapter 1").
"""
match = _match_chapter_number(line)
if match is not None:
diff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.py
index 739b10f..b985f16 100644
--- a/tests/test_book_to_skill.py
+++ b/tests/test_book_to_skill.py
@@ -672,6 +672,145 @@ class TestDetectStructure:
text = "제1편 총칙\n내용\n제2장 정의\n내용\n제3절 통칙\n내용"
assert detect_structure(text)["chapters_detected"] == 3
+ # ── Persian chapter headings ───────────────────────────────────────────
+
+ # Canonical ordinals 1–34 used by the FA word-numeral map (integration fixture).
+ _FA_ORDINAL_1_TO_34 = (
+ "اول", "دوم", "سوم", "چهارم", "پنجم", "ششم", "هفتم", "هشتم", "نهم", "دهم",
+ "یازدهم", "دوازدهم", "سیزدهم", "چهاردهم", "پانزدهم", "شانزدهم", "هفدهم",
+ "هجدهم", "نوزدهم", "بیستم",
+ "بیست و یکم", "بیست و دوم", "بیست و سوم", "بیست و چهارم", "بیست و پنجم",
+ "بیست و ششم", "بیست و هفتم", "بیست و هشتم", "بیست و نهم", "سی ام",
+ "سی و یکم", "سی و دوم", "سی و سوم", "سی و چهارم",
+ )
+
+ def test_persian_digit_scripts(self):
+ """`فصل N` with Persian, Arabic-Indic, and ASCII digits."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("فصل ۱") == 1
+ assert _chapter_number("فصل ١") == 1
+ assert _chapter_number("فصل 1") == 1
+ assert _chapter_number("فصل ۱۰") == 10
+ assert _chapter_number("فصل ١٠") == 10
+ assert _chapter_number("فصل 10") == 10
+ assert _chapter_number("فصل ۳۴") == 34
+
+ def test_persian_word_numerals_1_to_34(self):
+ """Word ordinals `اول` … `سی و چهارم` map to integers 1–34."""
+ from book_to_skill.utils import _chapter_number
+
+ for n, word in enumerate(self._FA_ORDINAL_1_TO_34, 1):
+ assert _chapter_number(f"فصل {word}") == n, word
+ # Common ZWNJ spelling of 30.
+ assert _chapter_number("فصل سی‌ام") == 30
+
+ def test_persian_compound_word_numerals(self):
+ """Explicit compound forms used in longer Persian books."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("فصل بیست و یکم") == 21
+ assert _chapter_number("فصل بیست و نهم") == 29
+ assert _chapter_number("فصل سی و یکم") == 31
+ assert _chapter_number("فصل سی و چهارم") == 34
+
+ def test_persian_hejdahom_spelling_variants(self):
+ """Both common spellings of 18: هجدهم and هیجدهم."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("فصل هجدهم") == 18
+ assert _chapter_number("فصل هجدهم: یک جاسوس") == 18
+ assert _chapter_number("فصل هیجدهم") == 18
+ assert _chapter_number("فصل هیجدهم: یک جاسوس") == 18
+
+ def test_persian_bakhsh_section(self):
+ """`بخش` (section/part) is accepted with digits or word numerals."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("بخش ۲") == 2
+ assert _chapter_number("بخش ٢") == 2
+ assert _chapter_number("بخش 2") == 2
+ assert _chapter_number("بخش دوم") == 2
+ assert _chapter_number("بخش سی و چهارم") == 34
+
+ def test_persian_titled_headings(self):
+ """Punctuation / dash / spaced titles after the numeral are headings."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("فصل ۱: مقدمه") == 1
+ assert _chapter_number("فصل اول — مبانی برنامه‌نویسی") == 1
+ assert _chapter_number("فصل ۲. اصول") == 2
+ assert _chapter_number("بخش ۳: مفاهیم") == 3
+ assert _chapter_number("فصل بیست و یکم پایان سفر") == 21
+
+ def test_persian_markdown_prefix(self):
+ """Markdown heading prefixes are stripped by `_chapter_number`."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("## فصل ۱: مقدمه") == 1
+ assert _chapter_number("### فصل دوم") == 2
+ assert _chapter_number("### فصل سی و چهارم خداحافظ فرانسه") == 34
+
+ def test_persian_pdf_glued_title(self):
+ """PDF glue is allowed after teens/compounds, not after short 1–10 ordinals.
+
+ Short ordinals are plausible prefixes of ordinary Persian words
+ ("اولویت‌ها", "اولیه", "دومینو"), so they require a separator. Longer
+ forms are not, and extractors do drop the space after them.
+ """
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("فصل سی و چهارمخداحافظ، فرانسه") == 34
+ assert _chapter_number("فصل بیست و یکمپایان سفر") == 21
+ assert _chapter_number("فصل هجدهمیک جاسوس") == 18
+ assert _chapter_number("فصل هیجدهمیک جاسوس") == 18
+ # Short 1–10 glued titles are rejected (see false-positive test below).
+ assert _chapter_number("فصل اولجایی که به نظر میرسید...") is None
+ assert _chapter_number("فصل دومشهادت یک جنایتکار علیه خودش") is None
+ assert _chapter_number("فصل سومعدالت") is None
+
+ def test_persian_short_ordinal_false_positives(self):
+ """Ordinary phrases that begin with a short ordinal must not be chapters."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("فصل اولویت‌ها") is None
+ assert _chapter_number("فصل اولیه") is None
+ assert _chapter_number("فصل دومینو") is None
+ assert _chapter_number("فصل سومین") is None
+
+ def test_persian_prose_is_not_a_chapter_heading(self):
+ """Inline / incomplete `فصل` references must not count as headings."""
+ from book_to_skill.utils import _chapter_number
+
+ assert _chapter_number("در فصل ۲ این موضوع را بررسی می‌کنیم") is None
+ assert _chapter_number("در فصل دوم این موضوع را بررسی می‌کنیم") is None
+ assert _chapter_number("این فصل اول یک توضیح است") is None
+ assert _chapter_number("فصل") is None
+ assert _chapter_number("بخش") is None
+ # Incomplete compounds are not headings.
+ assert _chapter_number("فصل بیست") is None
+ assert _chapter_number("فصل سی و") is None
+ # Existing hard length guard in `_match_chapter_number`.
+ assert _chapter_number("فصل ۱: " + ("الف" * 40)) is None
+
+ def test_detects_persian_chapters(self):
+ """Plain-text Persian headings are numeric chapters, not MD fallback."""
+ text = "فصل ۱\nمحتوا\nفصل ۲\nمحتوا\nفصل ۳\nمحتوا"
+ result = detect_structure(text)
+ assert result["chapters_detected"] == 3
+ # Non-empty sample proves the numeric path, not structural Markdown.
+ assert result["chapter_headings_sample"] == ["فصل ۱", "فصل ۲", "فصل ۳"]
+
+ def test_detects_persian_word_chapters_1_to_34(self):
+ """All 34 word-numeral headings count as distinct numeric chapters."""
+ text = "\n".join(
+ f"فصل {word}\nمحتوا فصل {n}."
+ for n, word in enumerate(self._FA_ORDINAL_1_TO_34, 1)
+ )
+ result = detect_structure(text)
+ assert result["chapters_detected"] == 34
+ assert result["chapter_headings_sample"] # numeric path, not MD fallback
+ assert result["chapter_headings_sample"][0] == "فصل اول"
def test_roman_footnote_reference_is_not_a_chapter(self):
"""Scholarly cross-references must stay rejected after the Roman change."""