Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
feat: improve Persian chapter detection (#139)
Hamed Farazia08859f69706
diff --git a/book_to_skill/utils.py b/book_to_skill/utils.pyindex efeca4f..e0674cf 100644--- a/book_to_skill/utils.py+++ b/book_to_skill/utils.py@@ -173,6 +173,88 @@ _KO_CHAPTER = re.compile(r"^\s*(?:#{1,6}\s+)?제\s*([0-9]+)\s*[장편절관](?:\s*의\s*[0-9]+)?(?:\s*$|[.:\-]|\s+\S)")+# Persian chapter headings: "فصل ۱", "فصل اول", "بخش ۲: مفاهیم",+# "فصل بیست و یکم", "فصل سی و چهارمخداحافظ…" (PDF glue on long forms).+# Labels are فصل / بخش. Digits may be ASCII, Persian (U+06F0–U+06F9), or+# Arabic-Indic (U+0660–U+0669); int() parses all three. Word numerals use a+# small ordinal map (1–34) with longest-prefix matching so compounds+# ("بیست و یکم") and teens ("یازدهم") stay maintainable. Markdown "#" prefixes+# are handled by `_chapter_number`'s second pass (Issue #91).+#+# Trailing rules (Persian has no letter case for a Latin-style `_HEADING_TAIL`):+# - digits: EOL / punctuation / spaced title (Korean-style);+# - short word ordinals 1–10: require a separator (space, punct, ZWNJ, or EOL)+# so "فصل اولویتها" / "فصل اولیه" are not read as chapter 1;+# - teens and compounds: also allow a glued title letter — PDF extractors+# often drop the space, and a long ordinal is not a plausible word prefix.+_FA_DIGITS = "۰-۹٠-٩" # Persian then Arabic-Indic+_FA_ONES = (+ "اول", "دوم", "سوم", "چهارم", "پنجم", "ششم", "هفتم", "هشتم", "نهم", "دهم",+)+# Ones used after "بیست و" / "سی و" (یکم, not اول).+_FA_COMPOUND_ONES = (+ "یکم", "دوم", "سوم", "چهارم", "پنجم", "ششم", "هفتم", "هشتم", "نهم",+)+_FA_TEENS = (+ "یازدهم", "دوازدهم", "سیزدهم", "چهاردهم", "پانزدهم",+ "شانزدهم", "هفدهم", "هجدهم", "نوزدهم",+)+_FA_ONES_SET = frozenset(_FA_ONES)+# After a short (1–10) word ordinal: end, whitespace, punctuation, or ZWNJ.+_FA_SHORT_ORDINAL_TAIL = re.compile(r"^(?:$|\s|[.:\-—–:]|\u200c)")+++def _fa_ordinal_map() -> dict[str, int]:+ """Persian chapter ordinals 1–34, including common spelling variants."""+ m: dict[str, int] = {}+ for i, w in enumerate(_FA_ONES, 1):+ m[w] = i+ for i, w in enumerate(_FA_TEENS, 11):+ m[w] = i+ m["هیجدهم"] = 18 # common alternate spelling of هجدهم+ # Fused "بیستم" only — "بیست ام" / "بیستام" are not common spellings+ # (unlike "سی ام" / "سیام" for 30), so they stay unmapped on purpose.+ m["بیستم"] = 20+ m["سی ام"] = 30+ m["سیام"] = 30 # ZWNJ spelling common in Persian typography+ for i, w in enumerate(_FA_COMPOUND_ONES, 1):+ m[f"بیست و {w}"] = 20 + i+ m[f"سی و {w}"] = 30 + i+ return m+++_FA_ORDINALS = _fa_ordinal_map()+# Longest first so "چهاردهم" wins over "چهارم", "بیست و یکم" over nothing shorter.+_FA_ORDINAL_KEYS = sorted(_FA_ORDINALS, key=len, reverse=True)+_FA_LABEL_REST = re.compile(r"^\s*(?:فصل|بخش)\s+(.*)$")+_FA_DIGIT_HEAD = re.compile(rf"^([0-9{_FA_DIGITS}]+)(.*)$")+# Digit form: same idea as the Korean trailing guard (no Latin case to lean on).+_FA_DIGIT_TAIL = re.compile(r"^(?:\s*$|[.:\-—–:]|\s+\S)")+++def _fa_chapter_number(s: str) -> int | None:+ """Return a Persian chapter number (1–99 digits / 1–34 words) or None."""+ m = _FA_LABEL_REST.match(s)+ if not m:+ return None+ rest = m.group(1)+ dm = _FA_DIGIT_HEAD.match(rest)+ if dm:+ n = int(dm.group(1))+ if 1 <= n <= 99 and _FA_DIGIT_TAIL.match(dm.group(2)) is not None:+ return n+ return None+ for key in _FA_ORDINAL_KEYS:+ if not rest.startswith(key):+ continue+ tail = rest[len(key):]+ # Short 1–10 ordinals need a separator; teens/compounds may be PDF-glued.+ if key in _FA_ONES_SET and _FA_SHORT_ORDINAL_TAIL.match(tail) is None:+ return None+ return _FA_ORDINALS[key]+ return None++# Table-of-contents header lines across common languages. Anchored to a whole# line (^\s*X\s*$) so an inline "the contents of this chapter" never matches._TOC_HEADERS = (@@ -411,6 +493,9 @@ def _match_chapter_number(line: str) -> int | None:km = _KO_CHAPTER.match(s)if km:return int(km.group(1))+ fa = _fa_chapter_number(s)+ if fa is not None:+ return fareturn None@@ -420,9 +505,11 @@ def _chapter_number(line: str) -> int | None:Handles Arabic ("Chapter 5", "Capítulo 5: ..."), Roman-numeral("I: Loomings", "## i. introduction", "II. The Carpet-Bag"),Chinese ("第三章 …", "## 一 · …", "## 第一讲"), Thai ("บทที่ 3",- "## บทที่ ๑"), and Korean ("제1장 총칙", "## 제4장 근로시간과 휴식")- heading styles — each optionally preceded by a Markdown/AsciiDoc heading- marker ("## Chapter 1" is a chapter heading just like "Chapter 1").+ "## บทที่ ๑"), Korean ("제1장 총칙", "## 제4장 근로시간과 휴식"), and+ Persian ("فصل ۱", "فصل اول", "فصل بیست و یکم", "بخش ۲: مفاهیم",+ "## فصل ۱: مقدمه", PDF-glued "فصل سی و چهارمخداحافظ…") heading styles — each+ optionally preceded by a Markdown/AsciiDoc heading marker+ ("## Chapter 1" is a chapter heading just like "Chapter 1")."""match = _match_chapter_number(line)if match is not None:diff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.pyindex 739b10f..b985f16 100644--- a/tests/test_book_to_skill.py+++ b/tests/test_book_to_skill.py@@ -672,6 +672,145 @@ class TestDetectStructure:text = "제1편 총칙\n내용\n제2장 정의\n내용\n제3절 통칙\n내용"assert detect_structure(text)["chapters_detected"] == 3+ # ── Persian chapter headings ───────────────────────────────────────────++ # Canonical ordinals 1–34 used by the FA word-numeral map (integration fixture).+ _FA_ORDINAL_1_TO_34 = (+ "اول", "دوم", "سوم", "چهارم", "پنجم", "ششم", "هفتم", "هشتم", "نهم", "دهم",+ "یازدهم", "دوازدهم", "سیزدهم", "چهاردهم", "پانزدهم", "شانزدهم", "هفدهم",+ "هجدهم", "نوزدهم", "بیستم",+ "بیست و یکم", "بیست و دوم", "بیست و سوم", "بیست و چهارم", "بیست و پنجم",+ "بیست و ششم", "بیست و هفتم", "بیست و هشتم", "بیست و نهم", "سی ام",+ "سی و یکم", "سی و دوم", "سی و سوم", "سی و چهارم",+ )++ def test_persian_digit_scripts(self):+ """`فصل N` with Persian, Arabic-Indic, and ASCII digits."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("فصل ۱") == 1+ assert _chapter_number("فصل ١") == 1+ assert _chapter_number("فصل 1") == 1+ assert _chapter_number("فصل ۱۰") == 10+ assert _chapter_number("فصل ١٠") == 10+ assert _chapter_number("فصل 10") == 10+ assert _chapter_number("فصل ۳۴") == 34++ def test_persian_word_numerals_1_to_34(self):+ """Word ordinals `اول` … `سی و چهارم` map to integers 1–34."""+ from book_to_skill.utils import _chapter_number++ for n, word in enumerate(self._FA_ORDINAL_1_TO_34, 1):+ assert _chapter_number(f"فصل {word}") == n, word+ # Common ZWNJ spelling of 30.+ assert _chapter_number("فصل سیام") == 30++ def test_persian_compound_word_numerals(self):+ """Explicit compound forms used in longer Persian books."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("فصل بیست و یکم") == 21+ assert _chapter_number("فصل بیست و نهم") == 29+ assert _chapter_number("فصل سی و یکم") == 31+ assert _chapter_number("فصل سی و چهارم") == 34++ def test_persian_hejdahom_spelling_variants(self):+ """Both common spellings of 18: هجدهم and هیجدهم."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("فصل هجدهم") == 18+ assert _chapter_number("فصل هجدهم: یک جاسوس") == 18+ assert _chapter_number("فصل هیجدهم") == 18+ assert _chapter_number("فصل هیجدهم: یک جاسوس") == 18++ def test_persian_bakhsh_section(self):+ """`بخش` (section/part) is accepted with digits or word numerals."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("بخش ۲") == 2+ assert _chapter_number("بخش ٢") == 2+ assert _chapter_number("بخش 2") == 2+ assert _chapter_number("بخش دوم") == 2+ assert _chapter_number("بخش سی و چهارم") == 34++ def test_persian_titled_headings(self):+ """Punctuation / dash / spaced titles after the numeral are headings."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("فصل ۱: مقدمه") == 1+ assert _chapter_number("فصل اول — مبانی برنامهنویسی") == 1+ assert _chapter_number("فصل ۲. اصول") == 2+ assert _chapter_number("بخش ۳: مفاهیم") == 3+ assert _chapter_number("فصل بیست و یکم پایان سفر") == 21++ def test_persian_markdown_prefix(self):+ """Markdown heading prefixes are stripped by `_chapter_number`."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("## فصل ۱: مقدمه") == 1+ assert _chapter_number("### فصل دوم") == 2+ assert _chapter_number("### فصل سی و چهارم خداحافظ فرانسه") == 34++ def test_persian_pdf_glued_title(self):+ """PDF glue is allowed after teens/compounds, not after short 1–10 ordinals.++ Short ordinals are plausible prefixes of ordinary Persian words+ ("اولویتها", "اولیه", "دومینو"), so they require a separator. Longer+ forms are not, and extractors do drop the space after them.+ """+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("فصل سی و چهارمخداحافظ، فرانسه") == 34+ assert _chapter_number("فصل بیست و یکمپایان سفر") == 21+ assert _chapter_number("فصل هجدهمیک جاسوس") == 18+ assert _chapter_number("فصل هیجدهمیک جاسوس") == 18+ # Short 1–10 glued titles are rejected (see false-positive test below).+ assert _chapter_number("فصل اولجایی که به نظر میرسید...") is None+ assert _chapter_number("فصل دومشهادت یک جنایتکار علیه خودش") is None+ assert _chapter_number("فصل سومعدالت") is None++ def test_persian_short_ordinal_false_positives(self):+ """Ordinary phrases that begin with a short ordinal must not be chapters."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("فصل اولویتها") is None+ assert _chapter_number("فصل اولیه") is None+ assert _chapter_number("فصل دومینو") is None+ assert _chapter_number("فصل سومین") is None++ def test_persian_prose_is_not_a_chapter_heading(self):+ """Inline / incomplete `فصل` references must not count as headings."""+ from book_to_skill.utils import _chapter_number++ assert _chapter_number("در فصل ۲ این موضوع را بررسی میکنیم") is None+ assert _chapter_number("در فصل دوم این موضوع را بررسی میکنیم") is None+ assert _chapter_number("این فصل اول یک توضیح است") is None+ assert _chapter_number("فصل") is None+ assert _chapter_number("بخش") is None+ # Incomplete compounds are not headings.+ assert _chapter_number("فصل بیست") is None+ assert _chapter_number("فصل سی و") is None+ # Existing hard length guard in `_match_chapter_number`.+ assert _chapter_number("فصل ۱: " + ("الف" * 40)) is None++ def test_detects_persian_chapters(self):+ """Plain-text Persian headings are numeric chapters, not MD fallback."""+ text = "فصل ۱\nمحتوا\nفصل ۲\nمحتوا\nفصل ۳\nمحتوا"+ result = detect_structure(text)+ assert result["chapters_detected"] == 3+ # Non-empty sample proves the numeric path, not structural Markdown.+ assert result["chapter_headings_sample"] == ["فصل ۱", "فصل ۲", "فصل ۳"]++ def test_detects_persian_word_chapters_1_to_34(self):+ """All 34 word-numeral headings count as distinct numeric chapters."""+ text = "\n".join(+ f"فصل {word}\nمحتوا فصل {n}."+ for n, word in enumerate(self._FA_ORDINAL_1_TO_34, 1)+ )+ result = detect_structure(text)+ assert result["chapters_detected"] == 34+ assert result["chapter_headings_sample"] # numeric path, not MD fallback+ assert result["chapter_headings_sample"][0] == "فصل اول"def test_roman_footnote_reference_is_not_a_chapter(self):"""Scholarly cross-references must stay rejected after the Roman change."""