Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
fix(extractor): resolve Persian ZWNJ ordinals after sanitization (#287)
Shubham Srivastavaa3ba8dccd587
diff --git a/book_to_skill/utils.py b/book_to_skill/utils.pyindex 18b71b0..1836e70 100644--- a/book_to_skill/utils.py+++ b/book_to_skill/utils.py@@ -405,6 +405,13 @@ def _fa_ordinal_map() -> dict[str, int]:for i, w in enumerate(_FA_COMPOUND_ONES, 1):m[f"بیست و {w}"] = 20 + im[f"سی و {w}"] = 30 + i+ # sanitize_extracted_text() strips U+200C (it is a prompt-injection spacer)+ # before detect_structure() runs, so a key spelled with a ZWNJ can never be+ # reached through the pipeline. Register the stripped spelling of every such+ # key as well, so both the typographic form and what detection actually+ # receives resolve to the same chapter number.+ for key, value in list(m.items()):+ m.setdefault(key.replace("\u200c", ""), value)return mdiff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.pyindex b37fb1a..7bc6a57 100644--- a/tests/test_book_to_skill.py+++ b/tests/test_book_to_skill.py@@ -975,6 +975,33 @@ class TestDetectStructure:# Common ZWNJ spelling of 30.assert _chapter_number("فصل سیام") == 30+ def test_persian_zwnj_ordinal_survives_sanitization(self):+ """The ZWNJ spelling of 30 must still resolve after U+200C is stripped.++ sanitize_extracted_text() removes U+200C (a prompt-injection spacer)+ before detect_structure() runs, so asserting on the raw heading alone+ passes while the real pipeline misses the chapter.+ """+ from book_to_skill.sanitize import sanitize_extracted_text+ from book_to_skill.utils import _chapter_number++ heading = "فصل سیام"+ sanitized, removed = sanitize_extracted_text(heading)+ assert removed == 1+ assert sanitized != heading+ assert _chapter_number(sanitized) == 30++ def test_persian_zwnj_chapter_detected_end_to_end(self):+ """A ZWNJ-spelled Persian heading is counted after sanitization."""+ from book_to_skill.sanitize import sanitize_extracted_text++ text = (+ "فصل بیستم مقدمه\nمتن\n"+ "فصل سیام نتیجه\nمتن"+ )+ sanitized, _ = sanitize_extracted_text(text)+ assert detect_structure(sanitized)["chapters_detected"] == 2+def test_persian_compound_word_numerals(self):"""Explicit compound forms used in longer Persian books."""from book_to_skill.utils import _chapter_number