Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

fix(extractor): resolve Persian ZWNJ ordinals after sanitization (#287)

Shubham Srivastavaa3ba8dccd587
diff --git a/book_to_skill/utils.py b/book_to_skill/utils.py
index 18b71b0..1836e70 100644
--- a/book_to_skill/utils.py
+++ b/book_to_skill/utils.py
@@ -405,6 +405,13 @@ def _fa_ordinal_map() -> dict[str, int]:
for i, w in enumerate(_FA_COMPOUND_ONES, 1):
m[f"بیست و {w}"] = 20 + i
m[f"سی و {w}"] = 30 + i
+ # sanitize_extracted_text() strips U+200C (it is a prompt-injection spacer)
+ # before detect_structure() runs, so a key spelled with a ZWNJ can never be
+ # reached through the pipeline. Register the stripped spelling of every such
+ # key as well, so both the typographic form and what detection actually
+ # receives resolve to the same chapter number.
+ for key, value in list(m.items()):
+ m.setdefault(key.replace("\u200c", ""), value)
return m
diff --git a/tests/test_book_to_skill.py b/tests/test_book_to_skill.py
index b37fb1a..7bc6a57 100644
--- a/tests/test_book_to_skill.py
+++ b/tests/test_book_to_skill.py
@@ -975,6 +975,33 @@ class TestDetectStructure:
# Common ZWNJ spelling of 30.
assert _chapter_number("فصل سی‌ام") == 30
+ def test_persian_zwnj_ordinal_survives_sanitization(self):
+ """The ZWNJ spelling of 30 must still resolve after U+200C is stripped.
+
+ sanitize_extracted_text() removes U+200C (a prompt-injection spacer)
+ before detect_structure() runs, so asserting on the raw heading alone
+ passes while the real pipeline misses the chapter.
+ """
+ from book_to_skill.sanitize import sanitize_extracted_text
+ from book_to_skill.utils import _chapter_number
+
+ heading = "فصل سی‌ام"
+ sanitized, removed = sanitize_extracted_text(heading)
+ assert removed == 1
+ assert sanitized != heading
+ assert _chapter_number(sanitized) == 30
+
+ def test_persian_zwnj_chapter_detected_end_to_end(self):
+ """A ZWNJ-spelled Persian heading is counted after sanitization."""
+ from book_to_skill.sanitize import sanitize_extracted_text
+
+ text = (
+ "فصل بیستم مقدمه\nمتن\n"
+ "فصل سی‌ام نتیجه\nمتن"
+ )
+ sanitized, _ = sanitize_extracted_text(text)
+ assert detect_structure(sanitized)["chapters_detected"] == 2
+
def test_persian_compound_word_numerals(self):
"""Explicit compound forms used in longer Persian books."""
from book_to_skill.utils import _chapter_number