Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.
Python agent-skillsai-agentsbook-to-skillcontext-engineeringdocument-processingedtechknowledge-baseknowledge-managementllmpdf-to-markdownragself-studystudy-tools
fix(extract): count supplementary-plane CJK in the token estimate (#136)
Hotragn Pettuganib5239b709725
diff --git a/book_to_skill/utils.py b/book_to_skill/utils.pyindex fa3cd5d..56d355f 100644--- a/book_to_skill/utils.py+++ b/book_to_skill/utils.py@@ -52,9 +52,21 @@ from book_to_skill.sanitize import sanitize_extracted_text# CJK codepoints: ideographs + extensions, kana, hangul, CJK punctuation, and# fullwidth forms. These are not whitespace-delimited, so counting "words" on a# Chinese/Japanese book collapses it to a handful of tokens; count them directly.+#+# The last range is Planes 2 and 3 (U+20000-U+3FFFF), the ideographic+# supplementary planes, taken end to end rather than enumerated block by block+# so a future extension does not silently fall through the way Extension H+# (U+31350-U+323AF) did. Nothing non-ideographic lives up here: emoji,+# mathematical alphanumerics and regional indicators are all in Plane 1, which+# this range does not touch. Classical Chinese, Cantonese, Hong Kong and+# Taiwan place/personal names, and Japanese 人名用漢字 all draw on it. Without it+# those characters fell through to the whitespace-word branch, where a+# space-less run of them counts as a single "word": the same ~1000x undercount+# #103 fixed for the BMP, one plane up._CJK_RE = re.compile(r"[ -〿-ヿ㐀-䶿一-鿿"- r"가-힣豈--]"+ r"가-힣豈--"+ r"\U00020000-\U0003FFFF]")diff --git a/tests/test_cjk_supplementary_plane.py b/tests/test_cjk_supplementary_plane.pynew file mode 100644index 0000000..4d073f7--- /dev/null+++ b/tests/test_cjk_supplementary_plane.py@@ -0,0 +1,130 @@+"""`estimate_tokens` must count supplementary-plane CJK, not just the BMP.++#103 made the estimate CJK-aware because ideographs are not whitespace-delimited:+without it a space-less Chinese or Japanese book collapses to a handful of+"words" and the cost pre-flight under-reports by ~1000x.++`_CJK_RE` only covered the Basic Multilingual Plane, so Unified Ideographs+Extension B and later (U+20000 and up) fell straight through to the+whitespace-word branch and hit exactly the undercount #103 set out to fix — one+plane up. Those extensions carry classical Chinese, Cantonese, Hong Kong and+Taiwan place and personal names, and Japanese 人名用漢字.+"""++import sys+from pathlib import Path++import pytest++ROOT_DIR = Path(__file__).resolve().parent.parent+sys.path.insert(0, str(ROOT_DIR))++from book_to_skill.config import CJK_CHARS_PER_TOKEN+from book_to_skill.utils import estimate_tokens++BMP = "一二三四五六七八九十" # common ideographs+SIP = "\U00020000\U00020001\U0002a700\U0002b740\U0002ceb0" # Ext B / C / D / E+++class TestSupplementaryPlaneCounted:+ def test_sip_matches_bmp_for_the_same_length(self):+ bmp_text = BMP * 200+ sip_text = SIP * 400 # same character count++ assert len(bmp_text) == len(sip_text)+ assert estimate_tokens(sip_text) == estimate_tokens(bmp_text)++ def test_sip_only_text_is_not_one_token(self):+ text = SIP * 400++ # The old behaviour: a space-less run counts as a single "word".+ assert estimate_tokens(text) > 1000++ def test_estimate_tracks_the_configured_ratio(self):+ text = SIP * 400++ assert estimate_tokens(text) == pytest.approx(+ len(text) / CJK_CHARS_PER_TOKEN, rel=0.01+ )++ def test_mixed_plane_text_is_consistent(self):+ """A book mixing common and rare ideographs estimates evenly."""+ mixed = (BMP * 180) + (SIP * 40)+ all_bmp = BMP * 200++ assert len(mixed) == len(all_bmp)+ assert estimate_tokens(mixed) == estimate_tokens(all_bmp)++ @pytest.mark.parametrize(+ "codepoint, name",+ [+ (0x20000, "Ext B start"),+ (0x2A6DF, "Ext B end"),+ (0x2A700, "Ext C start"),+ (0x2B740, "Ext D start"),+ (0x2CEB0, "Ext E start"),+ (0x2EBF0, "Ext I start"),+ (0x30000, "Ext G start"),+ (0x3134E, "Ext G end"),+ # Extension H sits ABOVE Extension G, so a range that stopped at the+ # end of G let 4,192 assigned ideographs fall through. Both ends are+ # probed so the plane boundary cannot regress to a block boundary.+ (0x31350, "Ext H start"),+ (0x323AF, "Ext H end"),+ ],+ )+ def test_extension_ranges_are_covered(self, codepoint, name):+ text = chr(codepoint) * 300++ assert estimate_tokens(text) > 100, name+++class TestExistingBehaviourPreserved:+ """The #103 behaviour for the BMP and for Latin must not change."""++ def test_bmp_cjk_unchanged(self):+ text = BMP * 200++ assert estimate_tokens(text) == pytest.approx(+ len(text) / CJK_CHARS_PER_TOKEN, rel=0.01+ )++ @pytest.mark.parametrize(+ "sample",+ ["第一章 緒論", "제1장 총칙", "こんにちは世界", "你好世界"],+ )+ def test_short_cjk_samples_still_counted(self, sample):+ assert estimate_tokens(sample) >= 1++ def test_latin_text_unaffected(self):+ text = "the quick brown fox jumps over the lazy dog " * 100++ # Pure Latin takes the word branch; no CJK found.+ assert estimate_tokens(text) == int(len(text.split()) / 0.75)++ def test_empty_text(self):+ assert estimate_tokens("") == 0++ def test_latin_with_a_single_sip_character(self):+ """One rare ideograph must not flip Latin onto a wrong scale."""+ text = ("word " * 100) + "\U00020000"++ # 100 Latin words plus one CJK char: dominated by the Latin branch.+ assert 120 < estimate_tokens(text) < 145+++class TestNonCjkSupplementaryPlanesExcluded:+ """Emoji and other astral characters are not ideographs."""++ @pytest.mark.parametrize(+ "char, name",+ [+ ("\U0001F600", "emoji"),+ ("\U0001D400", "math bold capital A"),+ ("\U0001F1E6", "regional indicator"),+ ],+ )+ def test_astral_non_cjk_takes_the_word_branch(self, char, name):+ # A space-less run of these is one "word", as before — they are not+ # CJK and must not be pulled into the ideograph branch.+ assert estimate_tokens(char * 300) == 1, name