Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): price CJK/Kana/Hangul at ~1 token per char in EstimatingTokenCounter (#1093)
Leoya35fe86e8772
diff --git a/headroom/tokenizers/estimator.py b/headroom/tokenizers/estimator.pyindex 89fb0d6d1..a28fc12d6 100644--- a/headroom/tokenizers/estimator.py+++ b/headroom/tokenizers/estimator.py@@ -30,6 +30,8 @@ class EstimatingTokenCounter(BaseTokenizer):- Base: ~4 characters per token (calibrated against GPT-4)- Adjustments for code, URLs, numbers, whitespace- Special handling for JSON structure+ - CJK / Kana / Hangul characters priced at ~1 token each (these scripts+ tokenize far denser than Latin text)Example:counter = EstimatingTokenCounter()@@ -41,6 +43,13 @@ class EstimatingTokenCounter(BaseTokenizer):CHARS_PER_TOKEN = 4.0 # Average for English textCHARS_PER_TOKEN_CODE = 3.5 # Code is denserCHARS_PER_TOKEN_JSON = 3.2 # JSON has more structure+ # CJK / Kana / Hangul scripts tokenize at roughly 0.6-1.7 tokens *per+ # character* (cl100k_base ~1.0-1.7, DeepSeek/Qwen native ~0.6-0.8), versus+ # ~0.25 tokens/char for English. A flat 4.0 ratio under-counts them ~4-6x,+ # so dense-script codepoints are priced separately. 1.5 chars/token keeps+ # the estimate on the conservative (slight-overestimate) side for native+ # CJK tokenizers while staying close for cl100k_base.+ CHARS_PER_TOKEN_CJK = 1.5# Patterns for content type detectionCODE_PATTERN = re.compile(@@ -50,6 +59,21 @@ class EstimatingTokenCounter(BaseTokenizer):re.MULTILINE,)JSON_PATTERN = re.compile(r"^\s*[\[\{]")+ # Dense scripts where one character is worth roughly one token: CJK+ # symbols/punctuation, Hiragana/Katakana, CJK Unified (+ Ext A), Hangul,+ # CJK compatibility ideographs, fullwidth forms, and astral CJK extensions.+ CJK_PATTERN = re.compile(+ "["+ "\u3000-\u303f" # CJK symbols and punctuation+ "\u3040-\u30ff" # Hiragana + Katakana+ "\u3400-\u4dbf" # CJK Unified Ideographs Extension A+ "\u4e00-\u9fff" # CJK Unified Ideographs+ "\uac00-\ud7af" # Hangul syllables+ "\uf900-\ufaff" # CJK compatibility ideographs+ "\uff00-\uffef" # Halfwidth and fullwidth forms+ "\U00020000-\U0002a6df" # CJK Unified Ideographs Extension B+ "]"+ )URL_PATTERN = re.compile(r"https?://\S+")UUID_PATTERN = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE@@ -83,14 +107,32 @@ class EstimatingTokenCounter(BaseTokenizer):# Auto-detect content type and adjust ratioratio = self._detect_ratio(text)- # Apply ratio with minimum of 1 token- base_count = int(len(text) / ratio + 0.5)+ # Price dense scripts (CJK/Kana/Hangul) separately: they tokenize at+ # roughly one token per character, so applying the Latin ratio to them+ # under-counts by 4-6x. The remaining characters keep the detected ratio.+ cjk_chars = self._count_cjk_chars(text)+ other_chars = len(text) - cjk_chars+ base_count = int(other_chars / ratio + cjk_chars / self.CHARS_PER_TOKEN_CJK + 0.5)# Add overhead for special patternsoverhead = self._count_special_overhead(text)return max(1, base_count + overhead)+ def _count_cjk_chars(self, text: str) -> int:+ """Count dense-script (CJK/Kana/Hangul/fullwidth) codepoints.++ These scripts encode at ~1 token per character, unlike Latin text+ (~4 chars per token), so they are priced with CHARS_PER_TOKEN_CJK.++ Args:+ text: Text to analyze.++ Returns:+ Number of dense-script characters in the text.+ """+ return len(self.CJK_PATTERN.findall(text))+def _detect_ratio(self, text: str) -> float:"""Detect optimal chars-per-token ratio based on content.diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.pyindex bc1b3b19b..6c89d8f32 100644--- a/tests/test_tokenizers.py+++ b/tests/test_tokenizers.py@@ -208,6 +208,47 @@ def hello():count = counter.count_text(code_text)assert count > 0+ def test_count_text_cjk_not_underestimated(self):+ """CJK text must not be priced at the Latin ~4-chars/token ratio.++ Regression: count_text divided the whole string length by the Latin+ ratio (4.0), so 100 Chinese characters estimated ~25 tokens while real+ tokenizers (cl100k_base / DeepSeek / Qwen) produce ~60-150. Dense+ scripts tokenize at roughly one token per character, so the estimate+ must be far above len/4 and on the order of the character count.+ """+ counter = EstimatingTokenCounter()+ text = "你好世界" * 25 # 100 CJK characters+ count = counter.count_text(text)+ # Old behavior returned len/4 == 25; require clearly above that floor.+ assert count > len(text) / 3+ # And in the right ballpark for one-token-per-char scripts.+ assert count >= int(len(text) * 0.6)++ def test_count_text_cjk_japanese_and_korean(self):+ """Japanese (Kana) and Korean (Hangul) are also dense scripts."""+ counter = EstimatingTokenCounter()+ for text in ("こんにちは世界" * 10, "안녕하세요" * 10):+ count = counter.count_text(text)+ assert count >= int(len(text) * 0.6)++ def test_count_text_mixed_latin_cjk(self):+ """Mixed text prices the Latin part and the CJK part independently."""+ counter = EstimatingTokenCounter()+ latin = "The quick brown fox jumps over the lazy dog. " # 45 chars+ cjk = "今天天气很好" # 6 CJK chars+ mixed = counter.count_text(latin + cjk)+ # Must exceed the all-Latin estimate of the same length, since the CJK+ # tail is priced denser than 4 chars/token.+ latin_only = counter.count_text(latin + "x" * len(cjk))+ assert mixed > latin_only++ def test_count_text_latin_unchanged(self):+ """Pure-Latin estimates are unchanged by the CJK adjustment."""+ counter = EstimatingTokenCounter()+ text = "Hello, world!"+ assert 2 <= counter.count_text(text) <= 6+def test_repr(self):"""Test string representation."""counter = EstimatingTokenCounter()