Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(evals): CJK-aware F1 tokenization + token estimation (#1527)
Zhenjia ZHOU99a8540e6574
diff --git a/headroom/evals/core.py b/headroom/evals/core.pyindex dca3cdc85..8fb7416f2 100644--- a/headroom/evals/core.py+++ b/headroom/evals/core.py@@ -260,8 +260,10 @@ class CompressionEvaluator:return f"Based on the context, {' '.join(words[:20])}..."def _estimate_tokens(self, text: str) -> int:- """Estimate token count (roughly 4 chars per token)."""- return len(text) // 4+ """Estimate token count: ~1.5 tokens per CJK char (CJK is dense, ~1-2+ tokens/char), ~4 chars per token otherwise."""+ cjk = sum(1 for c in text if " " <= c <= "鿿" or "가" <= c <= "")+ return int(cjk * 1.5) + (len(text) - cjk) // 4def evaluate_case(self,diff --git a/headroom/evals/metrics.py b/headroom/evals/metrics.pyindex c45d7366c..f2d96f881 100644--- a/headroom/evals/metrics.py+++ b/headroom/evals/metrics.py@@ -22,11 +22,26 @@ def normalize_text(text: str) -> str:return text+# CJK has no word spaces, so a CJK run is split into overlapping char bigrams+# (unigram if length 1) -- this makes token-level F1/recall meaningful instead of+# all-or-nothing on a whole-string CJK token. ASCII/digit runs are kept whole.+_CJK = re.compile(r"[㐀-鿿-ヿ가-]")+_CJK_OR_OTHER = re.compile(r"[㐀-鿿-ヿ가-]+|[^㐀-鿿-ヿ가-]+")++def tokenize(text: str) -> list[str]:- """Simple word tokenization."""- # Split on whitespace and punctuation- tokens = re.findall(r"\b\w+\b", text.lower())- return tokens+ """Word tokenization (CJK-aware: CJK runs -> overlapping char bigrams)."""+ out: list[str] = []+ for tok in re.findall(r"\b\w+\b", text.lower()):+ for run in _CJK_OR_OTHER.findall(tok):+ if _CJK.match(run):+ if len(run) > 1:+ out.extend(run[i : i + 2] for i in range(len(run) - 1))+ else:+ out.append(run)+ else:+ out.append(run)+ return outdef compute_exact_match(response_a: str, response_b: str) -> bool:diff --git a/tests/test_evals_cjk_tokenization.py b/tests/test_evals_cjk_tokenization.pynew file mode 100644index 000000000..f6680a02d--- /dev/null+++ b/tests/test_evals_cjk_tokenization.py@@ -0,0 +1,34 @@+"""CJK correctness for the evals metric layer.++The shared eval metrics assumed ASCII: `tokenize` used `\\b\\w+\\b` (a space-free+CJK string collapses to ONE token, so token-F1 is all-or-nothing), and+`_estimate_tokens` used `len(text)//4` (CJK is ~1-2 tokens/char, not 0.25), so+CJK compression savings were reported ~4-6x wrong.+"""++from headroom.evals.core import CompressionEvaluator+from headroom.evals.metrics import compute_f1, tokenize+++def test_tokenize_splits_cjk_into_units():+ toks = tokenize("数据库连接失败")+ assert len(toks) >= 3, f"CJK must split into multiple units, got {toks}"+++def test_tokenize_ascii_unchanged():+ assert tokenize("Hello, World 42") == ["hello", "world", "42"]+++def test_f1_partial_credit_on_overlapping_cjk():+ # two CJK strings that share most characters must score strictly between 0 and 1+ f1 = compute_f1("数据库连接失败", "数据库连接成功")+ assert 0.0 < f1 < 1.0, f"overlapping CJK should be partial credit, got {f1}"+++def test_estimate_tokens_cjk_not_underestimated():+ # 20 CJK chars: len//4 gives 5; CJK-aware should be >= ~13+ assert CompressionEvaluator._estimate_tokens(None, "数" * 20) >= 13+++def test_estimate_tokens_ascii_unchanged():+ assert CompressionEvaluator._estimate_tokens(None, "x" * 40) == 10