Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(evals): CJK-aware F1 tokenization + token estimation (#1527)

Zhenjia ZHOU99a8540e6574
diff --git a/headroom/evals/core.py b/headroom/evals/core.py
index dca3cdc85..8fb7416f2 100644
--- a/headroom/evals/core.py
+++ b/headroom/evals/core.py
@@ -260,8 +260,10 @@ class CompressionEvaluator:
return f"Based on the context, {' '.join(words[:20])}..."
def _estimate_tokens(self, text: str) -> int:
- """Estimate token count (roughly 4 chars per token)."""
- return len(text) // 4
+ """Estimate token count: ~1.5 tokens per CJK char (CJK is dense, ~1-2
+ tokens/char), ~4 chars per token otherwise."""
+ cjk = sum(1 for c in text if " " <= c <= "鿿" or "가" <= c <= "힯")
+ return int(cjk * 1.5) + (len(text) - cjk) // 4
def evaluate_case(
self,
diff --git a/headroom/evals/metrics.py b/headroom/evals/metrics.py
index c45d7366c..f2d96f881 100644
--- a/headroom/evals/metrics.py
+++ b/headroom/evals/metrics.py
@@ -22,11 +22,26 @@ def normalize_text(text: str) -> str:
return text
+# CJK has no word spaces, so a CJK run is split into overlapping char bigrams
+# (unigram if length 1) -- this makes token-level F1/recall meaningful instead of
+# all-or-nothing on a whole-string CJK token. ASCII/digit runs are kept whole.
+_CJK = re.compile(r"[㐀-鿿぀-ヿ가-힯]")
+_CJK_OR_OTHER = re.compile(r"[㐀-鿿぀-ヿ가-힯]+|[^㐀-鿿぀-ヿ가-힯]+")
+
+
def tokenize(text: str) -> list[str]:
- """Simple word tokenization."""
- # Split on whitespace and punctuation
- tokens = re.findall(r"\b\w+\b", text.lower())
- return tokens
+ """Word tokenization (CJK-aware: CJK runs -> overlapping char bigrams)."""
+ out: list[str] = []
+ for tok in re.findall(r"\b\w+\b", text.lower()):
+ for run in _CJK_OR_OTHER.findall(tok):
+ if _CJK.match(run):
+ if len(run) > 1:
+ out.extend(run[i : i + 2] for i in range(len(run) - 1))
+ else:
+ out.append(run)
+ else:
+ out.append(run)
+ return out
def compute_exact_match(response_a: str, response_b: str) -> bool:
diff --git a/tests/test_evals_cjk_tokenization.py b/tests/test_evals_cjk_tokenization.py
new file mode 100644
index 000000000..f6680a02d
--- /dev/null
+++ b/tests/test_evals_cjk_tokenization.py
@@ -0,0 +1,34 @@
+"""CJK correctness for the evals metric layer.
+
+The shared eval metrics assumed ASCII: `tokenize` used `\\b\\w+\\b` (a space-free
+CJK string collapses to ONE token, so token-F1 is all-or-nothing), and
+`_estimate_tokens` used `len(text)//4` (CJK is ~1-2 tokens/char, not 0.25), so
+CJK compression savings were reported ~4-6x wrong.
+"""
+
+from headroom.evals.core import CompressionEvaluator
+from headroom.evals.metrics import compute_f1, tokenize
+
+
+def test_tokenize_splits_cjk_into_units():
+ toks = tokenize("数据库连接失败")
+ assert len(toks) >= 3, f"CJK must split into multiple units, got {toks}"
+
+
+def test_tokenize_ascii_unchanged():
+ assert tokenize("Hello, World 42") == ["hello", "world", "42"]
+
+
+def test_f1_partial_credit_on_overlapping_cjk():
+ # two CJK strings that share most characters must score strictly between 0 and 1
+ f1 = compute_f1("数据库连接失败", "数据库连接成功")
+ assert 0.0 < f1 < 1.0, f"overlapping CJK should be partial credit, got {f1}"
+
+
+def test_estimate_tokens_cjk_not_underestimated():
+ # 20 CJK chars: len//4 gives 5; CJK-aware should be >= ~13
+ assert CompressionEvaluator._estimate_tokens(None, "数" * 20) >= 13
+
+
+def test_estimate_tokens_ascii_unchanged():
+ assert CompressionEvaluator._estimate_tokens(None, "x" * 40) == 10