Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): price Claude against a real BPE (tiktoken o200k) not a char estimate (#2543)
Tejas Chopra285176be54e1
diff --git a/headroom/tokenizers/registry.py b/headroom/tokenizers/registry.pyindex ad59aeec0..eddce6fa6 100644--- a/headroom/tokenizers/registry.py+++ b/headroom/tokenizers/registry.py@@ -36,7 +36,9 @@ MODEL_PATTERNS: list[tuple[str, str]] = [(r"^curie", "tiktoken"),(r"^babbage", "tiktoken"),(r"^ada", "tiktoken"),- # Anthropic models -> estimation (Claude uses custom tokenizer)+ # Anthropic models -> real BPE proxy (Claude's tokenizer is private; priced+ # against tiktoken o200k_base for consistent, monotone counts — see+ # _create_anthropic)(r"^claude-", "anthropic"),# Llama family -> huggingface (when available)(r"^llama", "huggingface"),@@ -335,13 +337,33 @@ class TokenizerRegistry:return EstimatingTokenCounter()def _create_anthropic(self, model: str) -> TokenCounter:- """Create Anthropic tokenizer.-- Anthropic uses a custom tokenizer that's not publicly available.- We use estimation calibrated for Claude models.+ """Create Anthropic (Claude) tokenizer.++ Anthropic's tokenizer isn't public. Rather than a character-ratio+ estimate — whose chars-per-token flips with the detected content type+ (JSON 3.2 / code 3.5 / English 4.0), so compressing text can appear to+ *increase* tokens and two components disagree on the same bytes — price+ Claude against a real BPE (tiktoken ``o200k_base``) as a stable, monotone+ proxy. It is not Claude's exact vocab, but it is deterministic,+ consistent before/after, and within ~10-20% of Claude's real counts —+ which is what compression ratios and context-pressure gating need. Falls+ back to the character estimator if the tiktoken vocab can't be loaded."""- # Claude models use ~3.5 chars per token on average- return EstimatingTokenCounter(chars_per_token=3.5)+ try:+ from .tiktoken_counter import (+ TiktokenCounter,+ TiktokenLoadError,+ load_encoding,+ )++ try:+ load_encoding("o200k_base")+ except TiktokenLoadError:+ logger.info("tiktoken o200k_base unavailable for %s; using char estimator", model)+ return EstimatingTokenCounter(chars_per_token=3.5)+ return TiktokenCounter(model, encoding="o200k_base")+ except Exception: # pragma: no cover - defensive; keep counting alive+ return EstimatingTokenCounter(chars_per_token=3.5)def _create_google(self, model: str) -> TokenCounter:"""Create Google tokenizer.diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.pyindex 91065e40f..5c05f308d 100644--- a/headroom/tokenizers/tiktoken_counter.py+++ b/headroom/tokenizers/tiktoken_counter.py@@ -212,15 +212,19 @@ class TiktokenCounter(BaseTokenizer):MESSAGE_OVERHEAD = 3REPLY_OVERHEAD = 3- def __init__(self, model: str = "gpt-4o"):+ def __init__(self, model: str = "gpt-4o", encoding: str | None = None):"""Initialize tiktoken counter.Args:model: Model name to determine encoding.Defaults to 'gpt-4o' (o200k_base encoding).+ encoding: Explicit tiktoken encoding name (e.g. 'o200k_base') that+ overrides model-based resolution. Used to price+ private-tokenizer models (Claude) against a real BPE proxy+ instead of a character estimate."""self.model = model- self.encoding_name = get_encoding_for_model(model)+ self.encoding_name = encoding or get_encoding_for_model(model)self._encoding = None # Lazy load@propertydiff --git a/tests/test_claude_session_mode_benchmark.py b/tests/test_claude_session_mode_benchmark.pyindex 20c824796..d02c9162d 100644--- a/tests/test_claude_session_mode_benchmark.py+++ b/tests/test_claude_session_mode_benchmark.py@@ -97,7 +97,20 @@ def test_load_session_replay_groups_assistant_request_events(tmp_path: Path) ->def test_simulation_and_winner_logic() -> None:- tool_blob = '{"rows":[1,2,3,4]}' * 80+ # Realistic varied tool output. A pathologically repetitive blob (the same+ # JSON object * N) is a degenerate case for a real BPE tokenizer — it merges+ # the repetition to near-nothing — so a token-mode rewrite can cost MORE real+ # tokens than the original, which the old character estimate masked by+ # over-counting the repetition. Varied records keep the fixture representative+ # of real agent tool output, where the rewrite is a genuine win.+ tool_blob = json.dumps(+ {+ "rows": [+ {"id": i, "label": f"row-{i}", "value": (i * 37) % 100, "ok": i % 3 == 0}+ for i in range(150)+ ]+ }+ )turn1 = ReplayTurn(session_id="s1",project_key="C--git-demo",diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.pyindex f95295ae1..c5d23a1dd 100644--- a/tests/test_tokenizer.py+++ b/tests/test_tokenizer.py@@ -22,6 +22,41 @@ class FakeTokenCounter:return sum(len(str(msg.get("content", "")).split()) for msg in messages)+def test_claude_priced_with_real_bpe_not_char_estimate() -> None:+ """Claude has no public tokenizer, so we price it against a real BPE+ (tiktoken o200k_base) instead of a content-adaptive character estimate —+ otherwise before/after counts drift between components and compressing text+ can appear to *increase* tokens. A tool_result fold must always register as+ a reduction; and when the vocab is available the count is the exact o200k+ count (proving it is a real BPE, not a chars/token ratio)."""+ from headroom.tokenizers import get_tokenizer++ tok = get_tokenizer("claude-opus-4-8")++ long_msg = [+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "t", "content": "alpha " * 300}],+ }+ ]+ short_msg = [+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "t", "content": "alpha " * 3}],+ }+ ]+ assert tok.count_messages(long_msg) > tok.count_messages(short_msg) # fold visible++ try:+ import tiktoken++ enc = tiktoken.get_encoding("o200k_base")+ except Exception: # vocab unavailable → estimator fallback; monotonicity above still holds+ return+ sample = "The quick brown fox jumps over the lazy dog. " * 10+ assert tok.count_text(sample) == len(enc.encode(sample))++def test_tokenizer_delegates_to_counter() -> None:counter = FakeTokenCounter()tokenizer = Tokenizer(counter, model="gpt-4o")diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.pyindex 7b76bb699..524fb38a0 100644--- a/tests/test_tokenizers.py+++ b/tests/test_tokenizers.py@@ -363,9 +363,15 @@ class TestTokenizerRegistry:assert isinstance(tokenizer, TiktokenCounter)def test_get_anthropic_model(self):- """Test getting tokenizer for Anthropic model."""+ """Anthropic (Claude) has no public tokenizer, so it is priced against a+ real BPE proxy (tiktoken o200k_base) rather than a character estimate —+ for consistent, monotone before/after counts. Falls back to the estimator+ only when the tiktoken vocab can't be loaded."""tokenizer = get_tokenizer("claude-3-sonnet")- assert isinstance(tokenizer, EstimatingTokenCounter)+ if isinstance(tokenizer, TiktokenCounter):+ assert tokenizer.encoding_name == "o200k_base"+ else: # vocab unavailable in this environment → documented fallback+ assert isinstance(tokenizer, EstimatingTokenCounter)def test_get_unknown_model_fallback(self):"""Test fallback for unknown model."""