Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizers): coerce structured text values in message token counters (#4472)
Parideboy6cc27a453d9c
diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.pyindex c18eb7a91..5ce1ac640 100644--- a/headroom/tokenizers/base.py+++ b/headroom/tokenizers/base.py@@ -214,16 +214,22 @@ class BaseTokenizer(ABC):# Count rolerole = message.get("role", "")- total += self.count_text(role)+ total += self.count_text(coerce_countable_text(role))# Count contentcontent = message.get("content")+ if isinstance(content, dict):+ # A single block sent without its list wrapper is priced like+ # the wrapped block, so images keep their fixed estimate (#4273).+ content = [content]if content is not None:if isinstance(content, str):total += self.count_text(content)elif isinstance(content, list):# Multi-part content (images, tool results, etc.)total += self._count_content_parts(content)+ else:+ total += self.count_text(coerce_countable_text(content))# Count tool callstool_calls = message.get("tool_calls")@@ -238,7 +244,7 @@ class BaseTokenizer(ABC):# Count name fieldname = message.get("name")if name:- total += self.count_text(name)+ total += self.count_text(coerce_countable_text(name))total += 1 # Name field overhead# Reply start overhead@@ -258,7 +264,7 @@ class BaseTokenizer(ABC):part_type = part.get("type", "")if part_type == "text":- total += self.count_text(part.get("text", ""))+ total += self.count_text(coerce_countable_text(part.get("text", "")))elif part_type in ("image_url", "image", "input_image"):# Images are NOT tokenized as text — they have a pixel-based cost.# Anthropic: tokens = (width * height) / 750, max ~1600 after resize.@@ -284,15 +290,15 @@ class BaseTokenizer(ABC):else:total += self._count_serialized(content)elif part_type == "tool_use":- total += self.count_text(part.get("name", ""))+ total += self.count_text(coerce_countable_text(part.get("name", "")))total += self._count_serialized(part.get("input", {}))elif not part_type and "text" in part:# Strands SDK format: {"text": "..."} without "type" field- total += self.count_text(part["text"])+ total += self.count_text(coerce_countable_text(part["text"]))elif not part_type and "toolUse" in part:# Strands SDK tool_use: {"toolUse": {"name": ..., "input": ...}}tool_use = part["toolUse"]- total += self.count_text(tool_use.get("name", ""))+ total += self.count_text(coerce_countable_text(tool_use.get("name", "")))total += self._count_serialized(tool_use.get("input", {}))elif not part_type and "toolResult" in part:# Strands SDK tool_result: {"toolResult": {"content": [...]}}@@ -311,7 +317,9 @@ class BaseTokenizer(ABC):reasoning = part["reasoningContent"]reasoning_text = reasoning.get("reasoningText", {})if isinstance(reasoning_text, dict):- total += self.count_text(reasoning_text.get("text", ""))+ total += self.count_text(+ coerce_countable_text(reasoning_text.get("text", ""))+ )elif isinstance(reasoning_text, str):total += self.count_text(reasoning_text)elif not part_type and "document" in part:@@ -361,7 +369,7 @@ class BaseTokenizer(ABC):# ~4 bytes/token (len * 3/4 / 4), which lands that session# near the provider figure and never prices an omitted# block at zero.- total += self.count_text(part.get("thinking", "") or "")+ total += self.count_text(coerce_countable_text(part.get("thinking", "") or ""))total += len(part.get("signature") or "") * 3 // 16else:# Unknown type - estimate from JSONdiff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.pyindex 4ec8fadc2..4cd4261d2 100644--- a/headroom/tokenizers/tiktoken_counter.py+++ b/headroom/tokenizers/tiktoken_counter.py@@ -297,6 +297,9 @@ class TiktokenCounter(BaseTokenizer):continueif key == "content":+ if isinstance(value, dict):+ # Unwrapped single block: price it like the list form (#4273).+ value = [value]if isinstance(value, str):total += self.count_text(value)elif isinstance(value, list):@@ -304,7 +307,9 @@ class TiktokenCounter(BaseTokenizer):for part in value:if isinstance(part, dict):if part.get("type") == "text":- total += self.count_text(part.get("text", ""))+ total += self.count_text(+ coerce_countable_text(part.get("text", ""))+ )elif part.get("type") == "image_url":# Image tokens vary by detail leveldetail = part.get("image_url", {}).get("detail", "auto")@@ -324,6 +329,8 @@ class TiktokenCounter(BaseTokenizer):total += self._count_content_parts([part])elif isinstance(part, str):total += self.count_text(part)+ else:+ total += self.count_text(coerce_countable_text(value))elif key == "role":total += self.count_text(coerce_countable_text(value))elif key == "name":diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.pyindex dd8791c2c..38d9e0236 100644--- a/tests/test_tokenizers.py+++ b/tests/test_tokenizers.py@@ -17,6 +17,7 @@ from headroom.tokenizers import (list_supported_models,register_tokenizer,)+from headroom.tokenizers.base import coerce_countable_textclass TestTiktokenCounter:@@ -771,3 +772,120 @@ class TestThinkingBlockCounting:- count_content_blocks(unsigned, counter.count_text)== 750)+++class TestDictValuedTextFields:+ """Structured (non-string) text fields must not crash token counting.++ Regression (GH #4273): a dict-valued ``text`` block reached+ ``tiktoken.encode`` unchanged and raised ``TypeError: unhashable type:+ 'dict'``, aborting compression for the whole request. The fix reuses+ :func:`coerce_countable_text` so a non-string text-ish field is priced+ like the JSON string it should have been.+ """++ @pytest.mark.parametrize(+ "counter_cls",+ [TiktokenCounter, EstimatingTokenCounter],+ ids=["tiktoken", "estimator"],+ )+ def test_dict_text_block_counts_like_coerced_string(self, counter_cls):+ counter = counter_cls()+ text_dict = {"message": "non-string provider text"}+ expected = coerce_countable_text(text_dict)++ bad = {"role": "user", "content": [{"type": "text", "text": text_dict}]}+ good = {"role": "user", "content": [{"type": "text", "text": expected}]}++ assert counter.count_messages([bad]) == counter.count_messages([good])++ @pytest.mark.parametrize(+ "counter_cls",+ [TiktokenCounter, EstimatingTokenCounter],+ ids=["tiktoken", "estimator"],+ )+ def test_dict_text_block_does_not_mutate_input(self, counter_cls):+ import copy++ counter = counter_cls()+ message = {+ "role": "user",+ "content": [{"type": "text", "text": {"message": "non-string provider text"}}],+ }+ original = copy.deepcopy(message)++ counter.count_messages([message])++ assert message == original++ @pytest.mark.parametrize(+ "counter_cls",+ [TiktokenCounter, EstimatingTokenCounter],+ ids=["tiktoken", "estimator"],+ )+ @pytest.mark.parametrize(+ "block",+ [+ {"type": "text", "text": {"message": "non-string provider text"}},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 20000}},+ ],+ ids=["text", "image"],+ )+ def test_unwrapped_content_block_counts_like_list_form(self, counter_cls, block):+ # An image must keep its fixed estimate, not be priced as base64 text.+ counter = counter_cls()+ bare = {"role": "user", "content": block}+ wrapped = {"role": "user", "content": [block]}++ assert counter.count_messages([bare]) == counter.count_messages([wrapped])++ @pytest.mark.parametrize(+ "counter_cls",+ [TiktokenCounter, EstimatingTokenCounter],+ ids=["tiktoken", "estimator"],+ )+ def test_dict_role_counts_like_coerced_string(self, counter_cls):+ counter = counter_cls()+ role_dict = {"role": "user"}+ expected = coerce_countable_text(role_dict)++ bad = {"role": role_dict, "content": "hi"}+ good = {"role": expected, "content": "hi"}++ assert counter.count_messages([bad]) == counter.count_messages([good])++ @pytest.mark.parametrize(+ "counter_cls",+ [TiktokenCounter, EstimatingTokenCounter],+ ids=["tiktoken", "estimator"],+ )+ def test_dict_tool_result_content_counts_like_coerced_string(self, counter_cls):+ counter = counter_cls()+ tr_dict = {"output": "non-string provider text"}+ expected = coerce_countable_text(tr_dict)++ bad = {+ "role": "user",+ "content": [{"type": "tool_result", "content": tr_dict}],+ }+ good = {+ "role": "user",+ "content": [{"type": "tool_result", "content": expected}],+ }++ assert counter.count_messages([bad]) == counter.count_messages([good])++ @pytest.mark.parametrize(+ "counter_cls",+ [TiktokenCounter, EstimatingTokenCounter],+ ids=["tiktoken", "estimator"],+ )+ def test_dict_thinking_counts_like_coerced_string(self, counter_cls):+ counter = counter_cls()+ thinking_dict = {"reasoning": "non-string provider text"}+ expected = coerce_countable_text(thinking_dict)++ bad = {"role": "assistant", "content": [{"type": "thinking", "thinking": thinking_dict}]}+ good = {"role": "assistant", "content": [{"type": "thinking", "thinking": expected}]}++ assert counter.count_messages([bad]) == counter.count_messages([good])