Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizers): coerce structured text values in message token counters (#4472)

Parideboy6cc27a453d9c
diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.py
index c18eb7a91..5ce1ac640 100644
--- a/headroom/tokenizers/base.py
+++ b/headroom/tokenizers/base.py
@@ -214,16 +214,22 @@ class BaseTokenizer(ABC):
# Count role
role = message.get("role", "")
- total += self.count_text(role)
+ total += self.count_text(coerce_countable_text(role))
# Count content
content = message.get("content")
+ if isinstance(content, dict):
+ # A single block sent without its list wrapper is priced like
+ # the wrapped block, so images keep their fixed estimate (#4273).
+ content = [content]
if content is not None:
if isinstance(content, str):
total += self.count_text(content)
elif isinstance(content, list):
# Multi-part content (images, tool results, etc.)
total += self._count_content_parts(content)
+ else:
+ total += self.count_text(coerce_countable_text(content))
# Count tool calls
tool_calls = message.get("tool_calls")
@@ -238,7 +244,7 @@ class BaseTokenizer(ABC):
# Count name field
name = message.get("name")
if name:
- total += self.count_text(name)
+ total += self.count_text(coerce_countable_text(name))
total += 1 # Name field overhead
# Reply start overhead
@@ -258,7 +264,7 @@ class BaseTokenizer(ABC):
part_type = part.get("type", "")
if part_type == "text":
- total += self.count_text(part.get("text", ""))
+ total += self.count_text(coerce_countable_text(part.get("text", "")))
elif part_type in ("image_url", "image", "input_image"):
# Images are NOT tokenized as text — they have a pixel-based cost.
# Anthropic: tokens = (width * height) / 750, max ~1600 after resize.
@@ -284,15 +290,15 @@ class BaseTokenizer(ABC):
else:
total += self._count_serialized(content)
elif part_type == "tool_use":
- total += self.count_text(part.get("name", ""))
+ total += self.count_text(coerce_countable_text(part.get("name", "")))
total += self._count_serialized(part.get("input", {}))
elif not part_type and "text" in part:
# Strands SDK format: {"text": "..."} without "type" field
- total += self.count_text(part["text"])
+ total += self.count_text(coerce_countable_text(part["text"]))
elif not part_type and "toolUse" in part:
# Strands SDK tool_use: {"toolUse": {"name": ..., "input": ...}}
tool_use = part["toolUse"]
- total += self.count_text(tool_use.get("name", ""))
+ total += self.count_text(coerce_countable_text(tool_use.get("name", "")))
total += self._count_serialized(tool_use.get("input", {}))
elif not part_type and "toolResult" in part:
# Strands SDK tool_result: {"toolResult": {"content": [...]}}
@@ -311,7 +317,9 @@ class BaseTokenizer(ABC):
reasoning = part["reasoningContent"]
reasoning_text = reasoning.get("reasoningText", {})
if isinstance(reasoning_text, dict):
- total += self.count_text(reasoning_text.get("text", ""))
+ total += self.count_text(
+ coerce_countable_text(reasoning_text.get("text", ""))
+ )
elif isinstance(reasoning_text, str):
total += self.count_text(reasoning_text)
elif not part_type and "document" in part:
@@ -361,7 +369,7 @@ class BaseTokenizer(ABC):
# ~4 bytes/token (len * 3/4 / 4), which lands that session
# near the provider figure and never prices an omitted
# block at zero.
- total += self.count_text(part.get("thinking", "") or "")
+ total += self.count_text(coerce_countable_text(part.get("thinking", "") or ""))
total += len(part.get("signature") or "") * 3 // 16
else:
# Unknown type - estimate from JSON
diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py
index 4ec8fadc2..4cd4261d2 100644
--- a/headroom/tokenizers/tiktoken_counter.py
+++ b/headroom/tokenizers/tiktoken_counter.py
@@ -297,6 +297,9 @@ class TiktokenCounter(BaseTokenizer):
continue
if key == "content":
+ if isinstance(value, dict):
+ # Unwrapped single block: price it like the list form (#4273).
+ value = [value]
if isinstance(value, str):
total += self.count_text(value)
elif isinstance(value, list):
@@ -304,7 +307,9 @@ class TiktokenCounter(BaseTokenizer):
for part in value:
if isinstance(part, dict):
if part.get("type") == "text":
- total += self.count_text(part.get("text", ""))
+ total += self.count_text(
+ coerce_countable_text(part.get("text", ""))
+ )
elif part.get("type") == "image_url":
# Image tokens vary by detail level
detail = part.get("image_url", {}).get("detail", "auto")
@@ -324,6 +329,8 @@ class TiktokenCounter(BaseTokenizer):
total += self._count_content_parts([part])
elif isinstance(part, str):
total += self.count_text(part)
+ else:
+ total += self.count_text(coerce_countable_text(value))
elif key == "role":
total += self.count_text(coerce_countable_text(value))
elif key == "name":
diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py
index dd8791c2c..38d9e0236 100644
--- a/tests/test_tokenizers.py
+++ b/tests/test_tokenizers.py
@@ -17,6 +17,7 @@ from headroom.tokenizers import (
list_supported_models,
register_tokenizer,
)
+from headroom.tokenizers.base import coerce_countable_text
class TestTiktokenCounter:
@@ -771,3 +772,120 @@ class TestThinkingBlockCounting:
- count_content_blocks(unsigned, counter.count_text)
== 750
)
+
+
+class TestDictValuedTextFields:
+ """Structured (non-string) text fields must not crash token counting.
+
+ Regression (GH #4273): a dict-valued ``text`` block reached
+ ``tiktoken.encode`` unchanged and raised ``TypeError: unhashable type:
+ 'dict'``, aborting compression for the whole request. The fix reuses
+ :func:`coerce_countable_text` so a non-string text-ish field is priced
+ like the JSON string it should have been.
+ """
+
+ @pytest.mark.parametrize(
+ "counter_cls",
+ [TiktokenCounter, EstimatingTokenCounter],
+ ids=["tiktoken", "estimator"],
+ )
+ def test_dict_text_block_counts_like_coerced_string(self, counter_cls):
+ counter = counter_cls()
+ text_dict = {"message": "non-string provider text"}
+ expected = coerce_countable_text(text_dict)
+
+ bad = {"role": "user", "content": [{"type": "text", "text": text_dict}]}
+ good = {"role": "user", "content": [{"type": "text", "text": expected}]}
+
+ assert counter.count_messages([bad]) == counter.count_messages([good])
+
+ @pytest.mark.parametrize(
+ "counter_cls",
+ [TiktokenCounter, EstimatingTokenCounter],
+ ids=["tiktoken", "estimator"],
+ )
+ def test_dict_text_block_does_not_mutate_input(self, counter_cls):
+ import copy
+
+ counter = counter_cls()
+ message = {
+ "role": "user",
+ "content": [{"type": "text", "text": {"message": "non-string provider text"}}],
+ }
+ original = copy.deepcopy(message)
+
+ counter.count_messages([message])
+
+ assert message == original
+
+ @pytest.mark.parametrize(
+ "counter_cls",
+ [TiktokenCounter, EstimatingTokenCounter],
+ ids=["tiktoken", "estimator"],
+ )
+ @pytest.mark.parametrize(
+ "block",
+ [
+ {"type": "text", "text": {"message": "non-string provider text"}},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64," + "A" * 20000}},
+ ],
+ ids=["text", "image"],
+ )
+ def test_unwrapped_content_block_counts_like_list_form(self, counter_cls, block):
+ # An image must keep its fixed estimate, not be priced as base64 text.
+ counter = counter_cls()
+ bare = {"role": "user", "content": block}
+ wrapped = {"role": "user", "content": [block]}
+
+ assert counter.count_messages([bare]) == counter.count_messages([wrapped])
+
+ @pytest.mark.parametrize(
+ "counter_cls",
+ [TiktokenCounter, EstimatingTokenCounter],
+ ids=["tiktoken", "estimator"],
+ )
+ def test_dict_role_counts_like_coerced_string(self, counter_cls):
+ counter = counter_cls()
+ role_dict = {"role": "user"}
+ expected = coerce_countable_text(role_dict)
+
+ bad = {"role": role_dict, "content": "hi"}
+ good = {"role": expected, "content": "hi"}
+
+ assert counter.count_messages([bad]) == counter.count_messages([good])
+
+ @pytest.mark.parametrize(
+ "counter_cls",
+ [TiktokenCounter, EstimatingTokenCounter],
+ ids=["tiktoken", "estimator"],
+ )
+ def test_dict_tool_result_content_counts_like_coerced_string(self, counter_cls):
+ counter = counter_cls()
+ tr_dict = {"output": "non-string provider text"}
+ expected = coerce_countable_text(tr_dict)
+
+ bad = {
+ "role": "user",
+ "content": [{"type": "tool_result", "content": tr_dict}],
+ }
+ good = {
+ "role": "user",
+ "content": [{"type": "tool_result", "content": expected}],
+ }
+
+ assert counter.count_messages([bad]) == counter.count_messages([good])
+
+ @pytest.mark.parametrize(
+ "counter_cls",
+ [TiktokenCounter, EstimatingTokenCounter],
+ ids=["tiktoken", "estimator"],
+ )
+ def test_dict_thinking_counts_like_coerced_string(self, counter_cls):
+ counter = counter_cls()
+ thinking_dict = {"reasoning": "non-string provider text"}
+ expected = coerce_countable_text(thinking_dict)
+
+ bad = {"role": "assistant", "content": [{"type": "thinking", "thinking": thinking_dict}]}
+ good = {"role": "assistant", "content": [{"type": "thinking", "thinking": expected}]}
+
+ assert counter.count_messages([bad]) == counter.count_messages([good])