Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tokenizer): coerce non-string tool_call fields before counting (#2801)

Parideboyb6f9877c78b3
diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.py
index 0bca5be84..5cac19a06 100644
--- a/headroom/providers/anthropic.py
+++ b/headroom/providers/anthropic.py
@@ -24,7 +24,7 @@ import warnings
from typing import Any, cast
from headroom import paths as _paths
-from headroom.tokenizers.base import count_content_blocks
+from headroom.tokenizers.base import coerce_countable_text, count_content_blocks
from .base import Provider, TokenCounter
@@ -411,9 +411,9 @@ class AnthropicTokenCounter(TokenCounter):
if "tool_calls" in message:
for tool_call in message.get("tool_calls", []):
if isinstance(tool_call, dict):
- func = tool_call.get("function", {})
- tokens += self.count_text(func.get("name", ""))
- tokens += self.count_text(func.get("arguments", ""))
+ func = tool_call.get("function") or {}
+ tokens += self.count_text(coerce_countable_text(func.get("name")))
+ tokens += self.count_text(coerce_countable_text(func.get("arguments")))
return tokens
diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py
index c7c2e1235..5b41bc8e1 100644
--- a/headroom/providers/openai.py
+++ b/headroom/providers/openai.py
@@ -16,7 +16,7 @@ from functools import lru_cache
from typing import Any, cast
from headroom import paths as _paths
-from headroom.tokenizers.base import count_content_blocks
+from headroom.tokenizers.base import coerce_countable_text, count_content_blocks
from .base import Provider, TokenCounter
@@ -403,10 +403,10 @@ class OpenAITokenCounter:
tool_calls = message.get("tool_calls")
if tool_calls:
for tc in tool_calls:
- func = tc.get("function", {})
- tokens += self.count_text(func.get("name", ""))
- tokens += self.count_text(func.get("arguments", ""))
- tokens += self.count_text(tc.get("id", ""))
+ func = tc.get("function") or {}
+ tokens += self.count_text(coerce_countable_text(func.get("name")))
+ tokens += self.count_text(coerce_countable_text(func.get("arguments")))
+ tokens += self.count_text(coerce_countable_text(tc.get("id")))
tokens += 10 # Structural overhead
# Tool call ID for tool responses
diff --git a/headroom/providers/openai_compatible.py b/headroom/providers/openai_compatible.py
index da0ebd3f4..4cb6673ce 100644
--- a/headroom/providers/openai_compatible.py
+++ b/headroom/providers/openai_compatible.py
@@ -24,7 +24,7 @@ from dataclasses import dataclass
from typing import Any
from headroom.tokenizers import get_tokenizer
-from headroom.tokenizers.base import count_content_blocks
+from headroom.tokenizers.base import coerce_countable_text, count_content_blocks
from .base import Provider
@@ -178,9 +178,9 @@ class OpenAICompatibleTokenCounter:
tool_calls = message.get("tool_calls")
if tool_calls:
for tc in tool_calls:
- func = tc.get("function", {})
- tokens += self.count_text(func.get("name", ""))
- tokens += self.count_text(func.get("arguments", ""))
+ func = tc.get("function") or {}
+ tokens += self.count_text(coerce_countable_text(func.get("name")))
+ tokens += self.count_text(coerce_countable_text(func.get("arguments")))
tokens += 10
tool_call_id = message.get("tool_call_id")
diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.py
index fcd38ad13..5d291216d 100644
--- a/headroom/tokenizers/base.py
+++ b/headroom/tokenizers/base.py
@@ -11,6 +11,40 @@ from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import Any, Protocol, runtime_checkable
+#: Cap on the serialized form of a non-string tool-call field. A malformed
+#: upstream can put an arbitrarily large object where a JSON string belongs;
+#: serializing it unbounded would turn a token *estimate* into a multi-megabyte
+#: encode. Truncating keeps the estimate finite and the request alive.
+_MAX_COERCED_FIELD_CHARS = 200_000
+
+
+def coerce_countable_text(value: Any) -> str:
+ """Return *value* as text safe to pass to ``count_text``.
+
+ Tool-call fields (``function.name``, ``function.arguments``, ``id``) are
+ strings per the OpenAI spec, but OpenAI-compatible upstreams do emit
+ ``None`` or a raw object there. Passing those straight to
+ ``tiktoken.encode()`` raises ``TypeError: expected string or buffer``, which
+ failed the whole compression with a 503 — and kept failing on every later
+ request, because the malformed message stays in conversation history.
+
+ ``None`` counts as nothing; dicts/lists are JSON-serialized so an object
+ ``arguments`` is priced roughly like the JSON string it should have been;
+ anything else falls back to ``str()``.
+ """
+ if isinstance(value, str):
+ return value
+ if value is None:
+ return ""
+ if isinstance(value, dict | list | tuple):
+ try:
+ text = json.dumps(value, ensure_ascii=False, default=str)
+ except (TypeError, ValueError):
+ text = str(value)
+ else:
+ text = str(value)
+ return text[:_MAX_COERCED_FIELD_CHARS]
+
@runtime_checkable
class TokenCounter(Protocol):
@@ -311,20 +345,20 @@ class BaseTokenizer(ABC):
total += 4 # Tool call overhead
if "function" in call:
- func = call["function"]
- total += self.count_text(func.get("name", ""))
- total += self.count_text(func.get("arguments", ""))
+ func = call["function"] or {}
+ total += self.count_text(coerce_countable_text(func.get("name")))
+ total += self.count_text(coerce_countable_text(func.get("arguments")))
if "id" in call:
- total += self.count_text(call["id"])
+ total += self.count_text(coerce_countable_text(call["id"]))
return total
def _count_function_call(self, function_call: dict[str, Any]) -> int:
"""Count tokens in legacy function call."""
total = 4 # Function call overhead
- total += self.count_text(function_call.get("name", ""))
- total += self.count_text(function_call.get("arguments", ""))
+ total += self.count_text(coerce_countable_text(function_call.get("name")))
+ total += self.count_text(coerce_countable_text(function_call.get("arguments")))
return total
def encode(self, text: str) -> list[int]:
diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py
index 87a4ef0fa..1aa987037 100644
--- a/headroom/tokenizers/tiktoken_counter.py
+++ b/headroom/tokenizers/tiktoken_counter.py
@@ -16,7 +16,7 @@ import threading
from functools import lru_cache
from typing import Any
-from .base import BaseTokenizer
+from .base import BaseTokenizer, coerce_countable_text
logger = logging.getLogger(__name__)
@@ -316,24 +316,25 @@ class TiktokenCounter(BaseTokenizer):
elif isinstance(part, str):
total += self.count_text(part)
elif key == "role":
- total += self.count_text(value)
+ total += self.count_text(coerce_countable_text(value))
elif key == "name":
- total += self.count_text(value)
+ total += self.count_text(coerce_countable_text(value))
total += 1 # Name adds 1 token
elif key == "tool_calls":
- for tool_call in value:
+ for tool_call in value or []:
total += 3 # Tool call overhead
if "function" in tool_call:
- func = tool_call["function"]
- total += self.count_text(func.get("name", ""))
- total += self.count_text(func.get("arguments", ""))
+ func = tool_call["function"] or {}
+ total += self.count_text(coerce_countable_text(func.get("name")))
+ total += self.count_text(coerce_countable_text(func.get("arguments")))
if "id" in tool_call:
- total += self.count_text(tool_call["id"])
+ total += self.count_text(coerce_countable_text(tool_call["id"]))
elif key == "tool_call_id":
- total += self.count_text(value)
+ total += self.count_text(coerce_countable_text(value))
elif key == "function_call":
- total += self.count_text(value.get("name", ""))
- total += self.count_text(value.get("arguments", ""))
+ value = value or {}
+ total += self.count_text(coerce_countable_text(value.get("name")))
+ total += self.count_text(coerce_countable_text(value.get("arguments")))
# Every reply is primed with assistant
total += self.REPLY_OVERHEAD
diff --git a/tests/test_tool_call_arguments_not_a_string.py b/tests/test_tool_call_arguments_not_a_string.py
new file mode 100644
index 000000000..4d1d2eb3c
--- /dev/null
+++ b/tests/test_tool_call_arguments_not_a_string.py
@@ -0,0 +1,97 @@
+"""Token counters must survive a tool_call whose fields aren't strings (GH #2782).
+
+``function.arguments`` is a JSON *string* per the OpenAI spec, but
+OpenAI-compatible upstreams do emit ``None`` or a raw object there. Every counter
+passed the value straight to ``tiktoken.encode()``, which raises
+``TypeError: expected string or buffer`` — so ``/v1/compress`` failed the whole
+request with a 503. Worse, the malformed message stays in conversation history,
+so every later request replaying that history failed too, regardless of provider.
+
+``arguments: None`` stopped raising once ``count_text`` grew its falsy guard, but
+any *truthy* non-string (``{"path": "x"}``, ``5``) still crashed all four
+counters. The fix is ``coerce_countable_text`` at the tool-call field sites, so a
+dict is priced roughly like the JSON string it should have been rather than
+either crashing or silently counting as zero.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from headroom.providers.anthropic import AnthropicProvider
+from headroom.providers.openai import OpenAITokenCounter
+from headroom.providers.openai_compatible import OpenAICompatibleTokenCounter
+from headroom.tokenizers.base import coerce_countable_text
+from headroom.tokenizers.tiktoken_counter import TiktokenCounter
+
+
+def _counters():
+ return {
+ "openai": OpenAITokenCounter("gpt-4o"),
+ "openai_compatible": OpenAICompatibleTokenCounter("gpt-4o"),
+ "anthropic": AnthropicProvider(warn=False).get_token_counter("claude-sonnet-4-6"),
+ "tiktoken": TiktokenCounter("gpt-4o"),
+ }
+
+
+def _message(arguments: object) -> dict:
+ return {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "read_file", "arguments": arguments},
+ }
+ ],
+ }
+
+
+ "arguments",
+ [None, {"path": "x"}, ["a", "b"], 5, True, 1.5],
+ ids=["none", "dict", "list", "int", "bool", "float"],
+)
+def test_non_string_arguments_do_not_raise(arguments: object) -> None:
+ """The reported crash: 503 + TypeError out of tiktoken.encode."""
+ for name, counter in _counters().items():
+ got = counter.count_messages([_message(arguments)])
+ assert got > 0, f"{name} priced the whole message at {got}"
+
+
+def test_object_arguments_are_priced_like_their_json_form() -> None:
+ """Not just non-crashing: a dict must not silently count as zero."""
+ payload = {"path": "src/very/long/path/to/a/file.py", "start": 1, "end": 400}
+ for name, counter in _counters().items():
+ as_object = counter.count_messages([_message(payload)])
+ as_json = counter.count_messages([_message(json.dumps(payload))])
+ assert abs(as_object - as_json) <= 5, f"{name}: {as_object} vs {as_json}"
+
+
+def test_null_function_and_id_do_not_raise() -> None:
+ """``{"function": null}`` / ``{"id": null}`` reach the same encode path."""
+ message = {"role": "assistant", "tool_calls": [{"id": None, "function": None}]}
+ for name, counter in _counters().items():
+ assert counter.count_messages([message]) > 0, name
+
+
+def test_legacy_function_call_with_object_arguments_does_not_raise() -> None:
+ message = {"role": "assistant", "function_call": {"name": "f", "arguments": {"a": 1}}}
+ for name, counter in _counters().items():
+ assert counter.count_messages([message]) > 0, name
+
+
+def test_oversized_object_arguments_are_bounded() -> None:
+ """A malformed upstream must not turn an estimate into a megabyte encode."""
+ huge = {"blob": "x" * 5_000_000}
+ assert len(coerce_countable_text(huge)) <= 200_000
+
+
+def test_string_arguments_are_untouched() -> None:
+ """The control: the spec-compliant shape must not move."""
+ args = json.dumps({"path": "a.py"})
+ assert coerce_countable_text(args) == args
+ assert coerce_countable_text(None) == ""