Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tokenizer): coerce non-string tool_call fields before counting (#2801)
Parideboyb6f9877c78b3
diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.pyindex 0bca5be84..5cac19a06 100644--- a/headroom/providers/anthropic.py+++ b/headroom/providers/anthropic.py@@ -24,7 +24,7 @@ import warningsfrom typing import Any, castfrom headroom import paths as _paths-from headroom.tokenizers.base import count_content_blocks+from headroom.tokenizers.base import coerce_countable_text, count_content_blocksfrom .base import Provider, TokenCounter@@ -411,9 +411,9 @@ class AnthropicTokenCounter(TokenCounter):if "tool_calls" in message:for tool_call in message.get("tool_calls", []):if isinstance(tool_call, dict):- func = tool_call.get("function", {})- tokens += self.count_text(func.get("name", ""))- tokens += self.count_text(func.get("arguments", ""))+ func = tool_call.get("function") or {}+ tokens += self.count_text(coerce_countable_text(func.get("name")))+ tokens += self.count_text(coerce_countable_text(func.get("arguments")))return tokensdiff --git a/headroom/providers/openai.py b/headroom/providers/openai.pyindex c7c2e1235..5b41bc8e1 100644--- a/headroom/providers/openai.py+++ b/headroom/providers/openai.py@@ -16,7 +16,7 @@ from functools import lru_cachefrom typing import Any, castfrom headroom import paths as _paths-from headroom.tokenizers.base import count_content_blocks+from headroom.tokenizers.base import coerce_countable_text, count_content_blocksfrom .base import Provider, TokenCounter@@ -403,10 +403,10 @@ class OpenAITokenCounter:tool_calls = message.get("tool_calls")if tool_calls:for tc in tool_calls:- func = tc.get("function", {})- tokens += self.count_text(func.get("name", ""))- tokens += self.count_text(func.get("arguments", ""))- tokens += self.count_text(tc.get("id", ""))+ func = tc.get("function") or {}+ tokens += self.count_text(coerce_countable_text(func.get("name")))+ tokens += self.count_text(coerce_countable_text(func.get("arguments")))+ tokens += self.count_text(coerce_countable_text(tc.get("id")))tokens += 10 # Structural overhead# Tool call ID for tool responsesdiff --git a/headroom/providers/openai_compatible.py b/headroom/providers/openai_compatible.pyindex da0ebd3f4..4cb6673ce 100644--- a/headroom/providers/openai_compatible.py+++ b/headroom/providers/openai_compatible.py@@ -24,7 +24,7 @@ from dataclasses import dataclassfrom typing import Anyfrom headroom.tokenizers import get_tokenizer-from headroom.tokenizers.base import count_content_blocks+from headroom.tokenizers.base import coerce_countable_text, count_content_blocksfrom .base import Provider@@ -178,9 +178,9 @@ class OpenAICompatibleTokenCounter:tool_calls = message.get("tool_calls")if tool_calls:for tc in tool_calls:- func = tc.get("function", {})- tokens += self.count_text(func.get("name", ""))- tokens += self.count_text(func.get("arguments", ""))+ func = tc.get("function") or {}+ tokens += self.count_text(coerce_countable_text(func.get("name")))+ tokens += self.count_text(coerce_countable_text(func.get("arguments")))tokens += 10tool_call_id = message.get("tool_call_id")diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.pyindex fcd38ad13..5d291216d 100644--- a/headroom/tokenizers/base.py+++ b/headroom/tokenizers/base.py@@ -11,6 +11,40 @@ from abc import ABC, abstractmethodfrom collections.abc import Callablefrom typing import Any, Protocol, runtime_checkable+#: Cap on the serialized form of a non-string tool-call field. A malformed+#: upstream can put an arbitrarily large object where a JSON string belongs;+#: serializing it unbounded would turn a token *estimate* into a multi-megabyte+#: encode. Truncating keeps the estimate finite and the request alive.+_MAX_COERCED_FIELD_CHARS = 200_000+++def coerce_countable_text(value: Any) -> str:+ """Return *value* as text safe to pass to ``count_text``.++ Tool-call fields (``function.name``, ``function.arguments``, ``id``) are+ strings per the OpenAI spec, but OpenAI-compatible upstreams do emit+ ``None`` or a raw object there. Passing those straight to+ ``tiktoken.encode()`` raises ``TypeError: expected string or buffer``, which+ failed the whole compression with a 503 — and kept failing on every later+ request, because the malformed message stays in conversation history.++ ``None`` counts as nothing; dicts/lists are JSON-serialized so an object+ ``arguments`` is priced roughly like the JSON string it should have been;+ anything else falls back to ``str()``.+ """+ if isinstance(value, str):+ return value+ if value is None:+ return ""+ if isinstance(value, dict | list | tuple):+ try:+ text = json.dumps(value, ensure_ascii=False, default=str)+ except (TypeError, ValueError):+ text = str(value)+ else:+ text = str(value)+ return text[:_MAX_COERCED_FIELD_CHARS]+@runtime_checkableclass TokenCounter(Protocol):@@ -311,20 +345,20 @@ class BaseTokenizer(ABC):total += 4 # Tool call overheadif "function" in call:- func = call["function"]- total += self.count_text(func.get("name", ""))- total += self.count_text(func.get("arguments", ""))+ func = call["function"] or {}+ total += self.count_text(coerce_countable_text(func.get("name")))+ total += self.count_text(coerce_countable_text(func.get("arguments")))if "id" in call:- total += self.count_text(call["id"])+ total += self.count_text(coerce_countable_text(call["id"]))return totaldef _count_function_call(self, function_call: dict[str, Any]) -> int:"""Count tokens in legacy function call."""total = 4 # Function call overhead- total += self.count_text(function_call.get("name", ""))- total += self.count_text(function_call.get("arguments", ""))+ total += self.count_text(coerce_countable_text(function_call.get("name")))+ total += self.count_text(coerce_countable_text(function_call.get("arguments")))return totaldef encode(self, text: str) -> list[int]:diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.pyindex 87a4ef0fa..1aa987037 100644--- a/headroom/tokenizers/tiktoken_counter.py+++ b/headroom/tokenizers/tiktoken_counter.py@@ -16,7 +16,7 @@ import threadingfrom functools import lru_cachefrom typing import Any-from .base import BaseTokenizer+from .base import BaseTokenizer, coerce_countable_textlogger = logging.getLogger(__name__)@@ -316,24 +316,25 @@ class TiktokenCounter(BaseTokenizer):elif isinstance(part, str):total += self.count_text(part)elif key == "role":- total += self.count_text(value)+ total += self.count_text(coerce_countable_text(value))elif key == "name":- total += self.count_text(value)+ total += self.count_text(coerce_countable_text(value))total += 1 # Name adds 1 tokenelif key == "tool_calls":- for tool_call in value:+ for tool_call in value or []:total += 3 # Tool call overheadif "function" in tool_call:- func = tool_call["function"]- total += self.count_text(func.get("name", ""))- total += self.count_text(func.get("arguments", ""))+ func = tool_call["function"] or {}+ total += self.count_text(coerce_countable_text(func.get("name")))+ total += self.count_text(coerce_countable_text(func.get("arguments")))if "id" in tool_call:- total += self.count_text(tool_call["id"])+ total += self.count_text(coerce_countable_text(tool_call["id"]))elif key == "tool_call_id":- total += self.count_text(value)+ total += self.count_text(coerce_countable_text(value))elif key == "function_call":- total += self.count_text(value.get("name", ""))- total += self.count_text(value.get("arguments", ""))+ value = value or {}+ total += self.count_text(coerce_countable_text(value.get("name")))+ total += self.count_text(coerce_countable_text(value.get("arguments")))# Every reply is primed with assistanttotal += self.REPLY_OVERHEADdiff --git a/tests/test_tool_call_arguments_not_a_string.py b/tests/test_tool_call_arguments_not_a_string.pynew file mode 100644index 000000000..4d1d2eb3c--- /dev/null+++ b/tests/test_tool_call_arguments_not_a_string.py@@ -0,0 +1,97 @@+"""Token counters must survive a tool_call whose fields aren't strings (GH #2782).++``function.arguments`` is a JSON *string* per the OpenAI spec, but+OpenAI-compatible upstreams do emit ``None`` or a raw object there. Every counter+passed the value straight to ``tiktoken.encode()``, which raises+``TypeError: expected string or buffer`` — so ``/v1/compress`` failed the whole+request with a 503. Worse, the malformed message stays in conversation history,+so every later request replaying that history failed too, regardless of provider.++``arguments: None`` stopped raising once ``count_text`` grew its falsy guard, but+any *truthy* non-string (``{"path": "x"}``, ``5``) still crashed all four+counters. The fix is ``coerce_countable_text`` at the tool-call field sites, so a+dict is priced roughly like the JSON string it should have been rather than+either crashing or silently counting as zero.+"""++from __future__ import annotations++import json++import pytest++from headroom.providers.anthropic import AnthropicProvider+from headroom.providers.openai import OpenAITokenCounter+from headroom.providers.openai_compatible import OpenAICompatibleTokenCounter+from headroom.tokenizers.base import coerce_countable_text+from headroom.tokenizers.tiktoken_counter import TiktokenCounter+++def _counters():+ return {+ "openai": OpenAITokenCounter("gpt-4o"),+ "openai_compatible": OpenAICompatibleTokenCounter("gpt-4o"),+ "anthropic": AnthropicProvider(warn=False).get_token_counter("claude-sonnet-4-6"),+ "tiktoken": TiktokenCounter("gpt-4o"),+ }+++def _message(arguments: object) -> dict:+ return {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "read_file", "arguments": arguments},+ }+ ],+ }+++ "arguments",+ [None, {"path": "x"}, ["a", "b"], 5, True, 1.5],+ ids=["none", "dict", "list", "int", "bool", "float"],+)+def test_non_string_arguments_do_not_raise(arguments: object) -> None:+ """The reported crash: 503 + TypeError out of tiktoken.encode."""+ for name, counter in _counters().items():+ got = counter.count_messages([_message(arguments)])+ assert got > 0, f"{name} priced the whole message at {got}"+++def test_object_arguments_are_priced_like_their_json_form() -> None:+ """Not just non-crashing: a dict must not silently count as zero."""+ payload = {"path": "src/very/long/path/to/a/file.py", "start": 1, "end": 400}+ for name, counter in _counters().items():+ as_object = counter.count_messages([_message(payload)])+ as_json = counter.count_messages([_message(json.dumps(payload))])+ assert abs(as_object - as_json) <= 5, f"{name}: {as_object} vs {as_json}"+++def test_null_function_and_id_do_not_raise() -> None:+ """``{"function": null}`` / ``{"id": null}`` reach the same encode path."""+ message = {"role": "assistant", "tool_calls": [{"id": None, "function": None}]}+ for name, counter in _counters().items():+ assert counter.count_messages([message]) > 0, name+++def test_legacy_function_call_with_object_arguments_does_not_raise() -> None:+ message = {"role": "assistant", "function_call": {"name": "f", "arguments": {"a": 1}}}+ for name, counter in _counters().items():+ assert counter.count_messages([message]) > 0, name+++def test_oversized_object_arguments_are_bounded() -> None:+ """A malformed upstream must not turn an estimate into a megabyte encode."""+ huge = {"blob": "x" * 5_000_000}+ assert len(coerce_countable_text(huge)) <= 200_000+++def test_string_arguments_are_untouched() -> None:+ """The control: the spec-compliant shape must not move."""+ args = json.dumps({"path": "a.py"})+ assert coerce_countable_text(args) == args+ assert coerce_countable_text(None) == ""