Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(providers): cache Google/Cohere token counters across requests (#3273)

Abhay Singhcc6a07cd651a
diff --git a/headroom/providers/cohere.py b/headroom/providers/cohere.py
index d557cc511..49e0ec571 100644
--- a/headroom/providers/cohere.py
+++ b/headroom/providers/cohere.py
@@ -230,6 +230,11 @@ class CohereProvider(Provider):
If provided, uses tokenize API for accurate counts.
"""
self._client = client
+ # Cache counters per model so their internal TokenCountCache persists
+ # across requests, like the Anthropic and OpenAI providers. A fresh
+ # counter per call threw the cache away every request, re-tokenizing a
+ # stable prefix (system prompt + tools) on every turn.
+ self._token_counters: dict[str, TokenCounter] = {}
@property
def name(self) -> str:
@@ -256,7 +261,9 @@ class CohereProvider(Provider):
f"Model '{model}' is not recognized as a Cohere model. "
f"Supported models: {list(_CONTEXT_LIMITS.keys())}"
)
- return CohereTokenCounter(model, client=self._client)
+ if model not in self._token_counters:
+ self._token_counters[model] = CohereTokenCounter(model, client=self._client)
+ return self._token_counters[model]
def get_context_limit(self, model: str) -> int:
"""Get context limit for a Cohere model.
diff --git a/headroom/providers/google.py b/headroom/providers/google.py
index 4dc919df3..436196f36 100644
--- a/headroom/providers/google.py
+++ b/headroom/providers/google.py
@@ -265,6 +265,12 @@ class GoogleProvider(Provider):
If provided, uses countTokens API for accurate counts.
"""
self._client = client
+ # Cache counters per model so their internal TokenCountCache (and any
+ # lazily-built genai model) persist across requests, like the Anthropic
+ # and OpenAI providers. Rebuilding a fresh counter every call threw the
+ # cache away every request, so a stable prefix (system prompt + tools)
+ # was re-tokenized on every turn.
+ self._token_counters: dict[str, TokenCounter] = {}
@property
def name(self) -> str:
@@ -291,7 +297,9 @@ class GoogleProvider(Provider):
f"Model '{model}' is not recognized as a Google model. "
f"Supported models: {list(_CONTEXT_LIMITS.keys())}"
)
- return GeminiTokenCounter(model, client=self._client)
+ if model not in self._token_counters:
+ self._token_counters[model] = GeminiTokenCounter(model, client=self._client)
+ return self._token_counters[model]
def get_context_limit(self, model: str) -> int:
"""Get context limit for a Gemini model.
diff --git a/tests/test_providers/test_cohere.py b/tests/test_providers/test_cohere.py
index b460e962c..b78c7c0c2 100644
--- a/tests/test_providers/test_cohere.py
+++ b/tests/test_providers/test_cohere.py
@@ -39,6 +39,16 @@ class TestCohereProvider:
count = counter.count_text("Hello, world!")
assert count > 0
+ def test_get_token_counter_is_cached_per_model(self, provider):
+ """Counters are memoized per model so their internal token-count cache
+ survives across requests, matching the Anthropic/OpenAI providers."""
+ first = provider.get_token_counter("command-r-plus")
+ assert provider.get_token_counter("command-r-plus") is first
+ # A different model gets its own counter.
+ other = provider.get_token_counter("command-r")
+ assert other is not first
+ assert provider.get_token_counter("command-r") is other
+
def test_get_context_limit_command_a(self, provider):
"""Test context limit for Command A (256K)."""
limit = provider.get_context_limit("command-a")
diff --git a/tests/test_providers/test_google.py b/tests/test_providers/test_google.py
new file mode 100644
index 000000000..4339e5218
--- /dev/null
+++ b/tests/test_providers/test_google.py
@@ -0,0 +1,38 @@
+"""Tests for Google (Gemini) provider."""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.providers import GoogleProvider
+
+
+class TestGoogleProvider:
+ """Tests for GoogleProvider."""
+
+ @pytest.fixture
+ def provider(self):
+ """Create Google provider without client (estimation mode)."""
+ return GoogleProvider()
+
+ def test_name(self, provider):
+ assert provider.name == "google"
+
+ def test_get_token_counter(self, provider):
+ counter = provider.get_token_counter("gemini-2.0-flash")
+ assert counter is not None
+ assert counter.count_text("Hello, world!") > 0
+
+ def test_get_token_counter_is_cached_per_model(self, provider):
+ """Counters are memoized per model so their internal token-count cache
+ survives across requests, matching the Anthropic/OpenAI providers."""
+ first = provider.get_token_counter("gemini-2.0-flash")
+ assert provider.get_token_counter("gemini-2.0-flash") is first
+ # A different model gets its own counter.
+ other = provider.get_token_counter("gemini-1.5-pro")
+ assert other is not first
+ assert provider.get_token_counter("gemini-1.5-pro") is other
+
+ def test_get_token_counter_rejects_unknown_model(self, provider):
+ with pytest.raises(ValueError):
+ provider.get_token_counter("gpt-4o")