Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(providers): cache Google/Cohere token counters across requests (#3273)
Abhay Singhcc6a07cd651a
diff --git a/headroom/providers/cohere.py b/headroom/providers/cohere.pyindex d557cc511..49e0ec571 100644--- a/headroom/providers/cohere.py+++ b/headroom/providers/cohere.py@@ -230,6 +230,11 @@ class CohereProvider(Provider):If provided, uses tokenize API for accurate counts."""self._client = client+ # Cache counters per model so their internal TokenCountCache persists+ # across requests, like the Anthropic and OpenAI providers. A fresh+ # counter per call threw the cache away every request, re-tokenizing a+ # stable prefix (system prompt + tools) on every turn.+ self._token_counters: dict[str, TokenCounter] = {}@propertydef name(self) -> str:@@ -256,7 +261,9 @@ class CohereProvider(Provider):f"Model '{model}' is not recognized as a Cohere model. "f"Supported models: {list(_CONTEXT_LIMITS.keys())}")- return CohereTokenCounter(model, client=self._client)+ if model not in self._token_counters:+ self._token_counters[model] = CohereTokenCounter(model, client=self._client)+ return self._token_counters[model]def get_context_limit(self, model: str) -> int:"""Get context limit for a Cohere model.diff --git a/headroom/providers/google.py b/headroom/providers/google.pyindex 4dc919df3..436196f36 100644--- a/headroom/providers/google.py+++ b/headroom/providers/google.py@@ -265,6 +265,12 @@ class GoogleProvider(Provider):If provided, uses countTokens API for accurate counts."""self._client = client+ # Cache counters per model so their internal TokenCountCache (and any+ # lazily-built genai model) persist across requests, like the Anthropic+ # and OpenAI providers. Rebuilding a fresh counter every call threw the+ # cache away every request, so a stable prefix (system prompt + tools)+ # was re-tokenized on every turn.+ self._token_counters: dict[str, TokenCounter] = {}@propertydef name(self) -> str:@@ -291,7 +297,9 @@ class GoogleProvider(Provider):f"Model '{model}' is not recognized as a Google model. "f"Supported models: {list(_CONTEXT_LIMITS.keys())}")- return GeminiTokenCounter(model, client=self._client)+ if model not in self._token_counters:+ self._token_counters[model] = GeminiTokenCounter(model, client=self._client)+ return self._token_counters[model]def get_context_limit(self, model: str) -> int:"""Get context limit for a Gemini model.diff --git a/tests/test_providers/test_cohere.py b/tests/test_providers/test_cohere.pyindex b460e962c..b78c7c0c2 100644--- a/tests/test_providers/test_cohere.py+++ b/tests/test_providers/test_cohere.py@@ -39,6 +39,16 @@ class TestCohereProvider:count = counter.count_text("Hello, world!")assert count > 0+ def test_get_token_counter_is_cached_per_model(self, provider):+ """Counters are memoized per model so their internal token-count cache+ survives across requests, matching the Anthropic/OpenAI providers."""+ first = provider.get_token_counter("command-r-plus")+ assert provider.get_token_counter("command-r-plus") is first+ # A different model gets its own counter.+ other = provider.get_token_counter("command-r")+ assert other is not first+ assert provider.get_token_counter("command-r") is other+def test_get_context_limit_command_a(self, provider):"""Test context limit for Command A (256K)."""limit = provider.get_context_limit("command-a")diff --git a/tests/test_providers/test_google.py b/tests/test_providers/test_google.pynew file mode 100644index 000000000..4339e5218--- /dev/null+++ b/tests/test_providers/test_google.py@@ -0,0 +1,38 @@+"""Tests for Google (Gemini) provider."""++from __future__ import annotations++import pytest++from headroom.providers import GoogleProvider+++class TestGoogleProvider:+ """Tests for GoogleProvider."""++ @pytest.fixture+ def provider(self):+ """Create Google provider without client (estimation mode)."""+ return GoogleProvider()++ def test_name(self, provider):+ assert provider.name == "google"++ def test_get_token_counter(self, provider):+ counter = provider.get_token_counter("gemini-2.0-flash")+ assert counter is not None+ assert counter.count_text("Hello, world!") > 0++ def test_get_token_counter_is_cached_per_model(self, provider):+ """Counters are memoized per model so their internal token-count cache+ survives across requests, matching the Anthropic/OpenAI providers."""+ first = provider.get_token_counter("gemini-2.0-flash")+ assert provider.get_token_counter("gemini-2.0-flash") is first+ # A different model gets its own counter.+ other = provider.get_token_counter("gemini-1.5-pro")+ assert other is not first+ assert provider.get_token_counter("gemini-1.5-pro") is other++ def test_get_token_counter_rejects_unknown_model(self, provider):+ with pytest.raises(ValueError):+ provider.get_token_counter("gpt-4o")