Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(gateway): keep input_includes_cache when re-drive usage is merged (#4423)
HetCreep3f941446bfbe
diff --git a/headroom/proxy/gateway_turn.py b/headroom/proxy/gateway_turn.pyindex 51f169622..c1a91275b 100644--- a/headroom/proxy/gateway_turn.py+++ b/headroom/proxy/gateway_turn.py@@ -257,6 +257,15 @@ class NormalizedUsage:return Truereturn self.cache_read is not None and self.cache_write is not None+ def _merged_includes_cache(self, other: NormalizedUsage) -> bool:+ """Accounting dialect across rounds. A usage block without cache fields+ carries no dialect (``normalize_usage`` defaults it to inclusive, which+ is wrong for a cacheless Anthropic round), so the round that reported+ cache fields decides; the latest such round wins."""+ if other.has_cache_signal or not self.has_cache_signal:+ return other.input_includes_cache+ return self.input_includes_cache+def merged_with(self, other: NormalizedUsage) -> NormalizedUsage:"""Sum the billed counters across re-drive rounds; cache fields comefrom the latest round that carried them (the last provider call is the@@ -272,6 +281,7 @@ class NormalizedUsage:output_tokens=_sum(self.output_tokens, other.output_tokens),cache_read=other.cache_read if other.has_cache_signal else self.cache_read,cache_write=other.cache_write if other.has_cache_signal else self.cache_write,+ input_includes_cache=self._merged_includes_cache(other),)def summed_with(self, other: NormalizedUsage) -> NormalizedUsage:@@ -290,6 +300,7 @@ class NormalizedUsage:output_tokens=_sum(self.output_tokens, other.output_tokens),cache_read=_sum(self.cache_read, other.cache_read),cache_write=_sum(self.cache_write, other.cache_write),+ input_includes_cache=self._merged_includes_cache(other),)def as_anthropic(self) -> dict[str, int]:diff --git a/tests/test_gateway_turn_unit.py b/tests/test_gateway_turn_unit.pyindex 6bb740047..6f410b412 100644--- a/tests/test_gateway_turn_unit.py+++ b/tests/test_gateway_turn_unit.py@@ -530,6 +530,45 @@ def test_merged_with_sums_billed_and_keeps_latest_cache() -> None:assert c.cache_read == 60 # a signal-free round does not erase the last signal+def test_redrive_merges_keep_input_includes_cache() -> None:+ """Re-drive rounds of an OpenAI-shaped turn must not be billed as Anthropic-shaped."""+ first = normalize_usage(+ {+ "prompt_tokens": 100_000,+ "completion_tokens": 10,+ "prompt_tokens_details": {"cached_tokens": 80_000},+ }+ )+ second = normalize_usage(+ {+ "prompt_tokens": 100_000,+ "completion_tokens": 10,+ "prompt_tokens_details": {"cached_tokens": 80_000},+ }+ )+ for merged in (first.merged_with(second), first.summed_with(second)):+ assert merged.input_includes_cache is True+ billed = gt.complete_outcome(_outcome(), first.summed_with(second), status=200, latency_ms=1)+ assert billed.provider_input_tokens == 200_000+ assert billed.uncached_input_tokens == 40_000+++def test_redrive_merges_keep_anthropic_dialect_after_cacheless_round() -> None:+ """A cacheless Anthropic round must not make a later cached round look inclusive."""+ cacheless = normalize_usage({"input_tokens": 100, "output_tokens": 5})+ cached = normalize_usage(+ {"input_tokens": 10, "output_tokens": 5, "cache_read_input_tokens": 80}+ )+ for first, second in ((cacheless, cached), (cached, cacheless)):+ for merged in (first.merged_with(second), first.summed_with(second)):+ assert merged.input_includes_cache is False+ billed = gt.complete_outcome(+ _outcome(), first.summed_with(second), status=200, latency_ms=1+ )+ assert billed.provider_input_tokens == 190+ assert billed.uncached_input_tokens == 110++# --------------------------------------------------------------------------- ## Capabilities, obligations, outcome completion, compaction ## --------------------------------------------------------------------------- #