Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(gateway): keep input_includes_cache when re-drive usage is merged (#4423)

HetCreep3f941446bfbe
diff --git a/headroom/proxy/gateway_turn.py b/headroom/proxy/gateway_turn.py
index 51f169622..c1a91275b 100644
--- a/headroom/proxy/gateway_turn.py
+++ b/headroom/proxy/gateway_turn.py
@@ -257,6 +257,15 @@ class NormalizedUsage:
return True
return self.cache_read is not None and self.cache_write is not None
+ def _merged_includes_cache(self, other: NormalizedUsage) -> bool:
+ """Accounting dialect across rounds. A usage block without cache fields
+ carries no dialect (``normalize_usage`` defaults it to inclusive, which
+ is wrong for a cacheless Anthropic round), so the round that reported
+ cache fields decides; the latest such round wins."""
+ if other.has_cache_signal or not self.has_cache_signal:
+ return other.input_includes_cache
+ return self.input_includes_cache
+
def merged_with(self, other: NormalizedUsage) -> NormalizedUsage:
"""Sum the billed counters across re-drive rounds; cache fields come
from the latest round that carried them (the last provider call is the
@@ -272,6 +281,7 @@ class NormalizedUsage:
output_tokens=_sum(self.output_tokens, other.output_tokens),
cache_read=other.cache_read if other.has_cache_signal else self.cache_read,
cache_write=other.cache_write if other.has_cache_signal else self.cache_write,
+ input_includes_cache=self._merged_includes_cache(other),
)
def summed_with(self, other: NormalizedUsage) -> NormalizedUsage:
@@ -290,6 +300,7 @@ class NormalizedUsage:
output_tokens=_sum(self.output_tokens, other.output_tokens),
cache_read=_sum(self.cache_read, other.cache_read),
cache_write=_sum(self.cache_write, other.cache_write),
+ input_includes_cache=self._merged_includes_cache(other),
)
def as_anthropic(self) -> dict[str, int]:
diff --git a/tests/test_gateway_turn_unit.py b/tests/test_gateway_turn_unit.py
index 6bb740047..6f410b412 100644
--- a/tests/test_gateway_turn_unit.py
+++ b/tests/test_gateway_turn_unit.py
@@ -530,6 +530,45 @@ def test_merged_with_sums_billed_and_keeps_latest_cache() -> None:
assert c.cache_read == 60 # a signal-free round does not erase the last signal
+def test_redrive_merges_keep_input_includes_cache() -> None:
+ """Re-drive rounds of an OpenAI-shaped turn must not be billed as Anthropic-shaped."""
+ first = normalize_usage(
+ {
+ "prompt_tokens": 100_000,
+ "completion_tokens": 10,
+ "prompt_tokens_details": {"cached_tokens": 80_000},
+ }
+ )
+ second = normalize_usage(
+ {
+ "prompt_tokens": 100_000,
+ "completion_tokens": 10,
+ "prompt_tokens_details": {"cached_tokens": 80_000},
+ }
+ )
+ for merged in (first.merged_with(second), first.summed_with(second)):
+ assert merged.input_includes_cache is True
+ billed = gt.complete_outcome(_outcome(), first.summed_with(second), status=200, latency_ms=1)
+ assert billed.provider_input_tokens == 200_000
+ assert billed.uncached_input_tokens == 40_000
+
+
+def test_redrive_merges_keep_anthropic_dialect_after_cacheless_round() -> None:
+ """A cacheless Anthropic round must not make a later cached round look inclusive."""
+ cacheless = normalize_usage({"input_tokens": 100, "output_tokens": 5})
+ cached = normalize_usage(
+ {"input_tokens": 10, "output_tokens": 5, "cache_read_input_tokens": 80}
+ )
+ for first, second in ((cacheless, cached), (cached, cacheless)):
+ for merged in (first.merged_with(second), first.summed_with(second)):
+ assert merged.input_includes_cache is False
+ billed = gt.complete_outcome(
+ _outcome(), first.summed_with(second), status=200, latency_ms=1
+ )
+ assert billed.provider_input_tokens == 190
+ assert billed.uncached_input_tokens == 110
+
+
# --------------------------------------------------------------------------- #
# Capabilities, obligations, outcome completion, compaction #
# --------------------------------------------------------------------------- #