Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(metrics): count an inferred OpenAI cache write as new input once (#4285)

gglucass290f72c550ba
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index a2534803d..a5710a113 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -903,6 +903,10 @@ class PrometheusMetrics:
from headroom.telemetry.context import MAX_DISTINCT_MODELS
ledger_input_tokens = input_tokens if local_input_tokens is None else local_input_tokens
+ # An inferred write is the uncached slice under another name (see
+ # ``cache_inferred``), so it adds neither new input nor a billed write.
+ billed_write_tokens = 0 if cache_inferred else cache_write_tokens
+ new_input_tokens = int(uncached_input_tokens) + int(billed_write_tokens)
# Post-guard invariant (all providers): Headroom never forwards a request
# larger than the original — handlers revert any inflation before sending
# (verified clean on the wire). So compression savings are >= 0; a negative
@@ -1001,14 +1005,14 @@ class PrometheusMetrics:
# New-input cohort, on the same predicate the ledger uses below.
if uncached_input_tokens > 0 or cache_write_tokens > 0:
- self.new_input_tokens_total += uncached_input_tokens + cache_write_tokens
+ self.new_input_tokens_total += new_input_tokens
self.new_input_saved_tokens_total += max(0, int(tokens_saved))
# Track provider-specific prefix cache metrics
if cache_read_tokens > 0 or cache_write_tokens > 0:
pc = self.cache_by_provider[provider]
pc["cache_read_tokens"] += cache_read_tokens
- pc["cache_write_tokens"] += cache_write_tokens
+ pc["cache_write_tokens"] += billed_write_tokens
pc["cache_write_5m_tokens"] += cache_write_5m_tokens
pc["cache_write_1h_tokens"] += cache_write_1h_tokens
if cache_write_5m_tokens > 0:
@@ -1199,9 +1203,7 @@ class PrometheusMetrics:
# show the same new-input rate the dashboard headline does.
# Omitted when there is no cache breakdown (e.g. Bedrock), so
# the ledger never divides savings by themselves.
- new_input_tokens=(
- int(uncached_input_tokens) + int(cache_write_tokens) if has_new_input else None
- ),
+ new_input_tokens=new_input_tokens if has_new_input else None,
deferred_tokens=deferral_saved,
)
diff --git a/tests/test_startup_log_noise.py b/tests/test_startup_log_noise.py
index b04c72b93..9e838d774 100644
--- a/tests/test_startup_log_noise.py
+++ b/tests/test_startup_log_noise.py
@@ -119,8 +119,11 @@ class TestLiteLLMLogSuppression:
import os
monkeypatch.delenv("LITELLM_SUPPRESS_DEBUG_INFO", raising=False)
- sys.modules.pop("headroom.providers.litellm", None)
- sys.modules.pop("litellm", None)
+ # delitem, not pop: teardown puts the real modules back. A bare pop made
+ # every later `import litellm` load a second copy, so a test pinning a
+ # rate on it missed modules still holding the first (CostTracker).
+ monkeypatch.delitem(sys.modules, "headroom.providers.litellm", raising=False)
+ monkeypatch.delitem(sys.modules, "litellm", raising=False)
original_import = builtins.__import__
fake_litellm = ModuleType("litellm")
diff --git a/tests/test_stats_new_input_savings_rate.py b/tests/test_stats_new_input_savings_rate.py
index 14b94881b..5a3d9fbcd 100644
--- a/tests/test_stats_new_input_savings_rate.py
+++ b/tests/test_stats_new_input_savings_rate.py
@@ -176,3 +176,112 @@ def test_stats_and_ledger_share_one_new_input_cohort(tmp_path, monkeypatch):
)
# The cache-read-only turn's savings still count on the whole-wire figure.
assert tokens["saved"] == 600
+
+
+def test_inferred_openai_write_counts_as_new_input_once(tmp_path, monkeypatch):
+ """OpenAI paths with no reported write pass ``cache_inferred=True`` with the
+ write equal to the uncached slice (input - cached): one set of tokens under
+ two names. Adding both doubled new input, so a true 50% rate read 33%, and
+ the provider row showed (and priced a premium on) a write nobody billed."""
+ ledger_path = tmp_path / "savings_events.jsonl"
+ monkeypatch.setattr(savings_ledger, "_resolve_path", lambda path=None: ledger_path)
+
+ with _make_client(tmp_path, monkeypatch) as client:
+ proxy = client.app.state.proxy
+ asyncio.run(
+ proxy.metrics.record_request(
+ provider="openai",
+ model="gpt-5.4",
+ input_tokens=2_000,
+ output_tokens=10,
+ tokens_saved=1_000,
+ latency_ms=10.0,
+ cache_read_tokens=1_000,
+ cache_write_tokens=1_000,
+ uncached_input_tokens=1_000,
+ cache_inferred=True,
+ )
+ )
+ stats = client.get("/stats").json()
+
+ lifetime = savings_ledger.aggregate_savings(path=ledger_path).lifetime
+ tokens = stats["tokens"]
+ totals = stats["prefix_cache"]["totals"]
+ openai_row = stats["prefix_cache"]["by_provider"]["openai"]
+
+ assert tokens["new_input_tokens"] == 1_000
+ assert lifetime["new_input_tokens"] == 1_000
+ # 1,000 saved / (1,000 new + 1,000 saved), not 1,000 / 3,000.
+ assert tokens["new_input_savings_percent"] == 50.0
+ assert lifetime["new_input_savings_percent"] == 50.0
+ assert openai_row["cache_write_tokens"] == 0
+ assert openai_row["hit_rate"] == 50.0
+ new_input = totals["cache_write_tokens"] + totals["uncached_input_tokens"]
+ assert new_input == totals["new_input_tokens"]
+
+
+def test_inferred_openai_write_carries_no_write_premium(tmp_path, monkeypatch):
+ """The OpenAI row prices its write premium from the cost tracker's model.
+ Where the catalog gives that model a write rate above input (gpt-5.6
+ publishes 1.25x), an inferred write counted as a write was charged the
+ premium on tokens OpenAI billed as plain input."""
+ import litellm
+
+ from headroom.pricing.counterfactual import resolve_rates
+ from headroom.proxy.outcome import RequestOutcome
+
+ ledger_path = tmp_path / "savings_events.jsonl"
+ monkeypatch.setattr(savings_ledger, "_resolve_path", lambda path=None: ledger_path)
+ # A real catalog id with known rates, so the live catalog cannot move them.
+ model = "gpt-4o"
+ monkeypatch.setitem(
+ litellm.model_cost,
+ model,
+ {
+ "input_cost_per_token": 4e-6,
+ "output_cost_per_token": 3.2e-5,
+ "cache_read_input_token_cost": 4e-7,
+ "cache_creation_input_token_cost": 5e-6,
+ "litellm_provider": "openai",
+ "mode": "chat",
+ },
+ )
+ resolve_rates.cache_clear()
+ try:
+ with _make_client(tmp_path, monkeypatch) as client:
+ proxy = client.app.state.proxy
+ # The Chat Completions handler's outcome: no reported write, so the
+ # write is inferred as input - cached. Feeds metrics AND the cost
+ # tracker, which supplies the price build_prefix_cache_stats uses.
+ asyncio.run(
+ proxy._record_request_outcome(
+ RequestOutcome(
+ request_id="req_inferred_write",
+ provider="openai",
+ model=model,
+ original_tokens=2_000,
+ optimized_tokens=2_000,
+ provider_input_tokens=2_000,
+ output_tokens=10,
+ tokens_saved=0,
+ attempted_input_tokens=2_000,
+ cache_read_tokens=1_000,
+ cache_write_tokens=1_000,
+ uncached_input_tokens=1_000,
+ cache_inferred=True,
+ )
+ )
+ )
+ openai_row = client.get("/stats").json()["prefix_cache"]["by_provider"]["openai"]
+ finally:
+ resolve_rates.cache_clear()
+
+ # The catalog write rate is in force, so a write would be charged 25%.
+ assert openai_row["cache_pricing_source"] == "catalog"
+ assert openai_row["write_premium"] == "25%"
+ # 1,000 reads x $4/M x 90% read discount.
+ assert openai_row["savings_usd"] == 0.0036
+ # Counted as a write: 1,000 x $4/M x 25% = $0.001 premium, net $0.0026.
+ assert openai_row["write_premium_usd"] == 0.0
+ assert openai_row["net_savings_usd"] == 0.0036
+ assert openai_row["cache_write_tokens"] == 0