Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(metrics): count an inferred OpenAI cache write as new input once (#4285)
gglucass290f72c550ba
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex a2534803d..a5710a113 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -903,6 +903,10 @@ class PrometheusMetrics:from headroom.telemetry.context import MAX_DISTINCT_MODELSledger_input_tokens = input_tokens if local_input_tokens is None else local_input_tokens+ # An inferred write is the uncached slice under another name (see+ # ``cache_inferred``), so it adds neither new input nor a billed write.+ billed_write_tokens = 0 if cache_inferred else cache_write_tokens+ new_input_tokens = int(uncached_input_tokens) + int(billed_write_tokens)# Post-guard invariant (all providers): Headroom never forwards a request# larger than the original — handlers revert any inflation before sending# (verified clean on the wire). So compression savings are >= 0; a negative@@ -1001,14 +1005,14 @@ class PrometheusMetrics:# New-input cohort, on the same predicate the ledger uses below.if uncached_input_tokens > 0 or cache_write_tokens > 0:- self.new_input_tokens_total += uncached_input_tokens + cache_write_tokens+ self.new_input_tokens_total += new_input_tokensself.new_input_saved_tokens_total += max(0, int(tokens_saved))# Track provider-specific prefix cache metricsif cache_read_tokens > 0 or cache_write_tokens > 0:pc = self.cache_by_provider[provider]pc["cache_read_tokens"] += cache_read_tokens- pc["cache_write_tokens"] += cache_write_tokens+ pc["cache_write_tokens"] += billed_write_tokenspc["cache_write_5m_tokens"] += cache_write_5m_tokenspc["cache_write_1h_tokens"] += cache_write_1h_tokensif cache_write_5m_tokens > 0:@@ -1199,9 +1203,7 @@ class PrometheusMetrics:# show the same new-input rate the dashboard headline does.# Omitted when there is no cache breakdown (e.g. Bedrock), so# the ledger never divides savings by themselves.- new_input_tokens=(- int(uncached_input_tokens) + int(cache_write_tokens) if has_new_input else None- ),+ new_input_tokens=new_input_tokens if has_new_input else None,deferred_tokens=deferral_saved,)diff --git a/tests/test_startup_log_noise.py b/tests/test_startup_log_noise.pyindex b04c72b93..9e838d774 100644--- a/tests/test_startup_log_noise.py+++ b/tests/test_startup_log_noise.py@@ -119,8 +119,11 @@ class TestLiteLLMLogSuppression:import osmonkeypatch.delenv("LITELLM_SUPPRESS_DEBUG_INFO", raising=False)- sys.modules.pop("headroom.providers.litellm", None)- sys.modules.pop("litellm", None)+ # delitem, not pop: teardown puts the real modules back. A bare pop made+ # every later `import litellm` load a second copy, so a test pinning a+ # rate on it missed modules still holding the first (CostTracker).+ monkeypatch.delitem(sys.modules, "headroom.providers.litellm", raising=False)+ monkeypatch.delitem(sys.modules, "litellm", raising=False)original_import = builtins.__import__fake_litellm = ModuleType("litellm")diff --git a/tests/test_stats_new_input_savings_rate.py b/tests/test_stats_new_input_savings_rate.pyindex 14b94881b..5a3d9fbcd 100644--- a/tests/test_stats_new_input_savings_rate.py+++ b/tests/test_stats_new_input_savings_rate.py@@ -176,3 +176,112 @@ def test_stats_and_ledger_share_one_new_input_cohort(tmp_path, monkeypatch):)# The cache-read-only turn's savings still count on the whole-wire figure.assert tokens["saved"] == 600+++def test_inferred_openai_write_counts_as_new_input_once(tmp_path, monkeypatch):+ """OpenAI paths with no reported write pass ``cache_inferred=True`` with the+ write equal to the uncached slice (input - cached): one set of tokens under+ two names. Adding both doubled new input, so a true 50% rate read 33%, and+ the provider row showed (and priced a premium on) a write nobody billed."""+ ledger_path = tmp_path / "savings_events.jsonl"+ monkeypatch.setattr(savings_ledger, "_resolve_path", lambda path=None: ledger_path)++ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy+ asyncio.run(+ proxy.metrics.record_request(+ provider="openai",+ model="gpt-5.4",+ input_tokens=2_000,+ output_tokens=10,+ tokens_saved=1_000,+ latency_ms=10.0,+ cache_read_tokens=1_000,+ cache_write_tokens=1_000,+ uncached_input_tokens=1_000,+ cache_inferred=True,+ )+ )+ stats = client.get("/stats").json()++ lifetime = savings_ledger.aggregate_savings(path=ledger_path).lifetime+ tokens = stats["tokens"]+ totals = stats["prefix_cache"]["totals"]+ openai_row = stats["prefix_cache"]["by_provider"]["openai"]++ assert tokens["new_input_tokens"] == 1_000+ assert lifetime["new_input_tokens"] == 1_000+ # 1,000 saved / (1,000 new + 1,000 saved), not 1,000 / 3,000.+ assert tokens["new_input_savings_percent"] == 50.0+ assert lifetime["new_input_savings_percent"] == 50.0+ assert openai_row["cache_write_tokens"] == 0+ assert openai_row["hit_rate"] == 50.0+ new_input = totals["cache_write_tokens"] + totals["uncached_input_tokens"]+ assert new_input == totals["new_input_tokens"]+++def test_inferred_openai_write_carries_no_write_premium(tmp_path, monkeypatch):+ """The OpenAI row prices its write premium from the cost tracker's model.+ Where the catalog gives that model a write rate above input (gpt-5.6+ publishes 1.25x), an inferred write counted as a write was charged the+ premium on tokens OpenAI billed as plain input."""+ import litellm++ from headroom.pricing.counterfactual import resolve_rates+ from headroom.proxy.outcome import RequestOutcome++ ledger_path = tmp_path / "savings_events.jsonl"+ monkeypatch.setattr(savings_ledger, "_resolve_path", lambda path=None: ledger_path)+ # A real catalog id with known rates, so the live catalog cannot move them.+ model = "gpt-4o"+ monkeypatch.setitem(+ litellm.model_cost,+ model,+ {+ "input_cost_per_token": 4e-6,+ "output_cost_per_token": 3.2e-5,+ "cache_read_input_token_cost": 4e-7,+ "cache_creation_input_token_cost": 5e-6,+ "litellm_provider": "openai",+ "mode": "chat",+ },+ )+ resolve_rates.cache_clear()+ try:+ with _make_client(tmp_path, monkeypatch) as client:+ proxy = client.app.state.proxy+ # The Chat Completions handler's outcome: no reported write, so the+ # write is inferred as input - cached. Feeds metrics AND the cost+ # tracker, which supplies the price build_prefix_cache_stats uses.+ asyncio.run(+ proxy._record_request_outcome(+ RequestOutcome(+ request_id="req_inferred_write",+ provider="openai",+ model=model,+ original_tokens=2_000,+ optimized_tokens=2_000,+ provider_input_tokens=2_000,+ output_tokens=10,+ tokens_saved=0,+ attempted_input_tokens=2_000,+ cache_read_tokens=1_000,+ cache_write_tokens=1_000,+ uncached_input_tokens=1_000,+ cache_inferred=True,+ )+ )+ )+ openai_row = client.get("/stats").json()["prefix_cache"]["by_provider"]["openai"]+ finally:+ resolve_rates.cache_clear()++ # The catalog write rate is in force, so a write would be charged 25%.+ assert openai_row["cache_pricing_source"] == "catalog"+ assert openai_row["write_premium"] == "25%"+ # 1,000 reads x $4/M x 90% read discount.+ assert openai_row["savings_usd"] == 0.0036+ # Counted as a write: 1,000 x $4/M x 25% = $0.001 premium, net $0.0026.+ assert openai_row["write_premium_usd"] == 0.0+ assert openai_row["net_savings_usd"] == 0.0036+ assert openai_row["cache_write_tokens"] == 0