Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/cost): record each request's savings exactly once (drop 3 double-counts) (#2545)
Tejas Chopra0845b26ee61c
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex b80d6a6bc..1ddd047fb 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -67,7 +67,7 @@ from headroom.proxy.auth_mode import (should_stamp_codex_client,)from headroom.proxy.compression_decision import CompressionDecision-from headroom.proxy.cost import _summarize_transforms, header_safe_transforms+from headroom.proxy.cost import header_safe_transformsfrom headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_valuefrom headroom.proxy.image_isolation import run_image_compression_isolatedfrom headroom.proxy.outcome import RequestOutcome@@ -4135,17 +4135,9 @@ class OpenAIHandlerMixin:# OpenAI has no write penalty — uncached = total - cacheduncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)- # (record_tokens clamps negative savings to 0 universally — the- # forwarded request is never larger than the original.)- if self.cost_tracker:- self.cost_tracker.record_tokens(- model,- tokens_saved,- optimized_tokens,- cache_read_tokens=cache_read_tokens,- cache_write_tokens=cache_write_tokens,- uncached_tokens=uncached_input_tokens,- )+ # Cost is recorded exactly once by the outcome funnel below+ # (_record_request_outcome -> emit_request_outcome -> cost_tracker.+ # record_tokens); recording here too double-counted spend + budget.# Memory: handle memory tool calls in OpenAI Chat Completions response.# After executing tools, send a continuation request so the model@@ -5213,27 +5205,14 @@ class OpenAIHandlerMixin:f"[{request_id}] Memory tool handling failed (responses): {e}")- if self.cost_tracker:- cache_write_tokens = _infer_openai_cache_write_tokens(- total_input_tokens,- cache_read_tokens,- )- uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)- # (record_tokens clamps negative savings to 0 universally.)- self.cost_tracker.record_tokens(- model,- tokens_saved,- total_input_tokens,- cache_read_tokens=cache_read_tokens,- cache_write_tokens=cache_write_tokens,- uncached_tokens=uncached_input_tokens,- )- else:- cache_write_tokens = _infer_openai_cache_write_tokens(- total_input_tokens,- cache_read_tokens,- )- uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)+ # Cost is recorded once by the outcome funnel below; here we only+ # compute the cache-write / uncached split the funnel needs.+ # (Recording here too double-counted spend + budget on this path.)+ cache_write_tokens = _infer_openai_cache_write_tokens(+ total_input_tokens,+ cache_read_tokens,+ )+ uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)effective_optimized_tokens = (total_input_tokens if total_input_tokens > 0 else optimized_tokens@@ -7266,41 +7245,10 @@ class OpenAIHandlerMixin:))- # Structured PERF log line so ``headroom perf``- # counts this Codex turn. Pre-P2 this emit was- # missing, which is why Codex traffic showed up- # as ``Requests: 0`` in the perf report even- # under heavy load — the same visibility bug- # class as #327's "Cache write: 0" report.- _perf_input_tokens = max(0, input_delta)- _perf_cache_read = max(0, cache_read_delta)- _perf_cache_write = max(0, cache_write_delta)- _perf_cache_hit_pct = (- round(- _perf_cache_read / (_perf_cache_read + _perf_cache_write) * 100- )- if (_perf_cache_read + _perf_cache_write) > 0- else 0- )- _perf_tok_before = _perf_input_tokens + max(0, saved_delta)- _perf_num_msgs = (- len(body.get("messages") or body.get("input") or [])- if isinstance(body, dict)- else 0- )- logger.info(- f"[{request_id}] PERF "- f"model={model_for_metrics} msgs={_perf_num_msgs} "- f"tok_before={_perf_tok_before} "- f"tok_after={_perf_input_tokens} "- f"tok_saved={max(0, saved_delta)} "- f"cache_read={_perf_cache_read} "- f"cache_write={_perf_cache_write} "- f"cache_hit_pct={_perf_cache_hit_pct} "- f"opt_ms={overhead_delta_ms:.0f} "- f"transforms={_summarize_transforms(transforms_applied)} "- f"client={client or ''}"- )+ # The PERF line for this Codex turn is emitted once by+ # the outcome funnel above (emit_request_outcome), using+ # the same per-turn deltas — a second emit here duplicated+ # every WS turn in `headroom perf` (double saved + requests).ws_recorded_input_tokens_total = ws_input_tokens_totalws_recorded_output_tokens_total = ws_output_tokens_totaldiff --git a/tests/test_openai_chat_turn_hooks.py b/tests/test_openai_chat_turn_hooks.pyindex 0d93a0873..3a7a373fa 100644--- a/tests/test_openai_chat_turn_hooks.py+++ b/tests/test_openai_chat_turn_hooks.py@@ -308,6 +308,36 @@ def test_in_place_message_fold_is_counted():assert any(int(lg.get("tokens_saved", 0) or 0) > 0 for lg in logs), logs+def test_cost_recorded_once_not_twice_nonstreaming():+ """Regression: the OpenAI chat non-streaming direct path recorded cost TWICE —+ an explicit `cost_tracker.record_tokens` plus the outcome funnel's own call —+ doubling spend, request count, and budget consumption. It must fire once."""++ async def fake_retry(method, url, headers, body, *args, **kwargs):+ return httpx.Response(+ 200, json=_final_response(), headers={"content-type": "application/json"}+ )++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._retry_request = fake_retry+ ct = client.app.state.proxy.cost_tracker+ calls = {"n": 0}+ _orig = ct.record_tokens++ def _counting(*a, **k):+ calls["n"] += 1+ return _orig(*a, **k)++ ct.record_tokens = _counting+ resp = _post(+ client,+ {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], "stream": False},+ )+ assert resp.status_code == 200, resp.text+ assert calls["n"] == 1, f"cost recorded {calls['n']}x — double-count regression"++def test_direct_path_noop_when_no_hook_registered():# No hook registered -> byte-identical passthrough, single upstream call.calls = {"n": 0}