Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/openai): keep /v1/responses savings triple coherent when schema savings exceed message tokens (#3106)
gglucass7278b5cbf83f
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 398985c54..463963e35 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -5637,6 +5637,11 @@ class OpenAIHandlerMixin:# Token counting on converted messages (offloaded off the event loop — GH #1701)tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)+ # Messages-only count, preserved for the output shaper's turn/size+ # classifier: the accounting pair below may widen original_tokens by+ # the tools schema after compression, and shaper strata must not+ # shift when it does.+ message_input_tokens = original_tokens# Defaults below feed downstream telemetry and memory injection.# If optimization remains enabled, the Responses payload is compressed@@ -5880,6 +5885,10 @@ class OpenAIHandlerMixin:# gating already happened upstream (auth_mode classify,# CompressionPolicy resolve at request entry).if self.config.optimize and not _bypass:+ # Pre-compression tools reference: the compression pass is+ # copy-on-write on the payload, so this still points at the+ # original schemas after `body` is rebound below.+ _tools_before = body.get("tools")try:(body,@@ -5908,6 +5917,17 @@ class OpenAIHandlerMixin:if _modified:body_mutation_tracker.mark_mutated("responses_compression")tokens_saved = int(_tokens_saved)+ # tokens_saved includes tool schema/desc compaction, which+ # is measured against the tools array — content the+ # messages-only original_tokens count above never+ # included. Fold the tools schema into the pair so it+ # stays coherent (original - optimized == saved, savings+ # rate <= 100%); without this, schema-heavy turns clamp+ # optimized to 0 and record rates above 100%.+ if _tools_before:+ original_tokens += await asyncio.to_thread(+ tokenizer.count_text, _json_debug_dumps(_tools_before)+ )responses_conversation_key = _pre_compression_conversation_keyoptimized_tokens = max(0, original_tokens - tokens_saved)logger.info(@@ -5981,7 +6001,7 @@ class OpenAIHandlerMixin:_http_conversation_key = request.headers.get("x-headroom-session-id")_shape_result = _shape_openai_responses_for_output(body,- input_tokens=original_tokens,+ input_tokens=message_input_tokens,model=str(model or ""),conversation_key=(f"header:x-headroom-session-id:{_http_conversation_key}"diff --git a/tests/test_responses_savings_denominator.py b/tests/test_responses_savings_denominator.pynew file mode 100644index 000000000..c02f30579--- /dev/null+++ b/tests/test_responses_savings_denominator.py@@ -0,0 +1,169 @@+"""Recorded /v1/responses savings must form a coherent triple.++``tokens_saved`` on the Responses HTTP path includes tool schema/desc+compaction, which is measured against the ``tools`` array — content the+messages-only ``original_tokens`` count never included. Deriving+``optimized_tokens = max(0, original - saved)`` from that mismatched pair+clamped to 0 on schema-heavy turns and recorded savings rates above 100%+(surfaced in the desktop feed as e.g. "4,436 → 0 (208.4%)"). The fix folds+the tools schema into the pair, so ``original - optimized == saved`` and+the rate stays <= 100%.+"""++from __future__ import annotations++import pytest++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++import httpx+from fastapi.testclient import TestClient++from headroom.proxy.loopback_guard import require_loopback+from headroom.proxy.server import ProxyConfig, create_app++_TOKENS_SAVED = 5_000+++def _make_app():+ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ app.dependency_overrides[require_loopback] = lambda: None+ return app+++def _fake_upstream_response(url: str) -> httpx.Response:+ return httpx.Response(+ 200,+ json={+ "id": "resp_test",+ "object": "response",+ "output": [],+ "usage": {"input_tokens": 12, "output_tokens": 3},+ },+ request=httpx.Request("POST", url),+ )+++def test_schema_heavy_compression_keeps_savings_triple_coherent():+ app = _make_app()+ server = app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs):+ return _fake_upstream_response(url)++ server._retry_request = _fake_retry++ async def _fake_compress(payload, **kwargs):+ # Simulate schema compaction: savings far larger than the message+ # tokens (the "4,436 -> 0" shape), sourced mostly from the tools+ # array.+ compressed = dict(payload)+ compressed["tools"] = [{"type": "function", "name": "t", "parameters": {}}]+ return (+ compressed,+ True,+ _TOKENS_SAVED,+ ["openai:responses:tool_schema_compaction"],+ None,+ 100_000,+ 10_000,+ _TOKENS_SAVED,+ {},+ )++ server._compress_openai_responses_payload_in_executor = _fake_compress++ outcomes = []++ async def _capture_outcome(outcome):+ outcomes.append(outcome)++ server._record_request_outcome = _capture_outcome++ payload = {+ "model": "gpt-5-codex",+ "input": "list files",+ "tools": [+ {+ "type": "function",+ "name": "big_tool",+ # ~9k tokens of schema so original (messages + tools) can+ # absorb _TOKENS_SAVED without clamping.+ "description": "word " * 9_000,+ "parameters": {"type": "object", "properties": {}},+ }+ ],+ }++ with TestClient(app) as client:+ resp = client.post(+ "/v1/responses",+ headers={+ "Authorization": "Bearer sk-test",+ "Content-Type": "application/json",+ },+ json=payload,+ )+ assert resp.status_code == 200++ assert outcomes, "no RequestOutcome recorded"+ o = outcomes[0]+ assert o.tokens_saved == _TOKENS_SAVED+ # Coherent triple: in - out == saved, out not clamped to zero, and the+ # savings rate (saved / original) never above 100%.+ assert o.original_tokens - o.optimized_tokens == o.tokens_saved+ assert o.optimized_tokens > 0+ assert o.tokens_saved <= o.original_tokens+++def test_no_tools_payload_keeps_messages_only_pair():+ app = _make_app()+ server = app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs):+ return _fake_upstream_response(url)++ server._retry_request = _fake_retry++ saved = 5++ async def _fake_compress(payload, **kwargs):+ return (dict(payload), True, saved, ["openai:responses:trim"], None, 1_000, 900, saved, {})++ server._compress_openai_responses_payload_in_executor = _fake_compress++ outcomes = []++ async def _capture_outcome(outcome):+ outcomes.append(outcome)++ server._record_request_outcome = _capture_outcome++ payload = {"model": "gpt-5-codex", "input": "word " * 100}++ with TestClient(app) as client:+ resp = client.post(+ "/v1/responses",+ headers={+ "Authorization": "Bearer sk-test",+ "Content-Type": "application/json",+ },+ json=payload,+ )+ assert resp.status_code == 200++ assert outcomes, "no RequestOutcome recorded"+ o = outcomes[0]+ assert o.tokens_saved == saved+ # No tools array: original_tokens must stay the messages-only count+ # (not widened), and the triple still holds.+ assert o.original_tokens - o.optimized_tokens == o.tokens_saved+ assert o.optimized_tokens > 0+ assert o.tokens_saved <= o.original_tokens