Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy/openai): keep /v1/responses savings triple coherent when schema savings exceed message tokens (#3106)

gglucass7278b5cbf83f
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 398985c54..463963e35 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -5637,6 +5637,11 @@ class OpenAIHandlerMixin:
# Token counting on converted messages (offloaded off the event loop — GH #1701)
tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages)
+ # Messages-only count, preserved for the output shaper's turn/size
+ # classifier: the accounting pair below may widen original_tokens by
+ # the tools schema after compression, and shaper strata must not
+ # shift when it does.
+ message_input_tokens = original_tokens
# Defaults below feed downstream telemetry and memory injection.
# If optimization remains enabled, the Responses payload is compressed
@@ -5880,6 +5885,10 @@ class OpenAIHandlerMixin:
# gating already happened upstream (auth_mode classify,
# CompressionPolicy resolve at request entry).
if self.config.optimize and not _bypass:
+ # Pre-compression tools reference: the compression pass is
+ # copy-on-write on the payload, so this still points at the
+ # original schemas after `body` is rebound below.
+ _tools_before = body.get("tools")
try:
(
body,
@@ -5908,6 +5917,17 @@ class OpenAIHandlerMixin:
if _modified:
body_mutation_tracker.mark_mutated("responses_compression")
tokens_saved = int(_tokens_saved)
+ # tokens_saved includes tool schema/desc compaction, which
+ # is measured against the tools array — content the
+ # messages-only original_tokens count above never
+ # included. Fold the tools schema into the pair so it
+ # stays coherent (original - optimized == saved, savings
+ # rate <= 100%); without this, schema-heavy turns clamp
+ # optimized to 0 and record rates above 100%.
+ if _tools_before:
+ original_tokens += await asyncio.to_thread(
+ tokenizer.count_text, _json_debug_dumps(_tools_before)
+ )
responses_conversation_key = _pre_compression_conversation_key
optimized_tokens = max(0, original_tokens - tokens_saved)
logger.info(
@@ -5981,7 +6001,7 @@ class OpenAIHandlerMixin:
_http_conversation_key = request.headers.get("x-headroom-session-id")
_shape_result = _shape_openai_responses_for_output(
body,
- input_tokens=original_tokens,
+ input_tokens=message_input_tokens,
model=str(model or ""),
conversation_key=(
f"header:x-headroom-session-id:{_http_conversation_key}"
diff --git a/tests/test_responses_savings_denominator.py b/tests/test_responses_savings_denominator.py
new file mode 100644
index 000000000..c02f30579
--- /dev/null
+++ b/tests/test_responses_savings_denominator.py
@@ -0,0 +1,169 @@
+"""Recorded /v1/responses savings must form a coherent triple.
+
+``tokens_saved`` on the Responses HTTP path includes tool schema/desc
+compaction, which is measured against the ``tools`` array — content the
+messages-only ``original_tokens`` count never included. Deriving
+``optimized_tokens = max(0, original - saved)`` from that mismatched pair
+clamped to 0 on schema-heavy turns and recorded savings rates above 100%
+(surfaced in the desktop feed as e.g. "4,436 → 0 (208.4%)"). The fix folds
+the tools schema into the pair, so ``original - optimized == saved`` and
+the rate stays <= 100%.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytest.importorskip("fastapi")
+pytest.importorskip("httpx")
+
+import httpx
+from fastapi.testclient import TestClient
+
+from headroom.proxy.loopback_guard import require_loopback
+from headroom.proxy.server import ProxyConfig, create_app
+
+_TOKENS_SAVED = 5_000
+
+
+def _make_app():
+ config = ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+ app.dependency_overrides[require_loopback] = lambda: None
+ return app
+
+
+def _fake_upstream_response(url: str) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ json={
+ "id": "resp_test",
+ "object": "response",
+ "output": [],
+ "usage": {"input_tokens": 12, "output_tokens": 3},
+ },
+ request=httpx.Request("POST", url),
+ )
+
+
+def test_schema_heavy_compression_keeps_savings_triple_coherent():
+ app = _make_app()
+ server = app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs):
+ return _fake_upstream_response(url)
+
+ server._retry_request = _fake_retry
+
+ async def _fake_compress(payload, **kwargs):
+ # Simulate schema compaction: savings far larger than the message
+ # tokens (the "4,436 -> 0" shape), sourced mostly from the tools
+ # array.
+ compressed = dict(payload)
+ compressed["tools"] = [{"type": "function", "name": "t", "parameters": {}}]
+ return (
+ compressed,
+ True,
+ _TOKENS_SAVED,
+ ["openai:responses:tool_schema_compaction"],
+ None,
+ 100_000,
+ 10_000,
+ _TOKENS_SAVED,
+ {},
+ )
+
+ server._compress_openai_responses_payload_in_executor = _fake_compress
+
+ outcomes = []
+
+ async def _capture_outcome(outcome):
+ outcomes.append(outcome)
+
+ server._record_request_outcome = _capture_outcome
+
+ payload = {
+ "model": "gpt-5-codex",
+ "input": "list files",
+ "tools": [
+ {
+ "type": "function",
+ "name": "big_tool",
+ # ~9k tokens of schema so original (messages + tools) can
+ # absorb _TOKENS_SAVED without clamping.
+ "description": "word " * 9_000,
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ],
+ }
+
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/responses",
+ headers={
+ "Authorization": "Bearer sk-test",
+ "Content-Type": "application/json",
+ },
+ json=payload,
+ )
+ assert resp.status_code == 200
+
+ assert outcomes, "no RequestOutcome recorded"
+ o = outcomes[0]
+ assert o.tokens_saved == _TOKENS_SAVED
+ # Coherent triple: in - out == saved, out not clamped to zero, and the
+ # savings rate (saved / original) never above 100%.
+ assert o.original_tokens - o.optimized_tokens == o.tokens_saved
+ assert o.optimized_tokens > 0
+ assert o.tokens_saved <= o.original_tokens
+
+
+def test_no_tools_payload_keeps_messages_only_pair():
+ app = _make_app()
+ server = app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs):
+ return _fake_upstream_response(url)
+
+ server._retry_request = _fake_retry
+
+ saved = 5
+
+ async def _fake_compress(payload, **kwargs):
+ return (dict(payload), True, saved, ["openai:responses:trim"], None, 1_000, 900, saved, {})
+
+ server._compress_openai_responses_payload_in_executor = _fake_compress
+
+ outcomes = []
+
+ async def _capture_outcome(outcome):
+ outcomes.append(outcome)
+
+ server._record_request_outcome = _capture_outcome
+
+ payload = {"model": "gpt-5-codex", "input": "word " * 100}
+
+ with TestClient(app) as client:
+ resp = client.post(
+ "/v1/responses",
+ headers={
+ "Authorization": "Bearer sk-test",
+ "Content-Type": "application/json",
+ },
+ json=payload,
+ )
+ assert resp.status_code == 200
+
+ assert outcomes, "no RequestOutcome recorded"
+ o = outcomes[0]
+ assert o.tokens_saved == saved
+ # No tools array: original_tokens must stay the messages-only count
+ # (not widened), and the triple still holds.
+ assert o.original_tokens - o.optimized_tokens == o.tokens_saved
+ assert o.optimized_tokens > 0
+ assert o.tokens_saved <= o.original_tokens