Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): keep OpenAI tool observations mutable in cache mode (#1884)
JD Davis55efb1c77d5b
diff --git a/headroom/proxy/cost.py b/headroom/proxy/cost.pyindex 0bb772f92..37670c309 100644--- a/headroom/proxy/cost.py+++ b/headroom/proxy/cost.py@@ -745,9 +745,7 @@ class CostTracker:# clamp it so `total_tokens_removed` reflects actually-forwarded bytes# instead of surfacing spurious negatives (verified clean on the wire).if tokens_saved < 0:- import logging as _lg-- _lg.getLogger(__name__).debug(+ logger.debug("record_tokens: clamping negative tokens_saved=%d to 0 for %s (artifact; wire not inflated)",tokens_saved,model,diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 86a12ce43..2823de0b9 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -1070,7 +1070,7 @@ class OpenAIHandlerMixin:return base_frozen_countfinal_idx = len(messages) - 1if messages[final_idx].get("role") in ("user", "tool", "function"):- return max(base_frozen_count, final_idx)+ return final_idxreturn len(messages)@staticmethod@@ -2363,8 +2363,8 @@ class OpenAIHandlerMixin:openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()if is_cache_mode(self.config.mode):- openai_frozen_count = self._strict_previous_turn_frozen_count(- original_client_messages,+ openai_frozen_count = OpenAIHandlerMixin._strict_previous_turn_frozen_count(+ messages,openai_frozen_count,)@@ -2399,8 +2399,14 @@ class OpenAIHandlerMixin:# Zone 1: Swap cached compressed versionsworking_messages = comp_cache.apply_cached(messages)- # Re-freeze boundary- openai_frozen_count = comp_cache.compute_frozen_count(messages)+ # Re-freeze boundary. Token mode can use the compression+ # cache's positional frozen count. Cache mode must keep the+ # latest observation mutable even when the compression+ # cache has no compressible entry for it yet; otherwise+ # OpenAI-compatible tool-call clients freeze the entire+ # conversation and report near-zero savings.+ if not is_cache_mode(self.config.mode):+ openai_frozen_count = comp_cache.compute_frozen_count(messages)result = await self._run_compression_in_executor(lambda: self.openai_pipeline.apply(@@ -2408,7 +2414,14 @@ class OpenAIHandlerMixin:model=model,model_limit=context_limit,context=extract_user_query(working_messages),- frozen_message_count=openai_frozen_count,+ frozen_message_count=(+ OpenAIHandlerMixin._strict_previous_turn_frozen_count(+ working_messages,+ openai_frozen_count,+ )+ if is_cache_mode(self.config.mode)+ else openai_frozen_count+ ),biases=_hook_biases,compression_policy=compression_policy,# Thread the savings-profile knobs (e.g.@@ -2434,13 +2447,21 @@ class OpenAIHandlerMixin:# so tokens_saved captures both Zone 1 + Zone 2 savings.optimized_tokens = result.tokens_afterelse:+ apply_frozen_count = (+ OpenAIHandlerMixin._strict_previous_turn_frozen_count(+ messages,+ openai_frozen_count,+ )+ if is_cache_mode(self.config.mode)+ else openai_frozen_count+ )result = await self._run_compression_in_executor(lambda: self.openai_pipeline.apply(messages=messages,model=model,model_limit=context_limit,context=extract_user_query(messages),- frozen_message_count=openai_frozen_count,+ frozen_message_count=apply_frozen_count,biases=_hook_biases,compression_policy=compression_policy,# Same savings-profile threading as the token-modediff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex 25033b6a0..b6ab41b15 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -597,9 +597,7 @@ class PrometheusMetrics:# model. Clamp so total_tokens_removed / avg_compression_pct reflect the# actually-forwarded bytes instead of surfacing spurious negatives.if tokens_saved < 0:- import logging as _lg-- _lg.getLogger(__name__).debug(+ logger.debug("metrics.record: clamping negative tokens_saved=%d to 0 for %s (artifact; wire not inflated)",tokens_saved,model,diff --git a/tests/test_cost_tracker_counterfactual.py b/tests/test_cost_tracker_counterfactual.pyindex f49c974db..63cc123cd 100644--- a/tests/test_cost_tracker_counterfactual.py+++ b/tests/test_cost_tracker_counterfactual.py@@ -80,6 +80,19 @@ def test_savings_zero_when_no_tokens_saved():assert stats["total_tokens_saved"] == 0+def test_negative_token_savings_are_clamped_to_zero():+ """Estimator artifacts must not reduce cumulative savings below reality."""+ from headroom.proxy.server import CostTracker++ ct = CostTracker()++ ct.record_tokens("openai-compatible", tokens_saved=-500, tokens_sent=5_000)+ stats = ct.stats()++ assert stats["total_tokens_saved"] == 0+ assert stats["per_model"]["openai-compatible"]["tokens_saved"] == 0++def test_multi_model_savings():"""Savings across multiple models use each model's own list price."""from headroom.proxy.server import CostTrackerdiff --git a/tests/test_observability_metrics.py b/tests/test_observability_metrics.pyindex f49c3b71c..6b7e09890 100644--- a/tests/test_observability_metrics.py+++ b/tests/test_observability_metrics.py@@ -198,3 +198,20 @@ async def test_prometheus_metrics_reads_late_configured_otel_metrics() -> None:assert spy.rate_limited_calls == [{"provider": "anthropic", "model": "claude-sonnet"}]finally:reset_otel_metrics()+++async def test_prometheus_metrics_clamps_negative_token_savings() -> None:+ metrics = PrometheusMetrics()++ await metrics.record_request(+ provider="openai",+ model="openai-compatible",+ input_tokens=100,+ output_tokens=5,+ tokens_saved=-25,+ latency_ms=1.0,+ )++ assert metrics.tokens_saved_total == 0+ assert metrics.savings_history[-1][1] == 0diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex e55662c75..2609d5fa1 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -221,6 +221,27 @@ def test_openai_handler_prefix_helpers_cover_edge_cases() -> None:)== 1)+ assert (+ OpenAIHandlerMixin._strict_previous_turn_frozen_count(+ [{"role": "assistant"}, {"role": "tool", "content": "observation"}],+ 0,+ )+ == 1+ )+ assert (+ OpenAIHandlerMixin._strict_previous_turn_frozen_count(+ [{"role": "user"}, {"role": "assistant"}, {"role": "tool", "content": "obs"}],+ 3,+ )+ == 2+ )+ assert (+ OpenAIHandlerMixin._strict_previous_turn_frozen_count(+ [{"role": "assistant"}, {"role": "function", "content": "legacy observation"}],+ 0,+ )+ == 1+ )assert (OpenAIHandlerMixin._strict_previous_turn_frozen_count([{"role": "user"}, {"role": "assistant"}],diff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.pyindex 44339c91a..4cbd18d5d 100644--- a/tests/test_proxy_openai_cache_stability.py+++ b/tests/test_proxy_openai_cache_stability.py@@ -112,6 +112,85 @@ def test_openai_cache_mode_freezes_previous_turns() -> None:assert captured["frozen_message_count"] == 2[email protected]("tail_role", ["tool", "function"])+def test_openai_cache_mode_keeps_final_tool_observation_mutable(tail_role: str) -> None:+ captured = {}+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "cache"++ fake_tracker = _FakePrefixTracker(frozen_count=0)+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "stable-session"+ )+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker++ def _fake_apply(**kwargs):+ captured.setdefault("calls", []).append(+ {+ "frozen_message_count": kwargs.get("frozen_message_count"),+ "roles": [msg.get("role") for msg in kwargs["messages"]],+ "mode": proxy.config.mode,+ }+ )+ return SimpleNamespace(+ messages=kwargs["messages"],+ transforms_applied=["test:compress-tail"],+ timing={},+ tokens_before=120,+ tokens_after=80,+ waste_signals=None,+ )++ proxy.openai_pipeline.apply = _fake_apply++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_tool_tail",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 80, "completion_tokens": 3, "total_tokens": 83},+ },+ )++ proxy._retry_request = _fake_retry++ tail = {+ "role": tail_role,+ "content": "large command observation " * 200,+ }+ if tail_role == "tool":+ tail["tool_call_id"] = "call_1"+ else:+ tail["name"] = "bash"++ response = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={+ "model": "gpt-4o-mini",+ "messages": [+ {"role": "user", "content": "turn1"},+ {"role": "assistant", "content": "run command"},+ tail,+ ],+ },+ )++ assert response.status_code == 200+ assert any(call["frozen_message_count"] == 2 for call in captured["calls"]), captured[+ "calls"+ ]++def test_openai_cache_mode_restores_mutated_frozen_prefix() -> None:captured = {}with _make_proxy_client() as client: