Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): never batch-compress already-forwarded small Responses outputs in cache mode (#3756)
donjorab62b9e0ad95
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 674e0e3af..efeecbd53 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -33,6 +33,7 @@ from headroom.proxy.helpers import ()from headroom.proxy.identity import resolve_memory_identityfrom headroom.proxy.loopback_guard import is_loopback_host+from headroom.proxy.modes import is_cache_modefrom headroom.proxy.stage_timer import StageTimer, emit_stage_timings_logfrom headroom.proxy.upstream_guard import is_safe_upstream_urlfrom headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS@@ -2511,10 +2512,16 @@ class OpenAIHandlerMixin:)else:large_unit_indexes.append(unit_idx)- small_batches, small_batch_skipped = build_compression_batches(- small_batch_entries,- min_batch_bytes=self.OPENAI_RESPONSES_ROUTER_MIN_BYTES,- )+ if is_cache_mode(getattr(getattr(self, "config", None), "mode", "token")):+ # Appending outputs can push an old below-floor unit into a batch.+ # Preserve its first-forwarded bytes instead of retroactively+ # compressing it. Large units retain their per-unit result cache.+ small_batches, small_batch_skipped = [], small_batch_entries+ else:+ small_batches, small_batch_skipped = build_compression_batches(+ small_batch_entries,+ min_batch_bytes=self.OPENAI_RESPONSES_ROUTER_MIN_BYTES,+ )cache_misses: list[tuple[int, str, RoutedCompressionUnit]] = []cache_miss_followers: dict[str, list[int]] = {}for unit_idx in large_unit_indexes:diff --git a/tests/test_openai_responses_compression_units.py b/tests/test_openai_responses_compression_units.pyindex 4b6d54705..4ba858808 100644--- a/tests/test_openai_responses_compression_units.py+++ b/tests/test_openai_responses_compression_units.py@@ -1300,3 +1300,53 @@ def test_openai_responses_adapter_excludes_custom_tool_beside_compressed_functioassert modified is Trueassert new_payload["input"][1]["output"] == excluded_outputassert new_payload["input"][3]["output"] == "compressed tool output"+++def test_openai_responses_cache_mode_never_batches_old_small_outputs():+ """Cache mode: an appended output must not pull earlier outputs into a batch.++ Turn 1 forwards four small outputs raw (under the batch floor). Turn 2+ appends two more, crossing the floor. Batching then would rewrite the+ four outputs already in the provider's cached prefix.+ """+ router = ContentRouter()+ calls: list[str] = []+ floor = OpenAIHandlerMixin.OPENAI_RESPONSES_ROUTER_MIN_BYTES+ outputs = [" ".join(f"unit{index}_{token}" for token in range(12)) for index in range(6)]+ assert sum(len(output.encode("utf-8")) for output in outputs[:4]) < floor+ assert sum(len(output.encode("utf-8")) for output in outputs) >= floor++ def compress(self, content: str, **_kwargs):+ calls.append(content)+ compressed = content+ for output in outputs:+ compressed = compressed.replace(output, "x")+ return RouterCompressionResult(+ compressed=compressed,+ original=content,+ strategy_used=CompressionStrategy.KOMPRESS,+ )++ router.compress = MethodType(compress, router)+ handler = _handler_with_router(router)+ handler.config = SimpleNamespace(mode="cache")++ def turn(count: int) -> dict:+ return {+ "model": "gpt-5",+ "input": [+ {"type": "local_shell_call_output", "call_id": f"c{index}", "output": output}+ for index, output in enumerate(outputs[:count])+ ],+ }++ first, *_ = handler._compress_openai_responses_live_text_units_with_router(+ turn(4), model="gpt-5", request_id="req_turn_1"+ )+ second, modified, *_ = handler._compress_openai_responses_live_text_units_with_router(+ turn(6), model="gpt-5", request_id="req_turn_2"+ )++ assert calls == []+ assert modified is False+ assert second["input"][:4] == first["input"]