Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): never batch-compress already-forwarded small Responses outputs in cache mode (#3756)

donjorab62b9e0ad95
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 674e0e3af..efeecbd53 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -33,6 +33,7 @@ from headroom.proxy.helpers import (
)
from headroom.proxy.identity import resolve_memory_identity
from headroom.proxy.loopback_guard import is_loopback_host
+from headroom.proxy.modes import is_cache_mode
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
from headroom.proxy.upstream_guard import is_safe_upstream_url
from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS
@@ -2511,10 +2512,16 @@ class OpenAIHandlerMixin:
)
else:
large_unit_indexes.append(unit_idx)
- small_batches, small_batch_skipped = build_compression_batches(
- small_batch_entries,
- min_batch_bytes=self.OPENAI_RESPONSES_ROUTER_MIN_BYTES,
- )
+ if is_cache_mode(getattr(getattr(self, "config", None), "mode", "token")):
+ # Appending outputs can push an old below-floor unit into a batch.
+ # Preserve its first-forwarded bytes instead of retroactively
+ # compressing it. Large units retain their per-unit result cache.
+ small_batches, small_batch_skipped = [], small_batch_entries
+ else:
+ small_batches, small_batch_skipped = build_compression_batches(
+ small_batch_entries,
+ min_batch_bytes=self.OPENAI_RESPONSES_ROUTER_MIN_BYTES,
+ )
cache_misses: list[tuple[int, str, RoutedCompressionUnit]] = []
cache_miss_followers: dict[str, list[int]] = {}
for unit_idx in large_unit_indexes:
diff --git a/tests/test_openai_responses_compression_units.py b/tests/test_openai_responses_compression_units.py
index 4b6d54705..4ba858808 100644
--- a/tests/test_openai_responses_compression_units.py
+++ b/tests/test_openai_responses_compression_units.py
@@ -1300,3 +1300,53 @@ def test_openai_responses_adapter_excludes_custom_tool_beside_compressed_functio
assert modified is True
assert new_payload["input"][1]["output"] == excluded_output
assert new_payload["input"][3]["output"] == "compressed tool output"
+
+
+def test_openai_responses_cache_mode_never_batches_old_small_outputs():
+ """Cache mode: an appended output must not pull earlier outputs into a batch.
+
+ Turn 1 forwards four small outputs raw (under the batch floor). Turn 2
+ appends two more, crossing the floor. Batching then would rewrite the
+ four outputs already in the provider's cached prefix.
+ """
+ router = ContentRouter()
+ calls: list[str] = []
+ floor = OpenAIHandlerMixin.OPENAI_RESPONSES_ROUTER_MIN_BYTES
+ outputs = [" ".join(f"unit{index}_{token}" for token in range(12)) for index in range(6)]
+ assert sum(len(output.encode("utf-8")) for output in outputs[:4]) < floor
+ assert sum(len(output.encode("utf-8")) for output in outputs) >= floor
+
+ def compress(self, content: str, **_kwargs):
+ calls.append(content)
+ compressed = content
+ for output in outputs:
+ compressed = compressed.replace(output, "x")
+ return RouterCompressionResult(
+ compressed=compressed,
+ original=content,
+ strategy_used=CompressionStrategy.KOMPRESS,
+ )
+
+ router.compress = MethodType(compress, router)
+ handler = _handler_with_router(router)
+ handler.config = SimpleNamespace(mode="cache")
+
+ def turn(count: int) -> dict:
+ return {
+ "model": "gpt-5",
+ "input": [
+ {"type": "local_shell_call_output", "call_id": f"c{index}", "output": output}
+ for index, output in enumerate(outputs[:count])
+ ],
+ }
+
+ first, *_ = handler._compress_openai_responses_live_text_units_with_router(
+ turn(4), model="gpt-5", request_id="req_turn_1"
+ )
+ second, modified, *_ = handler._compress_openai_responses_live_text_units_with_router(
+ turn(6), model="gpt-5", request_id="req_turn_2"
+ )
+
+ assert calls == []
+ assert modified is False
+ assert second["input"][:4] == first["input"]