Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): share one compression deadline across a Responses request (#3938)

gglucass4d27d024b09f
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 9054a19cf..678be4579 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import asyncio
import contextlib
+import contextvars
import copy
import hashlib
import hmac
@@ -664,6 +665,31 @@ def _openai_responses_unit_parallelism() -> int:
return max(1, min(_OPENAI_RESPONSES_UNIT_PARALLELISM_MAX, requested))
+def _openai_responses_deadline_started_at(timeout: float) -> float | None:
+ """Deadline origin shared by every compress call of one Responses request.
+
+ A Responses request fans out over many ``ContentRouter.compress`` calls,
+ and Kompress starts a fresh ``HEADROOM_COMPRESSION_DEADLINE_MS`` clock in
+ each one unless it is handed an origin. Without one, a request past its
+ ``timeout`` keeps running inference long after the awaiter gave up, and
+ that timeout debt quarantines compression for every request behind it.
+ Stamped before the job is queued, because ``timeout`` counts queue wait
+ too. ``None`` when the deadline is disabled (``0``).
+
+ The budget is also capped at 75% of ``timeout``: the deadline only stops
+ the NEXT Kompress chunk, so the chunk already running when it expires, the
+ splice and the output serialization need the rest. Kompress checks
+ ``now - origin > deadline``, so a shorter budget is a backdated origin.
+ """
+ from headroom.transforms.content_router import _compression_deadline_seconds
+
+ deadline_s = _compression_deadline_seconds()
+ if deadline_s <= 0:
+ return None
+ budget_s = min(deadline_s, 0.75 * timeout)
+ return time.perf_counter() - (deadline_s - budget_s)
+
+
def _openai_responses_unit_executor() -> ThreadPoolExecutor:
global _OPENAI_RESPONSES_UNIT_EXECUTOR
with _OPENAI_RESPONSES_UNIT_EXECUTOR_LOCK:
@@ -2130,6 +2156,7 @@ class OpenAIHandlerMixin:
request_id: str,
pass_id: str | None = None,
timing: dict[str, float] | None = None,
+ deadline_started_at: float | None = None,
) -> tuple[dict[str, Any], bool, int, list[str], dict[str, int], list[str], int]:
"""Run ContentRouter on OpenAI Responses text units.
@@ -2723,6 +2750,7 @@ class OpenAIHandlerMixin:
router=router,
tokenizer=tokenizer,
target_ratio=unit_target_ratio,
+ deadline_started_at=deadline_started_at,
)
elapsed_ms = (time.perf_counter() - unit_started) * 1000.0
return routed.slot, result, elapsed_ms
@@ -2792,12 +2820,18 @@ class OpenAIHandlerMixin:
0.0,
)
+ # Each pool task runs in its own copy of this Context: the router binds
+ # the request's deadline per task (see share_request_deadline), and
+ # pool threads would otherwise keep that binding between requests.
parallelism = _openai_responses_unit_parallelism()
if len(cache_misses) > 1 and parallelism > 1:
executor = _openai_responses_unit_executor()
for start in range(0, len(cache_misses), parallelism):
batch = cache_misses[start : start + parallelism]
- futures = [executor.submit(_compress_and_store, *item) for item in batch]
+ futures = [
+ executor.submit(contextvars.copy_context().run, _compress_and_store, *item)
+ for item in batch
+ ]
for future in as_completed(futures):
unit_idx, cache_key, routed_result = future.result()
_record_routed_result(unit_idx, cache_key, routed_result)
@@ -2822,6 +2856,7 @@ class OpenAIHandlerMixin:
router=router,
tokenizer=tokenizer,
target_ratio=unit_target_ratio,
+ deadline_started_at=deadline_started_at,
)
return results, (time.perf_counter() - batch_started) * 1000.0
@@ -2835,7 +2870,10 @@ class OpenAIHandlerMixin:
executor = _openai_responses_unit_executor()
for start in range(0, len(small_batches), parallelism):
batch_group = small_batches[start : start + parallelism]
- futures = [executor.submit(_compress_batch, batch) for batch in batch_group]
+ futures = [
+ executor.submit(contextvars.copy_context().run, _compress_batch, batch)
+ for batch in batch_group
+ ]
for future in as_completed(futures):
_record_batch_result(future.result())
else:
@@ -3023,6 +3061,7 @@ class OpenAIHandlerMixin:
timing: dict[str, float] | None = None,
client: str | None = None,
savings_tags: dict[str, Any] | None = None,
+ deadline_started_at: float | None = None,
) -> tuple[dict[str, Any], bool, int, list[str], str | None, int, int, int]:
"""Compress an OpenAI Responses payload through the shared router.
@@ -3290,6 +3329,7 @@ class OpenAIHandlerMixin:
request_id=request_id,
pass_id=pass_id,
timing=timing_sink,
+ deadline_started_at=deadline_started_at,
)
_add_timing("compression_live_units_total", live_units_started)
if router_modified:
@@ -3437,6 +3477,8 @@ class OpenAIHandlerMixin:
exc_info=True,
)
+ deadline_started_at = _openai_responses_deadline_started_at(timeout)
+
def _compress(): # noqa: ANN202
# Output shaping (opt-in via HEADROOM_OUTPUT_SHAPER) runs before
# compression so the turn classifier sees the client's input as
@@ -3456,6 +3498,7 @@ class OpenAIHandlerMixin:
"request_id": request_id,
"timing": timing,
"client": client,
+ "deadline_started_at": deadline_started_at,
}
if savings_tags is not None:
compression_kwargs["savings_tags"] = savings_tags
@@ -3470,7 +3513,12 @@ class OpenAIHandlerMixin:
unsupported_kwarg = next(
(
name
- for name in ("savings_tags", "client", "timing")
+ for name in (
+ "savings_tags",
+ "client",
+ "timing",
+ "deadline_started_at",
+ )
if f"unexpected keyword argument '{name}'" in str(exc)
and name in compression_kwargs
),
diff --git a/headroom/transforms/compression_batches.py b/headroom/transforms/compression_batches.py
index 4fae8ebeb..84b429c2d 100644
--- a/headroom/transforms/compression_batches.py
+++ b/headroom/transforms/compression_batches.py
@@ -230,8 +230,15 @@ def compress_batch_with_router(
router: ContentRouter,
tokenizer: TokenCounterLike,
target_ratio: float | None = None,
+ deadline_started_at: float | None = None,
) -> list[tuple[object, UnitCompressionResult]]:
- """Compress one tagged batch and split only structurally valid output."""
+ """Compress one tagged batch and split only structurally valid output.
+
+ ``deadline_started_at`` works as in ``compress_unit_with_router``.
+ """
+
+ if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
+ return _passthrough_batch_results(batch, tokenizer=tokenizer, reason="deadline_exceeded")
nonce = _batch_nonce(batch)
batch_texts, marker_blocks = _protect_ccr_markers(batch, nonce)
diff --git a/headroom/transforms/compression_units.py b/headroom/transforms/compression_units.py
index c7296533f..6df3c31d0 100644
--- a/headroom/transforms/compression_units.py
+++ b/headroom/transforms/compression_units.py
@@ -56,6 +56,8 @@ class CompressionUnit:
# Responses handler's unit-result cache) and is surfaced
# via RouterCompressionResult.cache_hit, not as a reason
# here — no code path produces this category today
+# - deadline: the request's compression deadline had passed before
+# this unit started, so it was left unchanged
UNIT_REASON_CATEGORIES = {
None: "applied",
"protected_user_message": "protected_role",
@@ -66,6 +68,7 @@ UNIT_REASON_CATEGORIES = {
"router_no_change": "compressor_noop",
"already_compressed": "already_compressed",
"rejected_not_smaller": "rejected_not_smaller",
+ "deadline_exceeded": "deadline",
}
@@ -221,11 +224,17 @@ def compress_unit_with_router(
router: ContentRouter,
tokenizer: TokenCounterLike,
target_ratio: float | None = None,
+ deadline_started_at: float | None = None,
) -> UnitCompressionResult:
"""Compress one safe text unit through ContentRouter.
The final accept/reject gate uses the provider/model tokenizer, not the
router's internal word-count estimates.
+
+ ``deadline_started_at`` is the deadline origin of the request this unit
+ belongs to (see ``ContentRouter.share_request_deadline``). Once that
+ deadline has passed the unit is returned unchanged without entering the
+ router.
"""
tokens_before = tokenizer.count_text(unit.text)
@@ -266,6 +275,8 @@ def compress_unit_with_router(
return _with_reason(reason=f"cache_zone_{unit.cache_zone}")
if text_bytes < unit.min_bytes:
return _with_reason(reason="below_unit_floor")
+ if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
+ return _with_reason(reason="deadline_exceeded")
prior_target_ratio = getattr(router, "_runtime_target_ratio", None)
if target_ratio is not None:
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 667dfc54c..200657468 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -2394,6 +2394,27 @@ class ContentRouter(Transform):
self._runtime_state_var.set(state)
return state
+ def share_request_deadline(self, started_at: float) -> bool:
+ """Join a request whose kompress deadline started at ``started_at``.
+
+ ``apply()`` stamps its own origin. A caller that fans ONE request out
+ over many ``compress()`` calls (the OpenAI Responses unit adapter) calls
+ this before each one instead, so every call draws down the same
+ ``HEADROOM_COMPRESSION_DEADLINE_MS`` budget rather than restarting it.
+ Binds a fresh ``_PerRequestRuntimeState`` in the CURRENT Context, so a
+ worker-pool task must run in its own ``contextvars.copy_context()``.
+
+ Returns ``False``, binding nothing, once the deadline has passed: the
+ caller should leave the content unchanged rather than start new work.
+ """
+ deadline_s = _compression_deadline_seconds()
+ if deadline_s and time.perf_counter() - started_at > deadline_s:
+ return False
+ self._runtime_state_var.set(
+ _PerRequestRuntimeState(kompress_deadline_started_at=started_at)
+ )
+ return True
+
@property
def _runtime_compression_policy(self) -> Any:
return self._runtime_state_var.get().compression_policy
diff --git a/tests/test_compression_batches.py b/tests/test_compression_batches.py
index e6d5a286a..55c110ba0 100644
--- a/tests/test_compression_batches.py
+++ b/tests/test_compression_batches.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import re
+import time
from headroom.transforms.compression_batches import (
CompressionBatchEntry,
@@ -229,3 +230,27 @@ def test_batch_uses_utf8_bytes_for_cjk_small_units():
assert batches[0].text_bytes == 1800
assert all(result.modified for _, result in results)
assert [result.compressed for _, result in results] == ["短"] * 4
+
+
+def test_batch_after_the_request_deadline_passes_through_without_routing(monkeypatch):
+ from headroom.transforms.content_router import ContentRouter
+
+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
+ entries = [_entry(index, "x" * 150) for index in range(4)]
+ batches, _ = build_compression_batches(entries, min_batch_bytes=512)
+ router = ContentRouter()
+
+ def _not_called(*_args, **_kwargs):
+ raise AssertionError("a batch past the deadline must not enter the router")
+
+ router.compress = _not_called
+
+ results = compress_batch_with_router(
+ batches[0],
+ router=router,
+ tokenizer=_CharacterCounter(),
+ deadline_started_at=time.perf_counter() - 21,
+ )
+
+ assert [result.compressed for _, result in results] == ["x" * 150] * 4
+ assert {result.reason for _, result in results} == {"deadline_exceeded"}
diff --git a/tests/test_openai_responses_compression_units.py b/tests/test_openai_responses_compression_units.py
index 3037d97cc..268db0a97 100644
--- a/tests/test_openai_responses_compression_units.py
+++ b/tests/test_openai_responses_compression_units.py
@@ -1,7 +1,9 @@
from __future__ import annotations
+import asyncio
import json
import threading
+import time
from types import MethodType, SimpleNamespace
import pytest
@@ -1417,6 +1419,114 @@ def test_openai_responses_adapter_compresses_historical_messages_not_current_use
assert any(t.startswith("router:openai:responses:message:") for t in transforms)
+class _RecordingKompress:
+ """Stands in for KompressCompressor, recording the deadline origin it gets."""
+
+ shares_request_deadline = True
+
+ def __init__(self, delay: float = 0.0) -> None:
+ self.delay = delay
+ self.deadlines: list[float | None] = []
+
+ def is_ready(self) -> bool:
+ return True
+
+ def compress(self, content: str, **kwargs):
+ self.deadlines.append(kwargs.get("_deadline_started_at"))
+ time.sleep(self.delay)
+ compressed = " ".join(content.split()[:20])
+ return SimpleNamespace(compressed=compressed, compressed_tokens=20)
+
+
+def _kompress_payload(units: int) -> dict:
+ return {
+ "model": "gpt-5",
+ "input": [
+ {
+ "type": "function_call_output",
+ "call_id": f"call_{i}",
+ "output": "\n".join(
+ " ".join(f"unit{i}line{line}word{j}" for j in range(10)) for line in range(70)
+ ),
+ }
+ for i in range(units)
+ ],
+ }
+
+
+def _compress_through_executor(router: ContentRouter, payload: dict) -> tuple:
+ handler = _handler_with_router(router)
+
+ async def _run_compression(fn, *, timeout): # noqa: ANN001, ANN202
+ return fn()
+
+ handler._run_compression_in_executor = _run_compression
+ return asyncio.run(
+ handler._compress_openai_responses_payload_in_executor(
+ payload, model="gpt-5", request_id="req_deadline"
+ )
+ )
+
+
+def test_openai_responses_units_share_one_kompress_deadline(monkeypatch):
+ """The fan-out must draw down ONE request deadline (#3693), pool threads too.
+
+ Each unit is its own ContentRouter.compress call, and Kompress started a
+ fresh HEADROOM_COMPRESSION_DEADLINE_MS clock in every one, so a request
+ that blew its timeout kept running inference for minutes afterwards.
+ """
+ monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "4")
+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")
+ router = ContentRouter()
+ kompress = _RecordingKompress()
+ monkeypatch.setattr(router, "_get_kompress", lambda: kompress)
+
+ _compress_through_executor(router, _kompress_payload(6))
+
+ assert len(kompress.deadlines) >= 6, kompress.deadlines
+ assert None not in kompress.deadlines, kompress.deadlines
+ assert len(set(kompress.deadlines)) == 1, kompress.deadlines
+
+
+def test_openai_responses_units_after_the_deadline_pass_through_unchanged(monkeypatch):
+ """Finished units are kept; units not started by the deadline are not started."""
+ monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "1")
+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "200")
+ router = ContentRouter()
+ kompress = _RecordingKompress(delay=0.3)
+ monkeypatch.setattr(router, "_get_kompress", lambda: kompress)
+ payload = _kompress_payload(4)
+ original = json.loads(json.dumps(payload))
+
+ result = _compress_through_executor(router, payload)
+
+ assert len(kompress.deadlines) == 1
+ outputs = [item["output"] for item in result[0]["input"]]
+ assert outputs[0] != original["input"][0]["output"]
+ assert outputs[1:] == [item["output"] for item in original["input"][1:]]
+ assert result[2] > 0
+
+
+ ("deadline_ms", "timeout", "budget"),
+ [("20000", 30.0, 20.0), ("20000", 5.0, 3.75), ("0", 30.0, None)],
+ ids=["deadline-binds", "timeout-binds", "disabled"],
+)
+def test_openai_responses_deadline_budget_stays_inside_the_timeout(
+ monkeypatch, deadline_ms, timeout, budget
+):
+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", deadline_ms)
+ before = time.perf_counter()
+ origin = openai_handler._openai_responses_deadline_started_at(timeout)
+ after = time.perf_counter()
+
+ if budget is None:
+ assert origin is None
+ return
+ expires_at = origin + int(deadline_ms) / 1000
+ assert before + budget <= expires_at <= after + budget
+
+
@pytest.mark.parametrize(
("compress_user_messages", "expected_user_text"),
[(False, None), (None, "compressed history")],
diff --git a/tests/test_proxy_openai.py b/tests/test_proxy_openai.py
index f705746ad..f7244b1ac 100644
--- a/tests/test_proxy_openai.py
+++ b/tests/test_proxy_openai.py
@@ -132,7 +132,10 @@ def test_native_responses_compressor_scopes_the_exclusion_per_call() -> None:
asyncio.run(_run())
- assert [{key: value for key, value in call.items() if key != "timing"} for call in seen] == [
+ assert [
+ {key: value for key, value in call.items() if key not in {"timing", "deadline_started_at"}}
+ for call in seen
+ ] == [
{"client": "opencode"},
{"client": None},
]
@@ -445,4 +448,6 @@ def test_websocket_route_carries_the_client_decision(
asyncio.run(handler.handle_openai_responses_ws(client_ws))
assert seen, "the Responses compressor was never reached on the WS path"
- assert {k: v for k, v in seen[0].items() if k != "timing"} == expected
+ assert {
+ k: v for k, v in seen[0].items() if k not in {"timing", "deadline_started_at"}
+ } == expected