Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): share one compression deadline across a Responses request (#3938)
gglucass4d27d024b09f
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 9054a19cf..678be4579 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport asyncioimport contextlib+import contextvarsimport copyimport hashlibimport hmac@@ -664,6 +665,31 @@ def _openai_responses_unit_parallelism() -> int:return max(1, min(_OPENAI_RESPONSES_UNIT_PARALLELISM_MAX, requested))+def _openai_responses_deadline_started_at(timeout: float) -> float | None:+ """Deadline origin shared by every compress call of one Responses request.++ A Responses request fans out over many ``ContentRouter.compress`` calls,+ and Kompress starts a fresh ``HEADROOM_COMPRESSION_DEADLINE_MS`` clock in+ each one unless it is handed an origin. Without one, a request past its+ ``timeout`` keeps running inference long after the awaiter gave up, and+ that timeout debt quarantines compression for every request behind it.+ Stamped before the job is queued, because ``timeout`` counts queue wait+ too. ``None`` when the deadline is disabled (``0``).++ The budget is also capped at 75% of ``timeout``: the deadline only stops+ the NEXT Kompress chunk, so the chunk already running when it expires, the+ splice and the output serialization need the rest. Kompress checks+ ``now - origin > deadline``, so a shorter budget is a backdated origin.+ """+ from headroom.transforms.content_router import _compression_deadline_seconds++ deadline_s = _compression_deadline_seconds()+ if deadline_s <= 0:+ return None+ budget_s = min(deadline_s, 0.75 * timeout)+ return time.perf_counter() - (deadline_s - budget_s)++def _openai_responses_unit_executor() -> ThreadPoolExecutor:global _OPENAI_RESPONSES_UNIT_EXECUTORwith _OPENAI_RESPONSES_UNIT_EXECUTOR_LOCK:@@ -2130,6 +2156,7 @@ class OpenAIHandlerMixin:request_id: str,pass_id: str | None = None,timing: dict[str, float] | None = None,+ deadline_started_at: float | None = None,) -> tuple[dict[str, Any], bool, int, list[str], dict[str, int], list[str], int]:"""Run ContentRouter on OpenAI Responses text units.@@ -2723,6 +2750,7 @@ class OpenAIHandlerMixin:router=router,tokenizer=tokenizer,target_ratio=unit_target_ratio,+ deadline_started_at=deadline_started_at,)elapsed_ms = (time.perf_counter() - unit_started) * 1000.0return routed.slot, result, elapsed_ms@@ -2792,12 +2820,18 @@ class OpenAIHandlerMixin:0.0,)+ # Each pool task runs in its own copy of this Context: the router binds+ # the request's deadline per task (see share_request_deadline), and+ # pool threads would otherwise keep that binding between requests.parallelism = _openai_responses_unit_parallelism()if len(cache_misses) > 1 and parallelism > 1:executor = _openai_responses_unit_executor()for start in range(0, len(cache_misses), parallelism):batch = cache_misses[start : start + parallelism]- futures = [executor.submit(_compress_and_store, *item) for item in batch]+ futures = [+ executor.submit(contextvars.copy_context().run, _compress_and_store, *item)+ for item in batch+ ]for future in as_completed(futures):unit_idx, cache_key, routed_result = future.result()_record_routed_result(unit_idx, cache_key, routed_result)@@ -2822,6 +2856,7 @@ class OpenAIHandlerMixin:router=router,tokenizer=tokenizer,target_ratio=unit_target_ratio,+ deadline_started_at=deadline_started_at,)return results, (time.perf_counter() - batch_started) * 1000.0@@ -2835,7 +2870,10 @@ class OpenAIHandlerMixin:executor = _openai_responses_unit_executor()for start in range(0, len(small_batches), parallelism):batch_group = small_batches[start : start + parallelism]- futures = [executor.submit(_compress_batch, batch) for batch in batch_group]+ futures = [+ executor.submit(contextvars.copy_context().run, _compress_batch, batch)+ for batch in batch_group+ ]for future in as_completed(futures):_record_batch_result(future.result())else:@@ -3023,6 +3061,7 @@ class OpenAIHandlerMixin:timing: dict[str, float] | None = None,client: str | None = None,savings_tags: dict[str, Any] | None = None,+ deadline_started_at: float | None = None,) -> tuple[dict[str, Any], bool, int, list[str], str | None, int, int, int]:"""Compress an OpenAI Responses payload through the shared router.@@ -3290,6 +3329,7 @@ class OpenAIHandlerMixin:request_id=request_id,pass_id=pass_id,timing=timing_sink,+ deadline_started_at=deadline_started_at,)_add_timing("compression_live_units_total", live_units_started)if router_modified:@@ -3437,6 +3477,8 @@ class OpenAIHandlerMixin:exc_info=True,)+ deadline_started_at = _openai_responses_deadline_started_at(timeout)+def _compress(): # noqa: ANN202# Output shaping (opt-in via HEADROOM_OUTPUT_SHAPER) runs before# compression so the turn classifier sees the client's input as@@ -3456,6 +3498,7 @@ class OpenAIHandlerMixin:"request_id": request_id,"timing": timing,"client": client,+ "deadline_started_at": deadline_started_at,}if savings_tags is not None:compression_kwargs["savings_tags"] = savings_tags@@ -3470,7 +3513,12 @@ class OpenAIHandlerMixin:unsupported_kwarg = next((name- for name in ("savings_tags", "client", "timing")+ for name in (+ "savings_tags",+ "client",+ "timing",+ "deadline_started_at",+ )if f"unexpected keyword argument '{name}'" in str(exc)and name in compression_kwargs),diff --git a/headroom/transforms/compression_batches.py b/headroom/transforms/compression_batches.pyindex 4fae8ebeb..84b429c2d 100644--- a/headroom/transforms/compression_batches.py+++ b/headroom/transforms/compression_batches.py@@ -230,8 +230,15 @@ def compress_batch_with_router(router: ContentRouter,tokenizer: TokenCounterLike,target_ratio: float | None = None,+ deadline_started_at: float | None = None,) -> list[tuple[object, UnitCompressionResult]]:- """Compress one tagged batch and split only structurally valid output."""+ """Compress one tagged batch and split only structurally valid output.++ ``deadline_started_at`` works as in ``compress_unit_with_router``.+ """++ if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):+ return _passthrough_batch_results(batch, tokenizer=tokenizer, reason="deadline_exceeded")nonce = _batch_nonce(batch)batch_texts, marker_blocks = _protect_ccr_markers(batch, nonce)diff --git a/headroom/transforms/compression_units.py b/headroom/transforms/compression_units.pyindex c7296533f..6df3c31d0 100644--- a/headroom/transforms/compression_units.py+++ b/headroom/transforms/compression_units.py@@ -56,6 +56,8 @@ class CompressionUnit:# Responses handler's unit-result cache) and is surfaced# via RouterCompressionResult.cache_hit, not as a reason# here — no code path produces this category today+# - deadline: the request's compression deadline had passed before+# this unit started, so it was left unchangedUNIT_REASON_CATEGORIES = {None: "applied","protected_user_message": "protected_role",@@ -66,6 +68,7 @@ UNIT_REASON_CATEGORIES = {"router_no_change": "compressor_noop","already_compressed": "already_compressed","rejected_not_smaller": "rejected_not_smaller",+ "deadline_exceeded": "deadline",}@@ -221,11 +224,17 @@ def compress_unit_with_router(router: ContentRouter,tokenizer: TokenCounterLike,target_ratio: float | None = None,+ deadline_started_at: float | None = None,) -> UnitCompressionResult:"""Compress one safe text unit through ContentRouter.The final accept/reject gate uses the provider/model tokenizer, not therouter's internal word-count estimates.++ ``deadline_started_at`` is the deadline origin of the request this unit+ belongs to (see ``ContentRouter.share_request_deadline``). Once that+ deadline has passed the unit is returned unchanged without entering the+ router."""tokens_before = tokenizer.count_text(unit.text)@@ -266,6 +275,8 @@ def compress_unit_with_router(return _with_reason(reason=f"cache_zone_{unit.cache_zone}")if text_bytes < unit.min_bytes:return _with_reason(reason="below_unit_floor")+ if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):+ return _with_reason(reason="deadline_exceeded")prior_target_ratio = getattr(router, "_runtime_target_ratio", None)if target_ratio is not None:diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 667dfc54c..200657468 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -2394,6 +2394,27 @@ class ContentRouter(Transform):self._runtime_state_var.set(state)return state+ def share_request_deadline(self, started_at: float) -> bool:+ """Join a request whose kompress deadline started at ``started_at``.++ ``apply()`` stamps its own origin. A caller that fans ONE request out+ over many ``compress()`` calls (the OpenAI Responses unit adapter) calls+ this before each one instead, so every call draws down the same+ ``HEADROOM_COMPRESSION_DEADLINE_MS`` budget rather than restarting it.+ Binds a fresh ``_PerRequestRuntimeState`` in the CURRENT Context, so a+ worker-pool task must run in its own ``contextvars.copy_context()``.++ Returns ``False``, binding nothing, once the deadline has passed: the+ caller should leave the content unchanged rather than start new work.+ """+ deadline_s = _compression_deadline_seconds()+ if deadline_s and time.perf_counter() - started_at > deadline_s:+ return False+ self._runtime_state_var.set(+ _PerRequestRuntimeState(kompress_deadline_started_at=started_at)+ )+ return True+@propertydef _runtime_compression_policy(self) -> Any:return self._runtime_state_var.get().compression_policydiff --git a/tests/test_compression_batches.py b/tests/test_compression_batches.pyindex e6d5a286a..55c110ba0 100644--- a/tests/test_compression_batches.py+++ b/tests/test_compression_batches.py@@ -1,6 +1,7 @@from __future__ import annotationsimport re+import timefrom headroom.transforms.compression_batches import (CompressionBatchEntry,@@ -229,3 +230,27 @@ def test_batch_uses_utf8_bytes_for_cjk_small_units():assert batches[0].text_bytes == 1800assert all(result.modified for _, result in results)assert [result.compressed for _, result in results] == ["短"] * 4+++def test_batch_after_the_request_deadline_passes_through_without_routing(monkeypatch):+ from headroom.transforms.content_router import ContentRouter++ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")+ entries = [_entry(index, "x" * 150) for index in range(4)]+ batches, _ = build_compression_batches(entries, min_batch_bytes=512)+ router = ContentRouter()++ def _not_called(*_args, **_kwargs):+ raise AssertionError("a batch past the deadline must not enter the router")++ router.compress = _not_called++ results = compress_batch_with_router(+ batches[0],+ router=router,+ tokenizer=_CharacterCounter(),+ deadline_started_at=time.perf_counter() - 21,+ )++ assert [result.compressed for _, result in results] == ["x" * 150] * 4+ assert {result.reason for _, result in results} == {"deadline_exceeded"}diff --git a/tests/test_openai_responses_compression_units.py b/tests/test_openai_responses_compression_units.pyindex 3037d97cc..268db0a97 100644--- a/tests/test_openai_responses_compression_units.py+++ b/tests/test_openai_responses_compression_units.py@@ -1,7 +1,9 @@from __future__ import annotations+import asyncioimport jsonimport threading+import timefrom types import MethodType, SimpleNamespaceimport pytest@@ -1417,6 +1419,114 @@ def test_openai_responses_adapter_compresses_historical_messages_not_current_useassert any(t.startswith("router:openai:responses:message:") for t in transforms)+class _RecordingKompress:+ """Stands in for KompressCompressor, recording the deadline origin it gets."""++ shares_request_deadline = True++ def __init__(self, delay: float = 0.0) -> None:+ self.delay = delay+ self.deadlines: list[float | None] = []++ def is_ready(self) -> bool:+ return True++ def compress(self, content: str, **kwargs):+ self.deadlines.append(kwargs.get("_deadline_started_at"))+ time.sleep(self.delay)+ compressed = " ".join(content.split()[:20])+ return SimpleNamespace(compressed=compressed, compressed_tokens=20)+++def _kompress_payload(units: int) -> dict:+ return {+ "model": "gpt-5",+ "input": [+ {+ "type": "function_call_output",+ "call_id": f"call_{i}",+ "output": "\n".join(+ " ".join(f"unit{i}line{line}word{j}" for j in range(10)) for line in range(70)+ ),+ }+ for i in range(units)+ ],+ }+++def _compress_through_executor(router: ContentRouter, payload: dict) -> tuple:+ handler = _handler_with_router(router)++ async def _run_compression(fn, *, timeout): # noqa: ANN001, ANN202+ return fn()++ handler._run_compression_in_executor = _run_compression+ return asyncio.run(+ handler._compress_openai_responses_payload_in_executor(+ payload, model="gpt-5", request_id="req_deadline"+ )+ )+++def test_openai_responses_units_share_one_kompress_deadline(monkeypatch):+ """The fan-out must draw down ONE request deadline (#3693), pool threads too.++ Each unit is its own ContentRouter.compress call, and Kompress started a+ fresh HEADROOM_COMPRESSION_DEADLINE_MS clock in every one, so a request+ that blew its timeout kept running inference for minutes afterwards.+ """+ monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "4")+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "20000")+ router = ContentRouter()+ kompress = _RecordingKompress()+ monkeypatch.setattr(router, "_get_kompress", lambda: kompress)++ _compress_through_executor(router, _kompress_payload(6))++ assert len(kompress.deadlines) >= 6, kompress.deadlines+ assert None not in kompress.deadlines, kompress.deadlines+ assert len(set(kompress.deadlines)) == 1, kompress.deadlines+++def test_openai_responses_units_after_the_deadline_pass_through_unchanged(monkeypatch):+ """Finished units are kept; units not started by the deadline are not started."""+ monkeypatch.setenv("HEADROOM_TOOL_OUTPUT_COMPRESSION_PARALLELISM", "1")+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", "200")+ router = ContentRouter()+ kompress = _RecordingKompress(delay=0.3)+ monkeypatch.setattr(router, "_get_kompress", lambda: kompress)+ payload = _kompress_payload(4)+ original = json.loads(json.dumps(payload))++ result = _compress_through_executor(router, payload)++ assert len(kompress.deadlines) == 1+ outputs = [item["output"] for item in result[0]["input"]]+ assert outputs[0] != original["input"][0]["output"]+ assert outputs[1:] == [item["output"] for item in original["input"][1:]]+ assert result[2] > 0+++ ("deadline_ms", "timeout", "budget"),+ [("20000", 30.0, 20.0), ("20000", 5.0, 3.75), ("0", 30.0, None)],+ ids=["deadline-binds", "timeout-binds", "disabled"],+)+def test_openai_responses_deadline_budget_stays_inside_the_timeout(+ monkeypatch, deadline_ms, timeout, budget+):+ monkeypatch.setenv("HEADROOM_COMPRESSION_DEADLINE_MS", deadline_ms)+ before = time.perf_counter()+ origin = openai_handler._openai_responses_deadline_started_at(timeout)+ after = time.perf_counter()++ if budget is None:+ assert origin is None+ return+ expires_at = origin + int(deadline_ms) / 1000+ assert before + budget <= expires_at <= after + budget++@pytest.mark.parametrize(("compress_user_messages", "expected_user_text"),[(False, None), (None, "compressed history")],diff --git a/tests/test_proxy_openai.py b/tests/test_proxy_openai.pyindex f705746ad..f7244b1ac 100644--- a/tests/test_proxy_openai.py+++ b/tests/test_proxy_openai.py@@ -132,7 +132,10 @@ def test_native_responses_compressor_scopes_the_exclusion_per_call() -> None:asyncio.run(_run())- assert [{key: value for key, value in call.items() if key != "timing"} for call in seen] == [+ assert [+ {key: value for key, value in call.items() if key not in {"timing", "deadline_started_at"}}+ for call in seen+ ] == [{"client": "opencode"},{"client": None},]@@ -445,4 +448,6 @@ def test_websocket_route_carries_the_client_decision(asyncio.run(handler.handle_openai_responses_ws(client_ws))assert seen, "the Responses compressor was never reached on the WS path"- assert {k: v for k, v in seen[0].items() if k != "timing"} == expected+ assert {+ k: v for k, v in seen[0].items() if k not in {"timing", "deadline_started_at"}+ } == expected