Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): bound the inference deadline to the request, not each block (#3693)

gglucass7f2766ca4beb
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index cec1039dd..41fcd749d 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1866,6 +1866,13 @@ class _PerRequestRuntimeState:
# a compression-policy misapplication.
protect_read_tool_ids: set[str] = field(default_factory=set)
protect_read_msg_indices: set[int] = field(default_factory=set)
+ # ``time.perf_counter()`` origin shared by every kompress call this
+ # request makes. ``HEADROOM_COMPRESSION_DEADLINE_MS`` is a deadline on
+ # kompress inference, but each ``compress()`` call started its own clock,
+ # so the budget bounded a BLOCK and a request with a dozen blocks could
+ # spend a dozen budgets. Set once per ``apply()``; see
+ # ``_try_ml_compressor``.
+ kompress_deadline_started_at: float | None = None
# Monotonic counter so each ContentRouter instance gets a uniquely named
@@ -4232,6 +4239,18 @@ class ContentRouter(Transform):
# don't accept the kwarg are unaffected on the common path.
if protected:
compress_kwargs["ccr_original"] = content
+ # One deadline for the whole request, not one per
+ # block. Without this a request with N compressible
+ # blocks gets N full deadlines and can run far past
+ # the pipeline's own compression timeout; the worker
+ # that overruns cannot be preempted, so it opens the
+ # timeout-debt quarantine and every request behind it
+ # forwards with no compression at all.
+ deadline_origin = self._runtime_state_var.get().kompress_deadline_started_at
+ if deadline_origin is not None and getattr(
+ compressor, "shares_request_deadline", False
+ ):
+ compress_kwargs["_deadline_started_at"] = deadline_origin
result = compressor.compress(text_to_compress, **compress_kwargs)
compressed = result.compressed
compressed_tokens = result.compressed_tokens
@@ -5141,7 +5160,12 @@ class ContentRouter(Transform):
# call does. No `reset()` is needed: every `apply()` call installs
# its own fresh object up front, so the next call on a reused worker
# thread simply overwrites the ambient value before reading it.
- self._runtime_state_var.set(_PerRequestRuntimeState())
+ # The kompress deadline origin is stamped HERE, with the rest of the
+ # per-request state, so every block this call compresses draws down one
+ # shared budget instead of restarting it.
+ self._runtime_state_var.set(
+ _PerRequestRuntimeState(kompress_deadline_started_at=time.perf_counter())
+ )
# Pre-process: Read lifecycle management (stale/superseded detection)
if self.config.read_lifecycle.enabled:
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index c220f7cdb..64cffc12f 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -1416,6 +1416,14 @@ class KompressCompressor(Transform):
name: str = "kompress_compressor"
+ # ``compress()`` accepts ``_deadline_started_at``, so a caller that
+ # compresses many blocks for ONE request can hand every call the same
+ # origin and have the deadline bound the request. Duck-typed rather than
+ # isinstance-checked at the call site because ``RemoteKompressCompressor``
+ # is the other compressor the router may get back and its ``compress()``
+ # does not take the argument.
+ shares_request_deadline: bool = True
+
def __init__(self, config: KompressConfig | None = None):
self.config = config or KompressConfig()
# Set by the preload canary when inference is too slow to be useful, or by
diff --git a/tests/test_transforms/test_kompress_deadline.py b/tests/test_transforms/test_kompress_deadline.py
index 2ae94f458..2af170040 100644
--- a/tests/test_transforms/test_kompress_deadline.py
+++ b/tests/test_transforms/test_kompress_deadline.py
@@ -108,3 +108,113 @@ def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(
# chunk 1 tripped the deadline -> its words kept verbatim (all present)
for i in range(half, n_words):
assert f"w{i}" in out
+
+
+def _two_block_messages(salt: str = "a") -> list[dict]:
+ """Two tool results, each big enough to reach the kompress stage.
+
+ ``salt`` varies the text so a second request is not served from the
+ router's content cache.
+ """
+ return [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": f"toolu_{n}",
+ "content": " ".join(
+ f'{{"file":"src/mod_{i}.py","line":{i},"text":"payload {salt}{n}"}}'
+ for i in range(160)
+ ),
+ }
+ for n in range(2)
+ ],
+ }
+ ]
+
+
+def _tokenizer():
+ from headroom.providers import OpenAIProvider
+ from headroom.tokenizer import Tokenizer
+
+ provider = OpenAIProvider()
+ return Tokenizer(provider.get_token_counter("gpt-4o"), "gpt-4o")
+
+
+class _RecordingKompress:
+ """Stands in for KompressCompressor, recording the deadline it is handed."""
+
+ shares_request_deadline = True
+
+ def __init__(self):
+ self.deadlines: list[float | None] = []
+
+ def is_ready(self) -> bool:
+ return True
+
+ def ensure_background_load(self) -> None: # pragma: no cover - never reached
+ pass
+
+ def compress(self, content, **kwargs):
+ from types import SimpleNamespace
+
+ self.deadlines.append(kwargs.get("_deadline_started_at"))
+ compressed = " ".join(content.split()[:20])
+ return SimpleNamespace(compressed=compressed, compressed_tokens=len(compressed.split()))
+
+
+def test_every_block_of_one_request_draws_down_the_same_deadline(monkeypatch):
+ """The budget must bound the REQUEST, not each block.
+
+ ``compress()`` starts its own clock when no origin is passed, so a request
+ with N compressible blocks used to get N full deadlines. That is how a
+ single request runs past the pipeline's compression timeout; the worker
+ that overruns cannot be preempted, so it opens the timeout-debt quarantine
+ and every request queued behind it forwards with no compression at all.
+ """
+ from headroom.transforms.content_router import ContentRouter
+
+ router = ContentRouter()
+ fake = _RecordingKompress()
+ monkeypatch.setattr(router, "_get_kompress", lambda: fake)
+
+ router.apply(_two_block_messages(), _tokenizer(), force_kompress=True)
+
+ assert len(fake.deadlines) >= 2, f"expected both blocks to reach kompress: {fake.deadlines}"
+ assert all(d is not None for d in fake.deadlines), fake.deadlines
+ assert len(set(fake.deadlines)) == 1, f"each block restarted the clock: {fake.deadlines}"
+
+
+def test_a_second_request_gets_a_fresh_deadline(monkeypatch):
+ """Sharing is per-request: the next request must not inherit a spent budget."""
+ from headroom.transforms.content_router import ContentRouter
+
+ router = ContentRouter()
+ fake = _RecordingKompress()
+ monkeypatch.setattr(router, "_get_kompress", lambda: fake)
+ tokenizer = _tokenizer()
+
+ router.apply(_two_block_messages("first"), tokenizer, force_kompress=True)
+ first = set(fake.deadlines)
+ router.apply(_two_block_messages("second"), tokenizer, force_kompress=True)
+
+ assert len(set(fake.deadlines) - first) == 1, (
+ f"second request reused a deadline: {fake.deadlines}"
+ )
+
+
+def test_a_compressor_that_cannot_take_the_deadline_is_not_handed_one(monkeypatch):
+ """``RemoteKompressCompressor.compress()`` has no ``_deadline_started_at``
+ parameter, so passing it would raise TypeError and fail compression open."""
+ from headroom.transforms.content_router import ContentRouter
+
+ router = ContentRouter()
+ fake = _RecordingKompress()
+ fake.shares_request_deadline = False
+ monkeypatch.setattr(router, "_get_kompress", lambda: fake)
+
+ router.apply(_two_block_messages(), _tokenizer(), force_kompress=True)
+
+ assert fake.deadlines, "the stub should still have been called"
+ assert all(d is None for d in fake.deadlines), fake.deadlines