Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(kompress): bound the inference deadline to the request, not each block (#3693)
gglucass7f2766ca4beb
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex cec1039dd..41fcd749d 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -1866,6 +1866,13 @@ class _PerRequestRuntimeState:# a compression-policy misapplication.protect_read_tool_ids: set[str] = field(default_factory=set)protect_read_msg_indices: set[int] = field(default_factory=set)+ # ``time.perf_counter()`` origin shared by every kompress call this+ # request makes. ``HEADROOM_COMPRESSION_DEADLINE_MS`` is a deadline on+ # kompress inference, but each ``compress()`` call started its own clock,+ # so the budget bounded a BLOCK and a request with a dozen blocks could+ # spend a dozen budgets. Set once per ``apply()``; see+ # ``_try_ml_compressor``.+ kompress_deadline_started_at: float | None = None# Monotonic counter so each ContentRouter instance gets a uniquely named@@ -4232,6 +4239,18 @@ class ContentRouter(Transform):# don't accept the kwarg are unaffected on the common path.if protected:compress_kwargs["ccr_original"] = content+ # One deadline for the whole request, not one per+ # block. Without this a request with N compressible+ # blocks gets N full deadlines and can run far past+ # the pipeline's own compression timeout; the worker+ # that overruns cannot be preempted, so it opens the+ # timeout-debt quarantine and every request behind it+ # forwards with no compression at all.+ deadline_origin = self._runtime_state_var.get().kompress_deadline_started_at+ if deadline_origin is not None and getattr(+ compressor, "shares_request_deadline", False+ ):+ compress_kwargs["_deadline_started_at"] = deadline_originresult = compressor.compress(text_to_compress, **compress_kwargs)compressed = result.compressedcompressed_tokens = result.compressed_tokens@@ -5141,7 +5160,12 @@ class ContentRouter(Transform):# call does. No `reset()` is needed: every `apply()` call installs# its own fresh object up front, so the next call on a reused worker# thread simply overwrites the ambient value before reading it.- self._runtime_state_var.set(_PerRequestRuntimeState())+ # The kompress deadline origin is stamped HERE, with the rest of the+ # per-request state, so every block this call compresses draws down one+ # shared budget instead of restarting it.+ self._runtime_state_var.set(+ _PerRequestRuntimeState(kompress_deadline_started_at=time.perf_counter())+ )# Pre-process: Read lifecycle management (stale/superseded detection)if self.config.read_lifecycle.enabled:diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex c220f7cdb..64cffc12f 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -1416,6 +1416,14 @@ class KompressCompressor(Transform):name: str = "kompress_compressor"+ # ``compress()`` accepts ``_deadline_started_at``, so a caller that+ # compresses many blocks for ONE request can hand every call the same+ # origin and have the deadline bound the request. Duck-typed rather than+ # isinstance-checked at the call site because ``RemoteKompressCompressor``+ # is the other compressor the router may get back and its ``compress()``+ # does not take the argument.+ shares_request_deadline: bool = True+def __init__(self, config: KompressConfig | None = None):self.config = config or KompressConfig()# Set by the preload canary when inference is too slow to be useful, or bydiff --git a/tests/test_transforms/test_kompress_deadline.py b/tests/test_transforms/test_kompress_deadline.pyindex 2ae94f458..2af170040 100644--- a/tests/test_transforms/test_kompress_deadline.py+++ b/tests/test_transforms/test_kompress_deadline.py@@ -108,3 +108,113 @@ def test_compress_partial_run_keeps_processed_head_plus_verbatim_tail(# chunk 1 tripped the deadline -> its words kept verbatim (all present)for i in range(half, n_words):assert f"w{i}" in out+++def _two_block_messages(salt: str = "a") -> list[dict]:+ """Two tool results, each big enough to reach the kompress stage.++ ``salt`` varies the text so a second request is not served from the+ router's content cache.+ """+ return [+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": f"toolu_{n}",+ "content": " ".join(+ f'{{"file":"src/mod_{i}.py","line":{i},"text":"payload {salt}{n}"}}'+ for i in range(160)+ ),+ }+ for n in range(2)+ ],+ }+ ]+++def _tokenizer():+ from headroom.providers import OpenAIProvider+ from headroom.tokenizer import Tokenizer++ provider = OpenAIProvider()+ return Tokenizer(provider.get_token_counter("gpt-4o"), "gpt-4o")+++class _RecordingKompress:+ """Stands in for KompressCompressor, recording the deadline it is handed."""++ shares_request_deadline = True++ def __init__(self):+ self.deadlines: list[float | None] = []++ def is_ready(self) -> bool:+ return True++ def ensure_background_load(self) -> None: # pragma: no cover - never reached+ pass++ def compress(self, content, **kwargs):+ from types import SimpleNamespace++ self.deadlines.append(kwargs.get("_deadline_started_at"))+ compressed = " ".join(content.split()[:20])+ return SimpleNamespace(compressed=compressed, compressed_tokens=len(compressed.split()))+++def test_every_block_of_one_request_draws_down_the_same_deadline(monkeypatch):+ """The budget must bound the REQUEST, not each block.++ ``compress()`` starts its own clock when no origin is passed, so a request+ with N compressible blocks used to get N full deadlines. That is how a+ single request runs past the pipeline's compression timeout; the worker+ that overruns cannot be preempted, so it opens the timeout-debt quarantine+ and every request queued behind it forwards with no compression at all.+ """+ from headroom.transforms.content_router import ContentRouter++ router = ContentRouter()+ fake = _RecordingKompress()+ monkeypatch.setattr(router, "_get_kompress", lambda: fake)++ router.apply(_two_block_messages(), _tokenizer(), force_kompress=True)++ assert len(fake.deadlines) >= 2, f"expected both blocks to reach kompress: {fake.deadlines}"+ assert all(d is not None for d in fake.deadlines), fake.deadlines+ assert len(set(fake.deadlines)) == 1, f"each block restarted the clock: {fake.deadlines}"+++def test_a_second_request_gets_a_fresh_deadline(monkeypatch):+ """Sharing is per-request: the next request must not inherit a spent budget."""+ from headroom.transforms.content_router import ContentRouter++ router = ContentRouter()+ fake = _RecordingKompress()+ monkeypatch.setattr(router, "_get_kompress", lambda: fake)+ tokenizer = _tokenizer()++ router.apply(_two_block_messages("first"), tokenizer, force_kompress=True)+ first = set(fake.deadlines)+ router.apply(_two_block_messages("second"), tokenizer, force_kompress=True)++ assert len(set(fake.deadlines) - first) == 1, (+ f"second request reused a deadline: {fake.deadlines}"+ )+++def test_a_compressor_that_cannot_take_the_deadline_is_not_handed_one(monkeypatch):+ """``RemoteKompressCompressor.compress()`` has no ``_deadline_started_at``+ parameter, so passing it would raise TypeError and fail compression open."""+ from headroom.transforms.content_router import ContentRouter++ router = ContentRouter()+ fake = _RecordingKompress()+ fake.shares_request_deadline = False+ monkeypatch.setattr(router, "_get_kompress", lambda: fake)++ router.apply(_two_block_messages(), _tokenizer(), force_kompress=True)++ assert fake.deadlines, "the stub should still have been called"+ assert all(d is None for d in fake.deadlines), fake.deadlines