Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): offload /v1/compress to the compression executor to stop blocking the loop (#1501)
Parideboy27e010e38f37
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 886905e90..720eb722c 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -6152,10 +6152,18 @@ class OpenAIHandlerMixin:if protect_analysis_context is not None:pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)- result = self.openai_pipeline.apply(- messages=messages,- model=model,- **pipeline_kwargs,+ # Offload the CPU-bound pipeline to the bounded compression executor+ # (mirrors the request handlers above). Running apply() inline blocked+ # the single event loop on a large payload, so even GET /health stalled+ # until it finished (#718). The executor also enforces a timeout so a+ # too-large body fails fast instead of hanging forever.+ result = await self._run_compression_in_executor(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ **pipeline_kwargs,+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,)return JSONResponse(@@ -6174,6 +6182,23 @@ class OpenAIHandlerMixin:"ccr_hashes": result.markers_inserted,})+ except TimeoutError:+ logger.warning(+ "Compression timed out after %.0fs (payload too large)",+ COMPRESSION_TIMEOUT_SECONDS,+ )+ return JSONResponse(+ status_code=503,+ content={+ "error": {+ "type": "compression_timeout",+ "message": (+ "Compression exceeded "+ f"{COMPRESSION_TIMEOUT_SECONDS:.0f}s; payload too large."+ ),+ }+ },+ )except Exception as e:logger.exception("Compression failed: %s", e)return JSONResponse(diff --git a/tests/test_proxy_compress_endpoint.py b/tests/test_proxy_compress_endpoint.pyindex daa20e0d9..bb11dabc6 100644--- a/tests/test_proxy_compress_endpoint.py+++ b/tests/test_proxy_compress_endpoint.py@@ -250,3 +250,75 @@ class TestCompressEndpointCompression:assert data["tokens_before"] >= 0assert data["tokens_after"] >= 0assert isinstance(data["transforms_applied"], list)+++class TestCompressEndpointDoesNotBlockLoop:+ """/v1/compress must offload to the compression executor so a slow/large+ payload cannot freeze the single event loop (#718)."""++ async def test_compress_does_not_block_liveness(self, monkeypatch):+ import asyncio+ import threading+ from types import SimpleNamespace++ import httpx++ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ proxy = app.state.proxy++ entered = threading.Event()+ release = threading.Event()++ def blocking_apply(**kwargs):+ # Stand in for a large CPU-bound compression: blocks its worker until+ # released. If this ran inline on the loop (the bug), the loop would+ # be frozen and /livez below could not be served.+ entered.set()+ release.wait(timeout=10)+ return SimpleNamespace(+ messages=kwargs["messages"],+ tokens_before=10,+ tokens_after=5,+ transforms_applied=[],+ transforms_summary={},+ markers_inserted=[],+ )++ monkeypatch.setattr(proxy.openai_pipeline, "apply", blocking_apply)++ # /v1/compress is loopback-gated (#1227) — present as 127.0.0.1.+ transport = httpx.ASGITransport(app=app, client=("127.0.0.1", 12345))+ async with httpx.AsyncClient(transport=transport, base_url="http://127.0.0.1") as client:+ compress = asyncio.create_task(+ client.post(+ "/v1/compress",+ json={+ "messages": [{"role": "user", "content": "hello world"}],+ "model": "gpt-4",+ },+ )+ )+ # Wait until the compression is actually in flight (running in the+ # executor thread), then prove the loop is still responsive.+ for _ in range(200):+ if entered.is_set():+ break+ await asyncio.sleep(0.01)+ assert entered.is_set(), "compression never started"++ livez = await asyncio.wait_for(client.get("/livez"), timeout=5)+ assert livez.status_code == 200+ assert livez.json()["alive"] is True+ # The compression is still blocked — /livez was served concurrently.+ assert not compress.done()++ release.set()+ resp = await asyncio.wait_for(compress, timeout=5)+ assert resp.status_code == 200+ assert resp.json()["tokens_saved"] == 5