Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): offload /v1/compress to the compression executor to stop blocking the loop (#1501)

Parideboy27e010e38f37
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 886905e90..720eb722c 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -6152,10 +6152,18 @@ class OpenAIHandlerMixin:
if protect_analysis_context is not None:
pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- **pipeline_kwargs,
+ # Offload the CPU-bound pipeline to the bounded compression executor
+ # (mirrors the request handlers above). Running apply() inline blocked
+ # the single event loop on a large payload, so even GET /health stalled
+ # until it finished (#718). The executor also enforces a timeout so a
+ # too-large body fails fast instead of hanging forever.
+ result = await self._run_compression_in_executor(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ **pipeline_kwargs,
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
)
return JSONResponse(
@@ -6174,6 +6182,23 @@ class OpenAIHandlerMixin:
"ccr_hashes": result.markers_inserted,
}
)
+ except TimeoutError:
+ logger.warning(
+ "Compression timed out after %.0fs (payload too large)",
+ COMPRESSION_TIMEOUT_SECONDS,
+ )
+ return JSONResponse(
+ status_code=503,
+ content={
+ "error": {
+ "type": "compression_timeout",
+ "message": (
+ "Compression exceeded "
+ f"{COMPRESSION_TIMEOUT_SECONDS:.0f}s; payload too large."
+ ),
+ }
+ },
+ )
except Exception as e:
logger.exception("Compression failed: %s", e)
return JSONResponse(
diff --git a/tests/test_proxy_compress_endpoint.py b/tests/test_proxy_compress_endpoint.py
index daa20e0d9..bb11dabc6 100644
--- a/tests/test_proxy_compress_endpoint.py
+++ b/tests/test_proxy_compress_endpoint.py
@@ -250,3 +250,75 @@ class TestCompressEndpointCompression:
assert data["tokens_before"] >= 0
assert data["tokens_after"] >= 0
assert isinstance(data["transforms_applied"], list)
+
+
+class TestCompressEndpointDoesNotBlockLoop:
+ """/v1/compress must offload to the compression executor so a slow/large
+ payload cannot freeze the single event loop (#718)."""
+
+ async def test_compress_does_not_block_liveness(self, monkeypatch):
+ import asyncio
+ import threading
+ from types import SimpleNamespace
+
+ import httpx
+
+ config = ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+ proxy = app.state.proxy
+
+ entered = threading.Event()
+ release = threading.Event()
+
+ def blocking_apply(**kwargs):
+ # Stand in for a large CPU-bound compression: blocks its worker until
+ # released. If this ran inline on the loop (the bug), the loop would
+ # be frozen and /livez below could not be served.
+ entered.set()
+ release.wait(timeout=10)
+ return SimpleNamespace(
+ messages=kwargs["messages"],
+ tokens_before=10,
+ tokens_after=5,
+ transforms_applied=[],
+ transforms_summary={},
+ markers_inserted=[],
+ )
+
+ monkeypatch.setattr(proxy.openai_pipeline, "apply", blocking_apply)
+
+ # /v1/compress is loopback-gated (#1227) — present as 127.0.0.1.
+ transport = httpx.ASGITransport(app=app, client=("127.0.0.1", 12345))
+ async with httpx.AsyncClient(transport=transport, base_url="http://127.0.0.1") as client:
+ compress = asyncio.create_task(
+ client.post(
+ "/v1/compress",
+ json={
+ "messages": [{"role": "user", "content": "hello world"}],
+ "model": "gpt-4",
+ },
+ )
+ )
+ # Wait until the compression is actually in flight (running in the
+ # executor thread), then prove the loop is still responsive.
+ for _ in range(200):
+ if entered.is_set():
+ break
+ await asyncio.sleep(0.01)
+ assert entered.is_set(), "compression never started"
+
+ livez = await asyncio.wait_for(client.get("/livez"), timeout=5)
+ assert livez.status_code == 200
+ assert livez.json()["alive"] is True
+ # The compression is still blocked — /livez was served concurrently.
+ assert not compress.done()
+
+ release.set()
+ resp = await asyncio.wait_for(compress, timeout=5)
+ assert resp.status_code == 200
+ assert resp.json()["tokens_saved"] == 5