Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: [BUG] A timeout in one image compression call resets the shared worker pool and affects concurrent requests (#4528)
Nandana Dileepe9bcfe73fadf
diff --git a/headroom/proxy/image_isolation.py b/headroom/proxy/image_isolation.pyindex 1e92b229c..4bd23cd33 100644--- a/headroom/proxy/image_isolation.py+++ b/headroom/proxy/image_isolation.py@@ -20,6 +20,15 @@ logger = logging.getLogger("headroom.proxy")_IMAGE_POOL_LOCK = threading.Lock()_IMAGE_POOL: ProcessPoolExecutor | None = None+# Number of calls currently awaiting a result from the *published*+# ``_IMAGE_POOL``. A timeout-driven reset tears down the whole single-worker+# pool (#4418), so it must only run when the failing call is the *only* one+# using that pool. The counter is paired with the published pool under+# ``_IMAGE_POOL_LOCK``: acquire creates/returns the pool and increments in one+# critical section, and unpublishing zeros the counter so a release of an+# already-reset pool cannot underflow a replacement.+_IMAGE_POOL_INFLIGHT = 0+# Per-worker-process cached compressor. The image pool is a persistent# single-worker ProcessPoolExecutor, so building a fresh ImageCompressor (and,# inside it, a fresh ONNX router loading native ort.InferenceSession models) on@@ -115,15 +124,58 @@ _IMAGE_WORKER: Callable[] = _compress_messages_worker-def _image_pool() -> ProcessPoolExecutor:+class _PoolJobCancelled(Exception):+ """The process-pool job was cancelled (typically by a pool reset).++ ``ProcessPoolExecutor.shutdown(cancel_futures=True)`` cancels the wrapped+ concurrent.futures job, which asyncio surfaces as ``CancelledError``. That+ is not a request to cancel *this* task, and ``Task.cancelling()`` (the+ discriminator) only exists on Python 3.11+. Mapping pool-job cancellation+ onto a regular exception lets genuine caller cancellation propagate from+ ``wait_for`` on 3.10 as well as 3.11+.+ """+++def _ensure_image_pool_locked() -> ProcessPoolExecutor:+ """Return the published pool, creating it if needed. Caller holds the lock."""global _IMAGE_POOL+ if _IMAGE_POOL is None:+ _IMAGE_POOL = ProcessPoolExecutor(+ max_workers=1,+ mp_context=multiprocessing.get_context("spawn"),+ )+ return _IMAGE_POOL+++def _image_pool() -> ProcessPoolExecutor:with _IMAGE_POOL_LOCK:- if _IMAGE_POOL is None:- _IMAGE_POOL = ProcessPoolExecutor(- max_workers=1,- mp_context=multiprocessing.get_context("spawn"),- )- return _IMAGE_POOL+ return _ensure_image_pool_locked()+++def _acquire_image_pool() -> ProcessPoolExecutor:+ """Return the shared pool and record one more call waiting on it.++ Pool lookup/creation and the inflight increment share one lock so a+ concurrent reset cannot unpublished the pool after we observed it and+ before we counted this call.+ """+ global _IMAGE_POOL_INFLIGHT+ with _IMAGE_POOL_LOCK:+ pool = _ensure_image_pool_locked()+ _IMAGE_POOL_INFLIGHT += 1+ return pool+++def _release_image_pool(pool: ProcessPoolExecutor) -> None:+ """Record that a call waiting on ``pool`` has finished.++ Inflight counts the currently published pool. A release for a pool that+ was already unpublished is a no-op so it cannot underflow a replacement.+ """+ global _IMAGE_POOL_INFLIGHT+ with _IMAGE_POOL_LOCK:+ if _IMAGE_POOL is pool and _IMAGE_POOL_INFLIGHT > 0:+ _IMAGE_POOL_INFLIGHT -= 1def _reset_image_pool(pool: ProcessPoolExecutor | None = None) -> None:@@ -134,13 +186,22 @@ def _reset_image_pool(pool: ProcessPoolExecutor | None = None) -> None:discarded, while the next request cold-loaded the models in a fresh workerbeside it. Only the current pool is unpublished, so a call that failsafter another request already rebuilt the pool leaves the rebuilt one alone.++ The reset kills the single shared worker, so it also cancels or kills every+ other call's queued or running job (#4418). Refuse the destructive teardown+ while any other call is still using the pool: those requests fail open on+ their own and keep their compression instead of losing it to this one's+ timeout. Usage is checked under the same lock as acquire."""- global _IMAGE_POOL+ global _IMAGE_POOL, _IMAGE_POOL_INFLIGHTwith _IMAGE_POOL_LOCK:if pool is None:pool = _IMAGE_POOL+ if _IMAGE_POOL_INFLIGHT > 1 and _IMAGE_POOL is pool:+ returnif _IMAGE_POOL is pool:_IMAGE_POOL = None+ _IMAGE_POOL_INFLIGHT = 0if pool is None:return# shutdown() sets _processes to None, so take the workers first.@@ -150,6 +211,51 @@ def _reset_image_pool(pool: ProcessPoolExecutor | None = None) -> None:worker.kill()+async def _await_pool_job(+ future: asyncio.Future[Any],+ timeout: float,+) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:+ """Await ``future`` without treating its cancellation as task cancellation.++ A cancelled executor future and a cancelled caller both used to surface as+ ``asyncio.CancelledError``. On Python 3.10 there is no ``Task.cancelling()``+ to tell them apart, so the previous handler either leaked ``AttributeError``+ or could not propagate genuine cancellation. Complete a private waiter+ instead: executor cancellation becomes ``_PoolJobCancelled``, while+ ``wait_for`` only raises ``CancelledError`` when *this* task is cancelled.++ Timeout and caller cancellation complete only that private waiter, so they+ would otherwise leave the ``run_in_executor`` future queued. Cancel it+ best-effort: a still-pending job must not run after this call already+ failed open. A job that is already running cannot be cancelled; isolation+ still refuses to kill the worker while another call is using the pool.+ """+ loop = asyncio.get_running_loop()+ waiter: asyncio.Future[Any] = loop.create_future()++ def _propagate(done: asyncio.Future[Any]) -> None:+ if waiter.done():+ return+ try:+ if done.cancelled():+ waiter.set_exception(_PoolJobCancelled())+ return+ exc = done.exception()+ if exc is not None:+ waiter.set_exception(exc)+ return+ waiter.set_result(done.result())+ except asyncio.InvalidStateError:+ return++ future.add_done_callback(_propagate)+ try:+ return await asyncio.wait_for(waiter, timeout=timeout)+ finally:+ if not future.done():+ future.cancel()++async def run_image_compression_isolated(messages: list[dict[str, Any]],provider: str,@@ -158,11 +264,17 @@ async def run_image_compression_isolated() -> tuple[list[dict[str, Any]], dict[str, Any] | None]:loop = asyncio.get_running_loop()pool: ProcessPoolExecutor | None = None+ acquired = Falsetry:deadline = time.time() + timeout * _OCR_BUDGET_FRACTION- pool = _image_pool()+ pool = _acquire_image_pool()+ acquired = Truefuture = loop.run_in_executor(pool, _IMAGE_WORKER, messages, provider, deadline)- return await asyncio.wait_for(future, timeout=timeout)+ return await _await_pool_job(future, timeout)+ except _PoolJobCancelled:+ logger.warning("Image compression worker cancelled; forwarding original image payload")+ _reset_image_pool(pool)+ return messages, Noneexcept BrokenProcessPool:logger.warning("Image compression worker crashed; forwarding original image payload")_reset_image_pool(pool)@@ -178,3 +290,6 @@ async def run_image_compression_isolated()_reset_image_pool(pool)return messages, None+ finally:+ if acquired and pool is not None:+ _release_image_pool(pool)diff --git a/tests/test_image_compression_isolation.py b/tests/test_image_compression_isolation.pyindex a7fde07e4..6ec507511 100644--- a/tests/test_image_compression_isolation.py+++ b/tests/test_image_compression_isolation.py@@ -1,6 +1,10 @@from __future__ import annotations+import asyncioimport inspect+import threading+import time+from pathlib import Pathimport pytest@@ -12,10 +16,12 @@ from headroom.proxy.handlers.openai import OpenAIHandlerMixin@pytest.fixture(autouse=True)def _reset_image_pool() -> None:image_isolation._reset_image_pool()+ image_isolation._IMAGE_POOL_INFLIGHT = 0try:yieldfinally:image_isolation._reset_image_pool()+ image_isolation._IMAGE_POOL_INFLIGHT = 0image_isolation._IMAGE_WORKER = image_isolation._compress_messages_worker@@ -106,6 +112,283 @@ async def test_worker_timeout_kills_the_abandoned_worker(monkeypatch) -> None:assert not worker.is_alive()+def _interference_worker(+ messages: list[dict[str, object]],+ provider: str,+ deadline: float | None = None,+) -> tuple[list[dict[str, object]], dict[str, object] | None]:+ if messages[-1]["content"] == "slow":+ time.sleep(0.3)+ raise RuntimeError("boom")+ return [{"role": "user", "content": "compressed:" + str(messages[-1]["content"])}], {+ "technique": "preserve"+ }+++def _queued_abandon_worker(+ messages: list[dict[str, object]],+ provider: str,+ deadline: float | None = None,+) -> tuple[list[dict[str, object]], dict[str, object] | None]:+ payload = messages[-1]["content"]+ if not isinstance(payload, dict):+ raise TypeError(f"expected coordination payload, got {type(payload)!r}")+ kind = payload["kind"]+ if kind == "peer":+ Path(str(payload["started"])).write_text("1", encoding="utf-8", newline="\n")+ release = Path(str(payload["release"]))+ give_up = time.time() + 15+ while not release.exists() and time.time() < give_up:+ time.sleep(0.01)+ Path(str(payload["done"])).write_text("1", encoding="utf-8", newline="\n")+ return [{"role": "user", "content": "compressed:peer"}], {"technique": "preserve"}+ if kind == "filler":+ return [{"role": "user", "content": "compressed:filler"}], {"technique": "preserve"}+ if kind == "target":+ Path(str(payload["ran"])).write_text("1", encoding="utf-8", newline="\n")+ return [{"role": "user", "content": "compressed:target"}], {"technique": "preserve"}+ raise RuntimeError(f"unknown kind:{kind}")++[email protected]("abandon", ["timeout", "cancel"])+async def test_abandoned_queued_job_never_runs_while_peer_completes(+ monkeypatch, tmp_path, abandon: str+) -> None:+ # A timeout or caller cancel used to complete only the private waiter, so+ # the still-queued executor job could run after the request already failed+ # open. Isolation correctly refuses to tear down the pool while a peer is+ # inflight; this call's pending future must still be cancelled so the+ # abandoned job never runs and the unrelated peer still finishes.+ monkeypatch.setattr(image_isolation, "_IMAGE_WORKER", _queued_abandon_worker)++ started = tmp_path / "peer-started"+ release = tmp_path / "peer-release"+ done = tmp_path / "peer-done"+ ran = tmp_path / "target-ran"+ peer_msgs = [+ {+ "role": "user",+ "content": {+ "kind": "peer",+ "started": str(started),+ "release": str(release),+ "done": str(done),+ },+ }+ ]+ target_msgs = [{"role": "user", "content": {"kind": "target", "ran": str(ran)}}]+ # ProcessPoolExecutor prefetches max_workers + 1 items onto its call+ # queue and marks those futures running, which makes cancel() a no-op.+ # Queue fillers so the abandoned target stays pending behind that prefetch.+ filler_count = 3++ loop = asyncio.get_running_loop()+ real_run_in_executor = loop.run_in_executor+ executor_futures: list[asyncio.Future[object]] = []++ def _track_run_in_executor(*args: object, **kwargs: object):+ future = real_run_in_executor(*args, **kwargs)+ executor_futures.append(future)+ return future++ monkeypatch.setattr(loop, "run_in_executor", _track_run_in_executor)++ peer_task = asyncio.create_task(+ image_isolation.run_image_compression_isolated(peer_msgs, provider="openai", timeout=15.0)+ )+ deadline = time.time() + 5+ while not started.exists() and time.time() < deadline:+ await asyncio.sleep(0.01)+ assert started.exists(), "peer never occupied the shared worker"++ filler_tasks = [+ asyncio.create_task(+ image_isolation.run_image_compression_isolated(+ [{"role": "user", "content": {"kind": "filler", "n": n}}],+ provider="openai",+ timeout=15.0,+ )+ )+ for n in range(filler_count)+ ]+ target_task = asyncio.create_task(+ image_isolation.run_image_compression_isolated(+ target_msgs,+ provider="openai",+ timeout=0.05 if abandon == "timeout" else 15.0,+ )+ )+ expected_jobs = 1 + filler_count + 1+ submitted = time.time() + 5+ while len(executor_futures) < expected_jobs and time.time() < submitted:+ await asyncio.sleep(0.01)+ assert len(executor_futures) >= expected_jobs+ target_future = executor_futures[-1]+ assert not target_future.done()++ if abandon == "timeout":+ target_returned, target_result = await target_task+ assert target_returned is target_msgs+ assert target_result is None+ else:+ target_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await target_task++ assert target_future.cancelled()+ assert not ran.exists()++ release.write_text("1", encoding="utf-8", newline="\n")+ peer_returned, peer_result = await peer_task+ assert peer_returned[-1]["content"] == "compressed:peer"+ assert peer_result == {"technique": "preserve"}+ assert not executor_futures[0].cancelled()++ for filler_returned, filler_result in await asyncio.gather(*filler_tasks):+ assert filler_returned[-1]["content"] == "compressed:filler"+ assert filler_result == {"technique": "preserve"}++ await asyncio.sleep(0.2)+ assert not ran.exists()+ assert done.read_text(encoding="utf-8") == "1"+++async def test_one_timeout_does_not_destroy_a_concurrent_request(monkeypatch) -> None:+ # #4418: request A times out while request B's image job is still queued on+ # the single shared worker. A's reset used to shutdown(cancel_futures=True)+ # and kill the worker, so B lost its compression and failed open. A timeout+ # in one call must leave the other request's job alone.+ monkeypatch.setattr(image_isolation, "_IMAGE_WORKER", _interference_worker)+ slow = [{"role": "user", "content": "slow"}]+ fast = [{"role": "user", "content": "fast"}]++ async def _call(messages: list[dict[str, object]], timeout: float):+ return await image_isolation.run_image_compression_isolated(+ messages, provider="openai", timeout=timeout+ )++ a_task = asyncio.create_task(_call(slow, 0.1))+ await asyncio.sleep(0.03)+ b_task = asyncio.create_task(_call(fast, 5.0))++ a_returned, a_result = await a_task+ b_returned, b_result = await b_task++ # A failed open: its own original payload, no result.+ assert a_returned is slow+ assert a_result is None++ # B was not harmed: it still got its compressed payload.+ assert b_returned[-1]["content"] == "compressed:fast"+ assert b_result == {"technique": "preserve"}+++async def test_executor_future_cancellation_fails_open(monkeypatch) -> None:+ # A future cancelled by a pool teardown (shutdown(cancel_futures=True))+ # must fail open on Python 3.10 as well as 3.11+. Task.cancelling() does+ # not exist on 3.10; discriminating via that API leaked AttributeError+ # instead of forwarding the original payload.+ messages = _messages()+ loop = asyncio.get_running_loop()+ cancelled: asyncio.Future[object] = loop.create_future()+ cancelled.cancel()+ monkeypatch.setattr(loop, "run_in_executor", lambda *args, **kwargs: cancelled)++ returned, result = await image_isolation.run_image_compression_isolated(+ messages, provider="openai", timeout=5.0+ )++ assert returned is messages+ assert result is None+++async def test_caller_cancellation_propagates(monkeypatch) -> None:+ # Genuine task cancellation must still propagate, including on Python 3.10+ # where Task.cancelling() is unavailable.+ entered = asyncio.Event()+ orig_wait_for = image_isolation.asyncio.wait_for++ async def _tapping_wait_for(aw, *args, **kwargs):+ entered.set()+ return await orig_wait_for(aw, *args, **kwargs)++ monkeypatch.setattr(image_isolation.asyncio, "wait_for", _tapping_wait_for)+ monkeypatch.setattr(image_isolation, "_IMAGE_WORKER", image_isolation._hang_worker)++ messages = _messages()+ task = asyncio.create_task(+ image_isolation.run_image_compression_isolated(messages, provider="openai", timeout=30.0)+ )+ await entered.wait()+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+++def test_acquire_counts_before_returning_the_pool() -> None:+ # Acquire must publish the pool and increment inflight under one lock so a+ # concurrent reset cannot unpublished it as idle before this call is counted.+ pool = image_isolation._acquire_image_pool()+ try:+ assert pool is image_isolation._IMAGE_POOL+ assert image_isolation._IMAGE_POOL_INFLIGHT == 1+ finally:+ image_isolation._release_image_pool(pool)+ assert image_isolation._IMAGE_POOL_INFLIGHT == 0+++def test_release_of_unpublished_pool_does_not_underflow_replacement() -> None:+ old = image_isolation._acquire_image_pool()+ image_isolation._reset_image_pool(old)+ replacement = image_isolation._acquire_image_pool()+ try:+ assert replacement is not old+ assert image_isolation._IMAGE_POOL_INFLIGHT == 1+ image_isolation._release_image_pool(old)+ assert image_isolation._IMAGE_POOL_INFLIGHT == 1+ finally:+ image_isolation._release_image_pool(replacement)+++def test_concurrent_event_loop_threads_keep_compression(monkeypatch) -> None:+ # Separate event-loop threads share the pool lock. A timeout on one thread+ # used to reset the pool after the other had already observed it but before+ # its inflight count, so that call submitted to a shut-down pool and+ # forwarded the image uncompressed.+ monkeypatch.setattr(image_isolation, "_IMAGE_WORKER", _interference_worker)+ slow = [{"role": "user", "content": "slow"}]+ fast = [{"role": "user", "content": "fast"}]+ outcomes: dict[str, tuple[list[dict[str, object]], dict[str, object] | None]] = {}+ errors: list[BaseException] = []++ def _run(name: str, messages: list[dict[str, object]], timeout: float) -> None:+ async def _call():+ return await image_isolation.run_image_compression_isolated(+ messages, provider="openai", timeout=timeout+ )++ try:+ outcomes[name] = asyncio.run(_call())+ except Exception as exc:+ errors.append(exc)++ slow_thread = threading.Thread(target=_run, args=("slow", slow, 0.1))+ fast_thread = threading.Thread(target=_run, args=("fast", fast, 5.0))+ slow_thread.start()+ time.sleep(0.03)+ fast_thread.start()+ slow_thread.join(timeout=10)+ fast_thread.join(timeout=10)++ assert errors == []+ a_returned, a_result = outcomes["slow"]+ b_returned, b_result = outcomes["fast"]+ assert a_returned is slow+ assert a_result is None+ assert b_returned[-1]["content"] == "compressed:fast"+ assert b_result == {"technique": "preserve"}++def test_late_failure_leaves_a_rebuilt_pool_alone() -> None:# A call that fails after another request already replaced its pool must# not drop (and kill the worker of) the replacement.diff --git a/tests/test_image_compression_offload.py b/tests/test_image_compression_offload.pyindex ae29cea7d..91f290835 100644--- a/tests/test_image_compression_offload.py+++ b/tests/test_image_compression_offload.py@@ -21,10 +21,12 @@ from headroom.proxy.handlers.openai import OpenAIHandlerMixin@pytest.fixture(autouse=True)def _reset_image_pool() -> None:image_isolation._reset_image_pool()+ image_isolation._IMAGE_POOL_INFLIGHT = 0try:yieldfinally:image_isolation._reset_image_pool()+ image_isolation._IMAGE_POOL_INFLIGHT = 0image_isolation._IMAGE_WORKER = image_isolation._compress_messages_worker