Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(proxy): bound pre-upstream Anthropic work with a semaphore

Adryan Eka Vandra1de753bd204f
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index a3863595e..0ea1bb03b 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -59,6 +59,20 @@ from .main import main
default=None,
help="Upstream connection timeout in seconds (default: 10)",
)
+ "--anthropic-pre-upstream-concurrency",
+ type=int,
+ default=None,
+ envvar="HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY",
+ help=(
+ "Cap the number of Anthropic HTTP requests that may run pre-upstream work "
+ "(request parse / deep-copy / first compression stage / memory context / upstream connect) "
+ "concurrently. Prevents cold-start replay storms from starving /livez and new Codex WS opens. "
+ "Default: max(2, min(8, os.cpu_count() or 4)). "
+ "Set to 0 or negative to disable (unbounded). "
+ "Env: HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY."
+ ),
+)
@click.option("--log-file", default=None, help="Path to JSONL log file")
@click.option(
"--budget",
@@ -197,6 +211,7 @@ def proxy(
no_rate_limit: bool,
retry_max_attempts: int | None,
connect_timeout_seconds: int | None,
+ anthropic_pre_upstream_concurrency: int | None,
log_file: str | None,
budget: float | None,
code_graph: bool,
@@ -323,6 +338,12 @@ def proxy(
license_key=license_key,
# Stateless mode: disable all filesystem writes
stateless=is_stateless,
+ # Unit 4: bounded pre-upstream concurrency on the Anthropic HTTP
+ # path. ``None`` -> HeadroomProxy computes ``max(2, min(8,
+ # os.cpu_count() or 4))``; ``<= 0`` -> disabled (unbounded).
+ # Precedence: CLI > env > auto-compute (click's ``envvar``
+ # handles the env-var fallback).
+ anthropic_pre_upstream_concurrency=anthropic_pre_upstream_concurrency,
)
memory_status = "DISABLED"
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 4f0c5e9ec..f2f838a3d 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -325,8 +325,50 @@ class AnthropicHandlerMixin:
pre_upstream_started_at = time.perf_counter()
_stage_timings_emitted = False
+ # Unit 4: bounded pre-upstream concurrency. When the proxy is
+ # configured with a semaphore, acquire it before reading the
+ # request body so a replay storm cannot starve ``/livez``, new
+ # Codex WS opens, or the HTTP thread pool. The release happens
+ # BEFORE we start streaming response bytes back to the client —
+ # the semaphore must not be held for the whole response
+ # lifetime. ``_release_pre_upstream_sem`` is idempotent and
+ # called on every exit path (early 4xx return, upstream error,
+ # exception, and just before each streaming handoff).
+ pre_upstream_sem = getattr(self, "anthropic_pre_upstream_sem", None)
+ _pre_upstream_sem_acquired = False
+
+ def _release_pre_upstream_sem() -> None:
+ nonlocal _pre_upstream_sem_acquired
+ if _pre_upstream_sem_acquired and pre_upstream_sem is not None:
+ _pre_upstream_sem_acquired = False
+ pre_upstream_sem.release()
+
+ if pre_upstream_sem is not None:
+ _wait_started_at = time.perf_counter()
+ await pre_upstream_sem.acquire()
+ _pre_upstream_sem_acquired = True
+ _wait_ms = (time.perf_counter() - _wait_started_at) * 1000.0
+ stage_timer.record("pre_upstream_wait", _wait_ms)
+ if _wait_ms > 100.0:
+ logger.info(
+ "[%s] pre_upstream_wait_ms=%.2f session_id=%s "
+ "(anthropic pre-upstream semaphore contention)",
+ request_id,
+ _wait_ms,
+ trace_session_id,
+ )
+ else:
+ stage_timer.record("pre_upstream_wait", 0.0)
+
async def _emit_pre_upstream_stage_timings() -> None:
nonlocal _stage_timings_emitted
+ # Always release the Unit 4 pre-upstream semaphore when we
+ # hand off to streaming / return a response / hit an error.
+ # Doing it here (rather than only at explicit handoff sites)
+ # guarantees release on every exit path already covered by
+ # stage-timings emission, including early 4xx returns and
+ # upstream errors.
+ _release_pre_upstream_sem()
if _stage_timings_emitted:
return
_stage_timings_emitted = True
@@ -341,6 +383,7 @@ class AnthropicHandlerMixin:
session_id=trace_session_id,
stage_timer=stage_timer,
expected_stages=(
+ "pre_upstream_wait",
"read_request_json",
"deep_copy",
"compression_first_stage",
@@ -390,6 +433,7 @@ class AnthropicHandlerMixin:
# Validate message array size
if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
+ await _emit_pre_upstream_stage_timings()
return JSONResponse(
status_code=400,
content={
@@ -450,6 +494,10 @@ class AnthropicHandlerMixin:
allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
if not allowed:
await self.metrics.record_rate_limited(provider="anthropic")
+ # Unit 4: release the pre-upstream semaphore before we
+ # bail out of the handler via HTTPException — FastAPI's
+ # exception handler will NOT run our ``finally``.
+ await _emit_pre_upstream_stage_timings()
raise HTTPException(
status_code=429,
detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
@@ -460,6 +508,9 @@ class AnthropicHandlerMixin:
if self.cost_tracker:
allowed, remaining = self.cost_tracker.check_budget()
if not allowed:
+ # Unit 4: release the pre-upstream semaphore before we
+ # bail out of the handler via HTTPException.
+ await _emit_pre_upstream_stage_timings()
raise HTTPException(
status_code=429,
detail=f"Budget exceeded for {self.config.budget_period} period",
@@ -496,6 +547,9 @@ class AnthropicHandlerMixin:
response_headers.pop("content-encoding", None)
response_headers.pop("content-length", None)
+ # Unit 4: release the pre-upstream semaphore on cache
+ # hit — no upstream call will happen.
+ await _emit_pre_upstream_stage_timings()
return Response(
content=cached.response_body,
headers=response_headers,
@@ -523,6 +577,9 @@ class AnthropicHandlerMixin:
if hasattr(e, "reason"):
from fastapi.responses import JSONResponse as _JSONResp
+ # Unit 4: release the pre-upstream semaphore on
+ # security block — no upstream call will happen.
+ await _emit_pre_upstream_stage_timings()
return _JSONResp(
status_code=403,
content={
@@ -1083,6 +1140,8 @@ class AnthropicHandlerMixin:
)
except Exception as e:
logger.error(f"[{request_id}] Bedrock backend error: {e}")
+ # Unit 4: release the pre-upstream semaphore on error.
+ await _emit_pre_upstream_stage_timings()
return JSONResponse(
status_code=500,
content={
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 51e609c5d..f9ba15ba8 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -227,3 +227,22 @@ class ProxyConfig:
# Stateless mode — disable all filesystem writes for read-only / container deployments
stateless: bool = False
+
+ # Unit 4: Bounded pre-upstream concurrency for Anthropic replay storms.
+ #
+ # Caps the number of simultaneous requests allowed to run the
+ # pre-upstream phase of ``handle_anthropic_messages`` (request JSON
+ # read → deep-copy → first compression stage → memory-context lookup
+ # → first upstream connect). Prevents cold-start replay storms from
+ # monopolising the event loop / thread pool and starving ``/livez``,
+ # ``/readyz``, and new Codex WS opens. Compression stays on.
+ #
+ # ``None`` (default) -> auto-compute ``max(2, min(8, os.cpu_count() or 4))``.
+ # ``0`` or negative -> disables the semaphore (unbounded); useful for
+ # the Unit 6 counter-factual and for deliberately reproducing the
+ # original starvation. Any positive integer is honored verbatim.
+ #
+ # CLI: ``--anthropic-pre-upstream-concurrency``.
+ # Env: ``HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY``.
+ # Precedence: CLI > env > auto-compute.
+ anthropic_pre_upstream_concurrency: int | None = None
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index f6a57de6a..058650ba7 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -379,6 +379,30 @@ class HeadroomProxy(
# outermost ``finally``. Consumed by ``/debug/ws-sessions``.
self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()
+ # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP
+ # path. Caps how many ``handle_anthropic_messages`` calls may be
+ # running deep-copy / first-stage compression / memory-context
+ # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,
+ # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are
+ # intentionally NOT gated by this semaphore.
+ #
+ # A value of ``0`` or negative disables the semaphore (unbounded
+ # mode); this is useful for the Unit 6 counter-factual where we
+ # deliberately reproduce the original starvation. The default is
+ # ``max(2, min(8, os.cpu_count() or 4))``.
+ _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency
+ if _pre_upstream_cfg is None:
+ _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))
+ else:
+ _pre_upstream_resolved = _pre_upstream_cfg
+ self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved
+ if _pre_upstream_resolved > 0:
+ self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = (
+ asyncio.Semaphore(_pre_upstream_resolved)
+ )
+ else:
+ self.anthropic_pre_upstream_sem = None
+
# Backend for Anthropic API (direct, LiteLLM, or any-llm)
# Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
self.anthropic_backend: Backend | None = None
diff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.py
new file mode 100644
index 000000000..80c35cfda
--- /dev/null
+++ b/tests/test_anthropic_pre_upstream_backpressure.py
@@ -0,0 +1,626 @@
+"""Unit 4: bounded pre-upstream concurrency for Anthropic replay storms.
+
+Verifies that ``HeadroomProxy`` gates the pre-upstream phase of
+``handle_anthropic_messages`` with a semaphore, so cold-start replay
+storms cannot starve ``/livez`` or new Codex WS opens.
+
+Covers:
+- happy path (single request, no contention)
+- N+1 contention (only the (N+1)th waiter records ``pre_upstream_wait`` > 0)
+- strict serialization under concurrency=1
+- unbounded mode (``anthropic_pre_upstream_concurrency=0`` -> no semaphore)
+- exception-safety (semaphore released when the critical section raises)
+- ``/livez`` unaffected under Anthropic backpressure
+- compression is not bypassed (the Unit 4 gate is additive, not a shortcut)
+- CLI flag ``--anthropic-pre-upstream-concurrency`` wires into ``ProxyConfig``
+- env var ``HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY`` with flag override
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import os
+import statistics
+import time
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import anyio
+import pytest
+from click.testing import CliRunner
+from fastapi import Request
+from fastapi.testclient import TestClient
+
+from headroom.cli.proxy import proxy as proxy_cli
+from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
+from headroom.proxy.models import ProxyConfig
+from headroom.proxy.server import HeadroomProxy, create_app
+
+# --------------------------------------------------------------------------- #
+# Dummy handler that gives tests control over the ``_retry_request`` duration #
+# so we can simulate long pre-upstream work (semaphore contention). #
+# --------------------------------------------------------------------------- #
+
+
+class _DummyTokenizer:
+ def count(self, messages) -> int: # noqa: D401 - stub
+ return 1
+
+ def count_messages(self, messages) -> int: # noqa: D401 - stub
+ return 1
+
+ def count_tokens(self, text) -> int: # noqa: D401 - stub
+ return 1
+
+
+class _DummyMetrics:
+ def __init__(self) -> None:
+ self.stage_timings: list[tuple[str, dict]] = []
+
+ async def record_request(self, **kwargs):
+ return None
+
+ async def record_stage_timings(self, path: str, timings: dict) -> None:
+ self.stage_timings.append((path, timings))
+
+ async def record_rate_limited(self, **kwargs) -> None:
+ return None
+
+ async def record_failed(self, **kwargs) -> None:
+ return None
+
+
+class _ResponseStub:
+ def __init__(self) -> None:
+ self.status_code = 200
+ self.headers = {"content-type": "application/json"}
+ self._text = json.dumps(
+ {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "ok"}],
+ "model": "claude-3-5-sonnet-latest",
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ }
+ )
+
+ @property
+ def text(self) -> str:
+ return self._text
+
+ @property
+ def content(self) -> bytes:
+ return self._text.encode("utf-8")
+
+ def json(self) -> dict:
+ return json.loads(self._text)
+
+
+class _DummyAnthropicHandler(AnthropicHandlerMixin):
+ """Minimal handler used across tests; allows controlling upstream delay."""
+
+ ANTHROPIC_API_URL = "https://api.anthropic.com"
+
+ def _extract_anthropic_cache_ttl_metrics(self, usage): # noqa: D401
+ return (0, 0)
+
+ def __init__(
+ self,
+ *,
+ anthropic_pre_upstream_sem: asyncio.Semaphore | None = None,
+ upstream_delay_s: float = 0.0,
+ raise_during_critical: bool = False,
+ ) -> None:
+ self.rate_limiter = None
+ self.metrics = _DummyMetrics()
+ self.config = ProxyConfig(
+ optimize=False,
+ image_optimize=False,
+ retry_max_attempts=1,
+ retry_base_delay_ms=1,
+ retry_max_delay_ms=1,
+ connect_timeout_seconds=10,
+ mode="token",
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ fallback_enabled=False,
+ fallback_provider=None,
+ prefix_freeze_enabled=False,
+ memory_enabled=False,
+ )
+ self.usage_reporter = None
+ self.anthropic_provider = SimpleNamespace(get_context_limit=lambda model: 200_000)
+ self.anthropic_pipeline = SimpleNamespace(apply=MagicMock())
+ self.anthropic_backend = None
+ self.cost_tracker = None
+ self.memory_handler = None
+ self.cache = None
+ self.security = None
+ self.ccr_context_tracker = None
+ self.ccr_injector = None
+ self.ccr_response_handler = None
+ self.ccr_feedback = None
+ self.ccr_batch_processor = None
+ self.ccr_mcp_server = None
+ self.traffic_learner = None
+ self.tool_injector = None
+ self.read_lifecycle_manager = None
+ self.logger = SimpleNamespace(log=lambda *a, **k: None)
+ self.request_logger = self.logger
+ self.usage_observer = None
+ self.image_compressor = None
+ self.session_tracker_store = SimpleNamespace(
+ compute_session_id=lambda *a, **k: "sess-1",
+ get_or_create=lambda *a, **k: SimpleNamespace(
+ get_frozen_message_count=lambda: 0,
+ get_last_original_messages=lambda: [],
+ get_last_forwarded_messages=lambda: [],
+ record_request=lambda *a, **k: None,
+ ),
+ )
+ # Unit 4: the only field this test cares about.
+ self.anthropic_pre_upstream_sem = anthropic_pre_upstream_sem
+ self.anthropic_pre_upstream_concurrency = (
+ 0 if anthropic_pre_upstream_sem is None else anthropic_pre_upstream_sem._value
+ )
+ self._upstream_delay_s = upstream_delay_s
+ self._raise_during_critical = raise_during_critical
+ self.upstream_enter_times: list[float] = []
+ self.upstream_exit_times: list[float] = []
+
+ async def _next_request_id(self) -> str:
+ # Unique IDs so log assertions remain disambiguated under parallelism.
+ return f"req-{id(object()):x}"
+
+ def _extract_tags(self, headers):
+ return {}
+
+ async def _retry_request(self, method: str, url: str, headers: dict, body: dict):
+ if self._raise_during_critical:
+ raise RuntimeError("synthetic pre-upstream failure")
+ enter = time.perf_counter()
+ self.upstream_enter_times.append(enter)
+ if self._upstream_delay_s > 0:
+ await asyncio.sleep(self._upstream_delay_s)
+ self.upstream_exit_times.append(time.perf_counter())
+ return _ResponseStub()
+
+ def _get_compression_cache(self, session_id):
+ return SimpleNamespace(
+ apply_cached=lambda m: m,
+ compute_frozen_count=lambda m: 0,
+ mark_stable_from_messages=lambda *a, **k: None,
+ should_defer_compression=lambda h: False,
+ mark_stable=lambda h: None,
+ content_hash=lambda c: "h",
+ update_from_result=lambda *a, **k: None,
+ _cache={},
+ _stable_hashes=set(),
+ )
+
+
+def _build_request(body: dict, headers: dict[str, str]) -> Request:
+ payload = json.dumps(body).encode("utf-8")
+
+ async def receive():
+ return {"type": "http.request", "body": payload, "more_body": False}
+
+ scope = {
+ "type": "http",
+ "asgi": {"version": "3.0"},
+ "http_version": "1.1",
+ "method": "POST",
+ "scheme": "https",
+ "path": "/v1/messages",
+ "raw_path": b"/v1/messages",
+ "query_string": b"",
+ "headers": [
+ (key.lower().encode("utf-8"), value.encode("utf-8")) for key, value in headers.items()
+ ],
+ "client": ("127.0.0.1", 12345),
+ "server": ("testserver", 443),
+ }
+ return Request(scope, receive)
+
+
+class _CapturingHandler(logging.Handler):
+ def __init__(self) -> None:
+ super().__init__(level=logging.INFO)
+ self.records: list[logging.LogRecord] = []
+
+ def emit(self, record: logging.LogRecord) -> None:
+ self.records.append(record)
+
+
+def stage_log_capture():
+ target = logging.getLogger("headroom.proxy")
+ handler = _CapturingHandler()
+ previous_level = target.level
+ target.addHandler(handler)
+ target.setLevel(logging.INFO)
+ try:
+ yield handler
+ finally:
+ target.removeHandler(handler)
+ target.setLevel(previous_level)
+
+
+def _parse_all_stage_logs(handler: _CapturingHandler) -> list[dict]:
+ payloads: list[dict] = []
+ for record in handler.records:
+ msg = record.getMessage()
+ if "STAGE_TIMINGS" in msg:
+ payload_start = msg.index("STAGE_TIMINGS ") + len("STAGE_TIMINGS ")
+ payloads.append(json.loads(msg[payload_start:]))
+ return payloads
+
+
+def _tokenizer_patch():
+ import headroom.tokenizers as _tk
+
+ orig_get = _tk.get_tokenizer
+
+ class _Ctx:
+ def __enter__(self):
+ _tk.get_tokenizer = lambda model: _DummyTokenizer()
+ return self
+
+ def __exit__(self, *exc):
+ _tk.get_tokenizer = orig_get
+
+ return _Ctx()
+
+
+# --------------------------------------------------------------------------- #
+# Happy path #
+# --------------------------------------------------------------------------- #
+
+
+def test_happy_path_single_request_negligible_wait(stage_log_capture):
+ sem = asyncio.Semaphore(2)
+ handler = _DummyAnthropicHandler(anthropic_pre_upstream_sem=sem)
+ request = _build_request(
+ {
+ "model": "claude-3-5-sonnet-latest",
+ "messages": [{"role": "user", "content": "hello"}],
+ },
+ {"authorization": "Bearer sk-ant-api-test"},
+ )
+
+ with _tokenizer_patch():
+ anyio.run(handler.handle_anthropic_messages, request)
+
+ payloads = _parse_all_stage_logs(stage_log_capture)
+ assert len(payloads) == 1
+ stages = payloads[0]["stages"]
+ assert "pre_upstream_wait" in stages
+ # Single request -> no contention, wait ms must be tiny.
+ assert stages["pre_upstream_wait"] is not None
+ assert stages["pre_upstream_wait"] < 25.0, stages
+ # Sanity: semaphore was released cleanly.
+ assert sem._value == 2
+
+
+# --------------------------------------------------------------------------- #
+# N+1 contention: with concurrency=2 and 3 concurrent requests, #
+# exactly one of them must observe a non-trivial ``pre_upstream_wait``. #
+# --------------------------------------------------------------------------- #
+
+
+def test_n_plus_one_contention_only_waiter_has_nonzero_wait(stage_log_capture):
+ async def _run() -> None:
+ sem = asyncio.Semaphore(2)
+ # Each request hogs the semaphore for ~150 ms. With concurrency=2,
+ # 3 concurrent requests mean exactly one waits ~150 ms.
+ handler = _DummyAnthropicHandler(
+ anthropic_pre_upstream_sem=sem, upstream_delay_s=0.15
+ )
+ reqs = [
+ _build_request(
+ {
+ "model": "claude-3-5-sonnet-latest",
+ "messages": [{"role": "user", "content": f"hello {i}"}],
+ },
+ {"authorization": "Bearer sk-ant-api-test"},
+ )
+ for i in range(3)
+ ]
+ await asyncio.gather(*(handler.handle_anthropic_messages(r) for r in reqs))
+ assert sem._value == 2 # semaphore fully released
+
+ with _tokenizer_patch():
+ anyio.run(_run)
+
+ payloads = _parse_all_stage_logs(stage_log_capture)
+ assert len(payloads) == 3
+ waits = sorted(p["stages"]["pre_upstream_wait"] for p in payloads)
+ # Exactly one request must have waited noticeably; the first two should
+ # be near zero (they acquired the sem immediately).
+ assert waits[0] < 25.0, waits
+ assert waits[1] < 25.0, waits
+ # The waiter should have waited roughly the upstream-delay budget.
+ assert waits[2] > 75.0, waits
+
+
+# --------------------------------------------------------------------------- #
+# Serialization: concurrency=1 => strict ordering of upstream enter timestamps #
+# --------------------------------------------------------------------------- #
+
+
+def test_concurrency_one_serializes_requests():
+ async def _run() -> float:
+ sem = asyncio.Semaphore(1)
+ handler = _DummyAnthropicHandler(
+ anthropic_pre_upstream_sem=sem, upstream_delay_s=0.10
+ )
+ reqs = [
+ _build_request(
+ {
+ "model": "claude-3-5-sonnet-latest",
+ "messages": [{"role": "user", "content": f"msg {i}"}],
+ },
+ {"authorization": "Bearer sk-ant-api-test"},
+ )
+ for i in range(2)
+ ]
+ start = time.perf_counter()
+ await asyncio.gather(*(handler.handle_anthropic_messages(r) for r in reqs))
+ elapsed = time.perf_counter() - start
+ # Strict ordering: second request enters upstream only AFTER the first exits.
+ assert len(handler.upstream_enter_times) == 2
+ assert handler.upstream_enter_times[1] >= handler.upstream_exit_times[0] - 1e-6, (
+ handler.upstream_enter_times,
+ handler.upstream_exit_times,
+ )
+ return elapsed
+
+ with _tokenizer_patch():
+ elapsed = anyio.run(_run)
+ # Two back-to-back 100 ms upstream calls under serialization: must take
+ # at least ~2 * 100 ms. (Give a little slack for scheduler jitter.)
+ assert elapsed >= 0.18, elapsed
+
+
+# --------------------------------------------------------------------------- #
+# Unbounded mode: ``anthropic_pre_upstream_concurrency=0`` disables the sem. #
+# --------------------------------------------------------------------------- #
+
+
+def test_unbounded_mode_no_semaphore_instance():
+ config = ProxyConfig(anthropic_pre_upstream_concurrency=0)
+ proxy = HeadroomProxy(config)
+ assert proxy.anthropic_pre_upstream_sem is None
+ assert proxy.anthropic_pre_upstream_concurrency == 0
+
+
+def test_unbounded_mode_requests_run_concurrently():
+ """With concurrency=0 (sem disabled), two slow requests overlap."""
+
+ async def _run() -> float:
+ handler = _DummyAnthropicHandler(
+ anthropic_pre_upstream_sem=None, upstream_delay_s=0.10
+ )
+ reqs = [
+ _build_request(
+ {
+ "model": "claude-3-5-sonnet-latest",
+ "messages": [{"role": "user", "content": f"msg {i}"}],
+ },
+ {"authorization": "Bearer sk-ant-api-test"},
+ )
+ for i in range(2)
+ ]
+ start = time.perf_counter()
+ await asyncio.gather(*(handler.handle_anthropic_messages(r) for r in reqs))
+ return time.perf_counter() - start
+
+ with _tokenizer_patch():
+ elapsed = anyio.run(_run)
+ # Unbounded -> both sleeps run in parallel. Total should be ~0.10 s,
+ # nowhere near 0.20 s.
+ assert elapsed < 0.18, elapsed
+
+
+# --------------------------------------------------------------------------- #
+# Exception releases the semaphore. #
+# --------------------------------------------------------------------------- #
+
+
+def test_exception_inside_critical_section_releases_semaphore():
+ async def _run() -> None:
+ sem = asyncio.Semaphore(2)
+ baseline = sem._value
+ handler = _DummyAnthropicHandler(
+ anthropic_pre_upstream_sem=sem, raise_during_critical=True
+ )
+ # Drive several cycles to ensure we don't leak on any path.
+ for i in range(5):
+ req = _build_request(
+ {
+ "model": "claude-3-5-sonnet-latest",
+ "messages": [{"role": "user", "content": f"msg {i}"}],
+ },
+ {"authorization": "Bearer sk-ant-api-test"},
+ )
+ # The handler catches upstream RuntimeError internally and
+ # returns a 5xx JSONResponse; this is the expected behaviour.
+ await handler.handle_anthropic_messages(req)
+ # After each cycle the semaphore must be fully restored.
+ assert sem._value == baseline, (i, sem._value, baseline)
+
+ with _tokenizer_patch():
+ anyio.run(_run)
+
+
+# --------------------------------------------------------------------------- #
+# /livez stays fast under Anthropic pre-upstream contention. #
+# --------------------------------------------------------------------------- #
+
+
+def test_livez_unaffected_under_anthropic_backpressure():
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ anthropic_pre_upstream_concurrency=2,
+ )
+ app = create_app(config)
+ assert app.state.proxy.anthropic_pre_upstream_sem is not None
+
+ # Drain the semaphore so any simulated Anthropic request would block.
+ proxy = app.state.proxy
+
+ async def _drain_sem() -> None:
+ # Acquire both permits — no request can enter the pre-upstream region.
+ await proxy.anthropic_pre_upstream_sem.acquire()
+ await proxy.anthropic_pre_upstream_sem.acquire()
+
+ # Run an event loop just to drain the semaphore.
+ loop = asyncio.new_event_loop()
+ try:
+ loop.run_until_complete(_drain_sem())
+ finally:
+ loop.close()
+
+ latencies: list[float] = []
+ with TestClient(app) as client:
+ for _ in range(20):
+ t0 = time.perf_counter()
+ resp = client.get("/livez")
+ latencies.append((time.perf_counter() - t0) * 1000.0)
+ assert resp.status_code == 200
+ assert resp.json()["alive"] is True
+
+ p99 = statistics.quantiles(latencies, n=100)[98] if len(latencies) >= 100 else max(latencies)
+ assert p99 < 100.0, (p99, latencies)
+
+
+# --------------------------------------------------------------------------- #
+# Compression is NOT bypassed by the gate. #
+# --------------------------------------------------------------------------- #
+
+
+def test_compression_is_not_bypassed_when_gated(stage_log_capture):
+ """With ``optimize=True`` the first compression stage must still run."""
+
+ class _Pipeline:
+ def __init__(self) -> None:
+ self.called = False
+
+ def apply(self, messages, *args, **kwargs):
+ self.called = True
+ return SimpleNamespace(messages=messages, metadata={"applied_steps": ["first"]})
+
+ sem = asyncio.Semaphore(2)
+ handler = _DummyAnthropicHandler(anthropic_pre_upstream_sem=sem)
+ handler.config = ProxyConfig(
+ optimize=True,
+ image_optimize=False,
+ retry_max_attempts=1,
+ retry_base_delay_ms=1,
+ retry_max_delay_ms=1,
+ connect_timeout_seconds=10,
+ mode="token",
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ fallback_enabled=False,
+ fallback_provider=None,
+ prefix_freeze_enabled=False,
+ memory_enabled=False,
+ anthropic_pre_upstream_concurrency=2,
+ )
+ pipeline = _Pipeline()
+ handler.anthropic_pipeline = pipeline
+
+ # Large synthetic body to ensure the pipeline triggers.
+ big_text = "x" * 50_000
+ request = _build_request(
+ {
+ "model": "claude-3-5-sonnet-latest",
+ "messages": [{"role": "user", "content": big_text}],
+ },
+ {"authorization": "Bearer sk-ant-api-test"},
+ )
+
+ with _tokenizer_patch():
+ anyio.run(handler.handle_anthropic_messages, request)
+
+ assert pipeline.called, "compression pipeline must still run under backpressure"
+ # Semaphore restored after the request.
+ assert sem._value == 2
+
+
+# --------------------------------------------------------------------------- #
+# CLI: --anthropic-pre-upstream-concurrency plumbs into ProxyConfig. #
+# --------------------------------------------------------------------------- #
+
+
+def _run_cli_capture(args: list[str], env: dict | None = None) -> ProxyConfig:
+ """Invoke the proxy CLI, intercepting ``run_server`` to capture config.
+
+ We do NOT want the CLI to actually start a server — monkeypatching the
+ ``run_server`` entry point (imported lazily inside the click command
+ via ``from headroom.proxy.server import ... run_server``) short-
+ circuits it and lets us inspect the ``ProxyConfig`` that was built.
+ """
+ import headroom.proxy.server as server_mod
+
+ captured: dict[str, ProxyConfig] = {}
+ orig_run = server_mod.run_server
+
+ def _fake_run(config: ProxyConfig): # noqa: D401 - stub
+ captured["config"] = config
+ return 0
+
+ server_mod.run_server = _fake_run
+ try:
+ runner = CliRunner()
+ result = runner.invoke(proxy_cli, args, env=env or {})
+ finally:
+ server_mod.run_server = orig_run
+
+ assert result.exit_code == 0, (result.output, result.exception)
+ assert "config" in captured, "run_server was not called"
+ return captured["config"]
+
+
+def test_cli_flag_sets_pre_upstream_concurrency():
+ config = _run_cli_capture(["--anthropic-pre-upstream-concurrency", "3"])
+ assert config.anthropic_pre_upstream_concurrency == 3
+
+
+def test_env_var_sets_pre_upstream_concurrency():
+ # Must set in the env passed to the runner (click reads envvar).
+ # Also strip the corresponding CLI flag.
+ env = {"HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY": "4"}
+ # Also make sure we don't pick up a host user env that could override.
+ config = _run_cli_capture([], env=env)
+ assert config.anthropic_pre_upstream_concurrency == 4
+
+
+def test_cli_flag_overrides_env_var():
+ env = {"HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY": "4"}
+ config = _run_cli_capture(
+ ["--anthropic-pre-upstream-concurrency", "7"], env=env
+ )
+ assert config.anthropic_pre_upstream_concurrency == 7
+
+
+# --------------------------------------------------------------------------- #
+# Sanity: HeadroomProxy auto-computes default when config value is None. #
+# --------------------------------------------------------------------------- #
+
+
+def test_auto_computed_default_on_this_machine():
+ config = ProxyConfig() # field left at None -> auto-compute.
+ proxy = HeadroomProxy(config)
+ expected = max(2, min(8, os.cpu_count() or 4))
+ assert proxy.anthropic_pre_upstream_concurrency == expected
+ assert proxy.anthropic_pre_upstream_sem is not None
+ assert proxy.anthropic_pre_upstream_sem._value == expected