Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(proxy): bound pre-upstream Anthropic work with a semaphore
Adryan Eka Vandra1de753bd204f
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex a3863595e..0ea1bb03b 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -59,6 +59,20 @@ from .main import maindefault=None,help="Upstream connection timeout in seconds (default: 10)",)+ "--anthropic-pre-upstream-concurrency",+ type=int,+ default=None,+ envvar="HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY",+ help=(+ "Cap the number of Anthropic HTTP requests that may run pre-upstream work "+ "(request parse / deep-copy / first compression stage / memory context / upstream connect) "+ "concurrently. Prevents cold-start replay storms from starving /livez and new Codex WS opens. "+ "Default: max(2, min(8, os.cpu_count() or 4)). "+ "Set to 0 or negative to disable (unbounded). "+ "Env: HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY."+ ),+)@click.option("--log-file", default=None, help="Path to JSONL log file")@click.option("--budget",@@ -197,6 +211,7 @@ def proxy(no_rate_limit: bool,retry_max_attempts: int | None,connect_timeout_seconds: int | None,+ anthropic_pre_upstream_concurrency: int | None,log_file: str | None,budget: float | None,code_graph: bool,@@ -323,6 +338,12 @@ def proxy(license_key=license_key,# Stateless mode: disable all filesystem writesstateless=is_stateless,+ # Unit 4: bounded pre-upstream concurrency on the Anthropic HTTP+ # path. ``None`` -> HeadroomProxy computes ``max(2, min(8,+ # os.cpu_count() or 4))``; ``<= 0`` -> disabled (unbounded).+ # Precedence: CLI > env > auto-compute (click's ``envvar``+ # handles the env-var fallback).+ anthropic_pre_upstream_concurrency=anthropic_pre_upstream_concurrency,)memory_status = "DISABLED"diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 4f0c5e9ec..f2f838a3d 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -325,8 +325,50 @@ class AnthropicHandlerMixin:pre_upstream_started_at = time.perf_counter()_stage_timings_emitted = False+ # Unit 4: bounded pre-upstream concurrency. When the proxy is+ # configured with a semaphore, acquire it before reading the+ # request body so a replay storm cannot starve ``/livez``, new+ # Codex WS opens, or the HTTP thread pool. The release happens+ # BEFORE we start streaming response bytes back to the client —+ # the semaphore must not be held for the whole response+ # lifetime. ``_release_pre_upstream_sem`` is idempotent and+ # called on every exit path (early 4xx return, upstream error,+ # exception, and just before each streaming handoff).+ pre_upstream_sem = getattr(self, "anthropic_pre_upstream_sem", None)+ _pre_upstream_sem_acquired = False++ def _release_pre_upstream_sem() -> None:+ nonlocal _pre_upstream_sem_acquired+ if _pre_upstream_sem_acquired and pre_upstream_sem is not None:+ _pre_upstream_sem_acquired = False+ pre_upstream_sem.release()++ if pre_upstream_sem is not None:+ _wait_started_at = time.perf_counter()+ await pre_upstream_sem.acquire()+ _pre_upstream_sem_acquired = True+ _wait_ms = (time.perf_counter() - _wait_started_at) * 1000.0+ stage_timer.record("pre_upstream_wait", _wait_ms)+ if _wait_ms > 100.0:+ logger.info(+ "[%s] pre_upstream_wait_ms=%.2f session_id=%s "+ "(anthropic pre-upstream semaphore contention)",+ request_id,+ _wait_ms,+ trace_session_id,+ )+ else:+ stage_timer.record("pre_upstream_wait", 0.0)+async def _emit_pre_upstream_stage_timings() -> None:nonlocal _stage_timings_emitted+ # Always release the Unit 4 pre-upstream semaphore when we+ # hand off to streaming / return a response / hit an error.+ # Doing it here (rather than only at explicit handoff sites)+ # guarantees release on every exit path already covered by+ # stage-timings emission, including early 4xx returns and+ # upstream errors.+ _release_pre_upstream_sem()if _stage_timings_emitted:return_stage_timings_emitted = True@@ -341,6 +383,7 @@ class AnthropicHandlerMixin:session_id=trace_session_id,stage_timer=stage_timer,expected_stages=(+ "pre_upstream_wait","read_request_json","deep_copy","compression_first_stage",@@ -390,6 +433,7 @@ class AnthropicHandlerMixin:# Validate message array sizeif len(messages) > MAX_MESSAGE_ARRAY_LENGTH:+ await _emit_pre_upstream_stage_timings()return JSONResponse(status_code=400,content={@@ -450,6 +494,10 @@ class AnthropicHandlerMixin:allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)if not allowed:await self.metrics.record_rate_limited(provider="anthropic")+ # Unit 4: release the pre-upstream semaphore before we+ # bail out of the handler via HTTPException — FastAPI's+ # exception handler will NOT run our ``finally``.+ await _emit_pre_upstream_stage_timings()raise HTTPException(status_code=429,detail=f"Rate limited. Retry after {wait_seconds:.1f}s",@@ -460,6 +508,9 @@ class AnthropicHandlerMixin:if self.cost_tracker:allowed, remaining = self.cost_tracker.check_budget()if not allowed:+ # Unit 4: release the pre-upstream semaphore before we+ # bail out of the handler via HTTPException.+ await _emit_pre_upstream_stage_timings()raise HTTPException(status_code=429,detail=f"Budget exceeded for {self.config.budget_period} period",@@ -496,6 +547,9 @@ class AnthropicHandlerMixin:response_headers.pop("content-encoding", None)response_headers.pop("content-length", None)+ # Unit 4: release the pre-upstream semaphore on cache+ # hit — no upstream call will happen.+ await _emit_pre_upstream_stage_timings()return Response(content=cached.response_body,headers=response_headers,@@ -523,6 +577,9 @@ class AnthropicHandlerMixin:if hasattr(e, "reason"):from fastapi.responses import JSONResponse as _JSONResp+ # Unit 4: release the pre-upstream semaphore on+ # security block — no upstream call will happen.+ await _emit_pre_upstream_stage_timings()return _JSONResp(status_code=403,content={@@ -1083,6 +1140,8 @@ class AnthropicHandlerMixin:)except Exception as e:logger.error(f"[{request_id}] Bedrock backend error: {e}")+ # Unit 4: release the pre-upstream semaphore on error.+ await _emit_pre_upstream_stage_timings()return JSONResponse(status_code=500,content={diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 51e609c5d..f9ba15ba8 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -227,3 +227,22 @@ class ProxyConfig:# Stateless mode — disable all filesystem writes for read-only / container deploymentsstateless: bool = False++ # Unit 4: Bounded pre-upstream concurrency for Anthropic replay storms.+ #+ # Caps the number of simultaneous requests allowed to run the+ # pre-upstream phase of ``handle_anthropic_messages`` (request JSON+ # read → deep-copy → first compression stage → memory-context lookup+ # → first upstream connect). Prevents cold-start replay storms from+ # monopolising the event loop / thread pool and starving ``/livez``,+ # ``/readyz``, and new Codex WS opens. Compression stays on.+ #+ # ``None`` (default) -> auto-compute ``max(2, min(8, os.cpu_count() or 4))``.+ # ``0`` or negative -> disables the semaphore (unbounded); useful for+ # the Unit 6 counter-factual and for deliberately reproducing the+ # original starvation. Any positive integer is honored verbatim.+ #+ # CLI: ``--anthropic-pre-upstream-concurrency``.+ # Env: ``HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY``.+ # Precedence: CLI > env > auto-compute.+ anthropic_pre_upstream_concurrency: int | None = Nonediff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex f6a57de6a..058650ba7 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -379,6 +379,30 @@ class HeadroomProxy(# outermost ``finally``. Consumed by ``/debug/ws-sessions``.self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()+ # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP+ # path. Caps how many ``handle_anthropic_messages`` calls may be+ # running deep-copy / first-stage compression / memory-context+ # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,+ # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are+ # intentionally NOT gated by this semaphore.+ #+ # A value of ``0`` or negative disables the semaphore (unbounded+ # mode); this is useful for the Unit 6 counter-factual where we+ # deliberately reproduce the original starvation. The default is+ # ``max(2, min(8, os.cpu_count() or 4))``.+ _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency+ if _pre_upstream_cfg is None:+ _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))+ else:+ _pre_upstream_resolved = _pre_upstream_cfg+ self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved+ if _pre_upstream_resolved > 0:+ self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = (+ asyncio.Semaphore(_pre_upstream_resolved)+ )+ else:+ self.anthropic_pre_upstream_sem = None+# Backend for Anthropic API (direct, LiteLLM, or any-llm)# Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"self.anthropic_backend: Backend | None = Nonediff --git a/tests/test_anthropic_pre_upstream_backpressure.py b/tests/test_anthropic_pre_upstream_backpressure.pynew file mode 100644index 000000000..80c35cfda--- /dev/null+++ b/tests/test_anthropic_pre_upstream_backpressure.py@@ -0,0 +1,626 @@+"""Unit 4: bounded pre-upstream concurrency for Anthropic replay storms.++Verifies that ``HeadroomProxy`` gates the pre-upstream phase of+``handle_anthropic_messages`` with a semaphore, so cold-start replay+storms cannot starve ``/livez`` or new Codex WS opens.++Covers:+- happy path (single request, no contention)+- N+1 contention (only the (N+1)th waiter records ``pre_upstream_wait`` > 0)+- strict serialization under concurrency=1+- unbounded mode (``anthropic_pre_upstream_concurrency=0`` -> no semaphore)+- exception-safety (semaphore released when the critical section raises)+- ``/livez`` unaffected under Anthropic backpressure+- compression is not bypassed (the Unit 4 gate is additive, not a shortcut)+- CLI flag ``--anthropic-pre-upstream-concurrency`` wires into ``ProxyConfig``+- env var ``HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY`` with flag override+"""++from __future__ import annotations++import asyncio+import json+import logging+import os+import statistics+import time+from types import SimpleNamespace+from unittest.mock import MagicMock++import anyio+import pytest+from click.testing import CliRunner+from fastapi import Request+from fastapi.testclient import TestClient++from headroom.cli.proxy import proxy as proxy_cli+from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin+from headroom.proxy.models import ProxyConfig+from headroom.proxy.server import HeadroomProxy, create_app++# --------------------------------------------------------------------------- #+# Dummy handler that gives tests control over the ``_retry_request`` duration #+# so we can simulate long pre-upstream work (semaphore contention). #+# --------------------------------------------------------------------------- #+++class _DummyTokenizer:+ def count(self, messages) -> int: # noqa: D401 - stub+ return 1++ def count_messages(self, messages) -> int: # noqa: D401 - stub+ return 1++ def count_tokens(self, text) -> int: # noqa: D401 - stub+ return 1+++class _DummyMetrics:+ def __init__(self) -> None:+ self.stage_timings: list[tuple[str, dict]] = []++ async def record_request(self, **kwargs):+ return None++ async def record_stage_timings(self, path: str, timings: dict) -> None:+ self.stage_timings.append((path, timings))++ async def record_rate_limited(self, **kwargs) -> None:+ return None++ async def record_failed(self, **kwargs) -> None:+ return None+++class _ResponseStub:+ def __init__(self) -> None:+ self.status_code = 200+ self.headers = {"content-type": "application/json"}+ self._text = json.dumps(+ {+ "id": "msg_test",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": "ok"}],+ "model": "claude-3-5-sonnet-latest",+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 1, "output_tokens": 1},+ }+ )++ @property+ def text(self) -> str:+ return self._text++ @property+ def content(self) -> bytes:+ return self._text.encode("utf-8")++ def json(self) -> dict:+ return json.loads(self._text)+++class _DummyAnthropicHandler(AnthropicHandlerMixin):+ """Minimal handler used across tests; allows controlling upstream delay."""++ ANTHROPIC_API_URL = "https://api.anthropic.com"++ def _extract_anthropic_cache_ttl_metrics(self, usage): # noqa: D401+ return (0, 0)++ def __init__(+ self,+ *,+ anthropic_pre_upstream_sem: asyncio.Semaphore | None = None,+ upstream_delay_s: float = 0.0,+ raise_during_critical: bool = False,+ ) -> None:+ self.rate_limiter = None+ self.metrics = _DummyMetrics()+ self.config = ProxyConfig(+ optimize=False,+ image_optimize=False,+ retry_max_attempts=1,+ retry_base_delay_ms=1,+ retry_max_delay_ms=1,+ connect_timeout_seconds=10,+ mode="token",+ cache_enabled=False,+ rate_limit_enabled=False,+ fallback_enabled=False,+ fallback_provider=None,+ prefix_freeze_enabled=False,+ memory_enabled=False,+ )+ self.usage_reporter = None+ self.anthropic_provider = SimpleNamespace(get_context_limit=lambda model: 200_000)+ self.anthropic_pipeline = SimpleNamespace(apply=MagicMock())+ self.anthropic_backend = None+ self.cost_tracker = None+ self.memory_handler = None+ self.cache = None+ self.security = None+ self.ccr_context_tracker = None+ self.ccr_injector = None+ self.ccr_response_handler = None+ self.ccr_feedback = None+ self.ccr_batch_processor = None+ self.ccr_mcp_server = None+ self.traffic_learner = None+ self.tool_injector = None+ self.read_lifecycle_manager = None+ self.logger = SimpleNamespace(log=lambda *a, **k: None)+ self.request_logger = self.logger+ self.usage_observer = None+ self.image_compressor = None+ self.session_tracker_store = SimpleNamespace(+ compute_session_id=lambda *a, **k: "sess-1",+ get_or_create=lambda *a, **k: SimpleNamespace(+ get_frozen_message_count=lambda: 0,+ get_last_original_messages=lambda: [],+ get_last_forwarded_messages=lambda: [],+ record_request=lambda *a, **k: None,+ ),+ )+ # Unit 4: the only field this test cares about.+ self.anthropic_pre_upstream_sem = anthropic_pre_upstream_sem+ self.anthropic_pre_upstream_concurrency = (+ 0 if anthropic_pre_upstream_sem is None else anthropic_pre_upstream_sem._value+ )+ self._upstream_delay_s = upstream_delay_s+ self._raise_during_critical = raise_during_critical+ self.upstream_enter_times: list[float] = []+ self.upstream_exit_times: list[float] = []++ async def _next_request_id(self) -> str:+ # Unique IDs so log assertions remain disambiguated under parallelism.+ return f"req-{id(object()):x}"++ def _extract_tags(self, headers):+ return {}++ async def _retry_request(self, method: str, url: str, headers: dict, body: dict):+ if self._raise_during_critical:+ raise RuntimeError("synthetic pre-upstream failure")+ enter = time.perf_counter()+ self.upstream_enter_times.append(enter)+ if self._upstream_delay_s > 0:+ await asyncio.sleep(self._upstream_delay_s)+ self.upstream_exit_times.append(time.perf_counter())+ return _ResponseStub()++ def _get_compression_cache(self, session_id):+ return SimpleNamespace(+ apply_cached=lambda m: m,+ compute_frozen_count=lambda m: 0,+ mark_stable_from_messages=lambda *a, **k: None,+ should_defer_compression=lambda h: False,+ mark_stable=lambda h: None,+ content_hash=lambda c: "h",+ update_from_result=lambda *a, **k: None,+ _cache={},+ _stable_hashes=set(),+ )+++def _build_request(body: dict, headers: dict[str, str]) -> Request:+ payload = json.dumps(body).encode("utf-8")++ async def receive():+ return {"type": "http.request", "body": payload, "more_body": False}++ scope = {+ "type": "http",+ "asgi": {"version": "3.0"},+ "http_version": "1.1",+ "method": "POST",+ "scheme": "https",+ "path": "/v1/messages",+ "raw_path": b"/v1/messages",+ "query_string": b"",+ "headers": [+ (key.lower().encode("utf-8"), value.encode("utf-8")) for key, value in headers.items()+ ],+ "client": ("127.0.0.1", 12345),+ "server": ("testserver", 443),+ }+ return Request(scope, receive)+++class _CapturingHandler(logging.Handler):+ def __init__(self) -> None:+ super().__init__(level=logging.INFO)+ self.records: list[logging.LogRecord] = []++ def emit(self, record: logging.LogRecord) -> None:+ self.records.append(record)+++def stage_log_capture():+ target = logging.getLogger("headroom.proxy")+ handler = _CapturingHandler()+ previous_level = target.level+ target.addHandler(handler)+ target.setLevel(logging.INFO)+ try:+ yield handler+ finally:+ target.removeHandler(handler)+ target.setLevel(previous_level)+++def _parse_all_stage_logs(handler: _CapturingHandler) -> list[dict]:+ payloads: list[dict] = []+ for record in handler.records:+ msg = record.getMessage()+ if "STAGE_TIMINGS" in msg:+ payload_start = msg.index("STAGE_TIMINGS ") + len("STAGE_TIMINGS ")+ payloads.append(json.loads(msg[payload_start:]))+ return payloads+++def _tokenizer_patch():+ import headroom.tokenizers as _tk++ orig_get = _tk.get_tokenizer++ class _Ctx:+ def __enter__(self):+ _tk.get_tokenizer = lambda model: _DummyTokenizer()+ return self++ def __exit__(self, *exc):+ _tk.get_tokenizer = orig_get++ return _Ctx()+++# --------------------------------------------------------------------------- #+# Happy path #+# --------------------------------------------------------------------------- #+++def test_happy_path_single_request_negligible_wait(stage_log_capture):+ sem = asyncio.Semaphore(2)+ handler = _DummyAnthropicHandler(anthropic_pre_upstream_sem=sem)+ request = _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": "hello"}],+ },+ {"authorization": "Bearer sk-ant-api-test"},+ )++ with _tokenizer_patch():+ anyio.run(handler.handle_anthropic_messages, request)++ payloads = _parse_all_stage_logs(stage_log_capture)+ assert len(payloads) == 1+ stages = payloads[0]["stages"]+ assert "pre_upstream_wait" in stages+ # Single request -> no contention, wait ms must be tiny.+ assert stages["pre_upstream_wait"] is not None+ assert stages["pre_upstream_wait"] < 25.0, stages+ # Sanity: semaphore was released cleanly.+ assert sem._value == 2+++# --------------------------------------------------------------------------- #+# N+1 contention: with concurrency=2 and 3 concurrent requests, #+# exactly one of them must observe a non-trivial ``pre_upstream_wait``. #+# --------------------------------------------------------------------------- #+++def test_n_plus_one_contention_only_waiter_has_nonzero_wait(stage_log_capture):+ async def _run() -> None:+ sem = asyncio.Semaphore(2)+ # Each request hogs the semaphore for ~150 ms. With concurrency=2,+ # 3 concurrent requests mean exactly one waits ~150 ms.+ handler = _DummyAnthropicHandler(+ anthropic_pre_upstream_sem=sem, upstream_delay_s=0.15+ )+ reqs = [+ _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": f"hello {i}"}],+ },+ {"authorization": "Bearer sk-ant-api-test"},+ )+ for i in range(3)+ ]+ await asyncio.gather(*(handler.handle_anthropic_messages(r) for r in reqs))+ assert sem._value == 2 # semaphore fully released++ with _tokenizer_patch():+ anyio.run(_run)++ payloads = _parse_all_stage_logs(stage_log_capture)+ assert len(payloads) == 3+ waits = sorted(p["stages"]["pre_upstream_wait"] for p in payloads)+ # Exactly one request must have waited noticeably; the first two should+ # be near zero (they acquired the sem immediately).+ assert waits[0] < 25.0, waits+ assert waits[1] < 25.0, waits+ # The waiter should have waited roughly the upstream-delay budget.+ assert waits[2] > 75.0, waits+++# --------------------------------------------------------------------------- #+# Serialization: concurrency=1 => strict ordering of upstream enter timestamps #+# --------------------------------------------------------------------------- #+++def test_concurrency_one_serializes_requests():+ async def _run() -> float:+ sem = asyncio.Semaphore(1)+ handler = _DummyAnthropicHandler(+ anthropic_pre_upstream_sem=sem, upstream_delay_s=0.10+ )+ reqs = [+ _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": f"msg {i}"}],+ },+ {"authorization": "Bearer sk-ant-api-test"},+ )+ for i in range(2)+ ]+ start = time.perf_counter()+ await asyncio.gather(*(handler.handle_anthropic_messages(r) for r in reqs))+ elapsed = time.perf_counter() - start+ # Strict ordering: second request enters upstream only AFTER the first exits.+ assert len(handler.upstream_enter_times) == 2+ assert handler.upstream_enter_times[1] >= handler.upstream_exit_times[0] - 1e-6, (+ handler.upstream_enter_times,+ handler.upstream_exit_times,+ )+ return elapsed++ with _tokenizer_patch():+ elapsed = anyio.run(_run)+ # Two back-to-back 100 ms upstream calls under serialization: must take+ # at least ~2 * 100 ms. (Give a little slack for scheduler jitter.)+ assert elapsed >= 0.18, elapsed+++# --------------------------------------------------------------------------- #+# Unbounded mode: ``anthropic_pre_upstream_concurrency=0`` disables the sem. #+# --------------------------------------------------------------------------- #+++def test_unbounded_mode_no_semaphore_instance():+ config = ProxyConfig(anthropic_pre_upstream_concurrency=0)+ proxy = HeadroomProxy(config)+ assert proxy.anthropic_pre_upstream_sem is None+ assert proxy.anthropic_pre_upstream_concurrency == 0+++def test_unbounded_mode_requests_run_concurrently():+ """With concurrency=0 (sem disabled), two slow requests overlap."""++ async def _run() -> float:+ handler = _DummyAnthropicHandler(+ anthropic_pre_upstream_sem=None, upstream_delay_s=0.10+ )+ reqs = [+ _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": f"msg {i}"}],+ },+ {"authorization": "Bearer sk-ant-api-test"},+ )+ for i in range(2)+ ]+ start = time.perf_counter()+ await asyncio.gather(*(handler.handle_anthropic_messages(r) for r in reqs))+ return time.perf_counter() - start++ with _tokenizer_patch():+ elapsed = anyio.run(_run)+ # Unbounded -> both sleeps run in parallel. Total should be ~0.10 s,+ # nowhere near 0.20 s.+ assert elapsed < 0.18, elapsed+++# --------------------------------------------------------------------------- #+# Exception releases the semaphore. #+# --------------------------------------------------------------------------- #+++def test_exception_inside_critical_section_releases_semaphore():+ async def _run() -> None:+ sem = asyncio.Semaphore(2)+ baseline = sem._value+ handler = _DummyAnthropicHandler(+ anthropic_pre_upstream_sem=sem, raise_during_critical=True+ )+ # Drive several cycles to ensure we don't leak on any path.+ for i in range(5):+ req = _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": f"msg {i}"}],+ },+ {"authorization": "Bearer sk-ant-api-test"},+ )+ # The handler catches upstream RuntimeError internally and+ # returns a 5xx JSONResponse; this is the expected behaviour.+ await handler.handle_anthropic_messages(req)+ # After each cycle the semaphore must be fully restored.+ assert sem._value == baseline, (i, sem._value, baseline)++ with _tokenizer_patch():+ anyio.run(_run)+++# --------------------------------------------------------------------------- #+# /livez stays fast under Anthropic pre-upstream contention. #+# --------------------------------------------------------------------------- #+++def test_livez_unaffected_under_anthropic_backpressure():+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ anthropic_pre_upstream_concurrency=2,+ )+ app = create_app(config)+ assert app.state.proxy.anthropic_pre_upstream_sem is not None++ # Drain the semaphore so any simulated Anthropic request would block.+ proxy = app.state.proxy++ async def _drain_sem() -> None:+ # Acquire both permits — no request can enter the pre-upstream region.+ await proxy.anthropic_pre_upstream_sem.acquire()+ await proxy.anthropic_pre_upstream_sem.acquire()++ # Run an event loop just to drain the semaphore.+ loop = asyncio.new_event_loop()+ try:+ loop.run_until_complete(_drain_sem())+ finally:+ loop.close()++ latencies: list[float] = []+ with TestClient(app) as client:+ for _ in range(20):+ t0 = time.perf_counter()+ resp = client.get("/livez")+ latencies.append((time.perf_counter() - t0) * 1000.0)+ assert resp.status_code == 200+ assert resp.json()["alive"] is True++ p99 = statistics.quantiles(latencies, n=100)[98] if len(latencies) >= 100 else max(latencies)+ assert p99 < 100.0, (p99, latencies)+++# --------------------------------------------------------------------------- #+# Compression is NOT bypassed by the gate. #+# --------------------------------------------------------------------------- #+++def test_compression_is_not_bypassed_when_gated(stage_log_capture):+ """With ``optimize=True`` the first compression stage must still run."""++ class _Pipeline:+ def __init__(self) -> None:+ self.called = False++ def apply(self, messages, *args, **kwargs):+ self.called = True+ return SimpleNamespace(messages=messages, metadata={"applied_steps": ["first"]})++ sem = asyncio.Semaphore(2)+ handler = _DummyAnthropicHandler(anthropic_pre_upstream_sem=sem)+ handler.config = ProxyConfig(+ optimize=True,+ image_optimize=False,+ retry_max_attempts=1,+ retry_base_delay_ms=1,+ retry_max_delay_ms=1,+ connect_timeout_seconds=10,+ mode="token",+ cache_enabled=False,+ rate_limit_enabled=False,+ fallback_enabled=False,+ fallback_provider=None,+ prefix_freeze_enabled=False,+ memory_enabled=False,+ anthropic_pre_upstream_concurrency=2,+ )+ pipeline = _Pipeline()+ handler.anthropic_pipeline = pipeline++ # Large synthetic body to ensure the pipeline triggers.+ big_text = "x" * 50_000+ request = _build_request(+ {+ "model": "claude-3-5-sonnet-latest",+ "messages": [{"role": "user", "content": big_text}],+ },+ {"authorization": "Bearer sk-ant-api-test"},+ )++ with _tokenizer_patch():+ anyio.run(handler.handle_anthropic_messages, request)++ assert pipeline.called, "compression pipeline must still run under backpressure"+ # Semaphore restored after the request.+ assert sem._value == 2+++# --------------------------------------------------------------------------- #+# CLI: --anthropic-pre-upstream-concurrency plumbs into ProxyConfig. #+# --------------------------------------------------------------------------- #+++def _run_cli_capture(args: list[str], env: dict | None = None) -> ProxyConfig:+ """Invoke the proxy CLI, intercepting ``run_server`` to capture config.++ We do NOT want the CLI to actually start a server — monkeypatching the+ ``run_server`` entry point (imported lazily inside the click command+ via ``from headroom.proxy.server import ... run_server``) short-+ circuits it and lets us inspect the ``ProxyConfig`` that was built.+ """+ import headroom.proxy.server as server_mod++ captured: dict[str, ProxyConfig] = {}+ orig_run = server_mod.run_server++ def _fake_run(config: ProxyConfig): # noqa: D401 - stub+ captured["config"] = config+ return 0++ server_mod.run_server = _fake_run+ try:+ runner = CliRunner()+ result = runner.invoke(proxy_cli, args, env=env or {})+ finally:+ server_mod.run_server = orig_run++ assert result.exit_code == 0, (result.output, result.exception)+ assert "config" in captured, "run_server was not called"+ return captured["config"]+++def test_cli_flag_sets_pre_upstream_concurrency():+ config = _run_cli_capture(["--anthropic-pre-upstream-concurrency", "3"])+ assert config.anthropic_pre_upstream_concurrency == 3+++def test_env_var_sets_pre_upstream_concurrency():+ # Must set in the env passed to the runner (click reads envvar).+ # Also strip the corresponding CLI flag.+ env = {"HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY": "4"}+ # Also make sure we don't pick up a host user env that could override.+ config = _run_cli_capture([], env=env)+ assert config.anthropic_pre_upstream_concurrency == 4+++def test_cli_flag_overrides_env_var():+ env = {"HEADROOM_ANTHROPIC_PRE_UPSTREAM_CONCURRENCY": "4"}+ config = _run_cli_capture(+ ["--anthropic-pre-upstream-concurrency", "7"], env=env+ )+ assert config.anthropic_pre_upstream_concurrency == 7+++# --------------------------------------------------------------------------- #+# Sanity: HeadroomProxy auto-computes default when config value is None. #+# --------------------------------------------------------------------------- #+++def test_auto_computed_default_on_this_machine():+ config = ProxyConfig() # field left at None -> auto-compute.+ proxy = HeadroomProxy(config)+ expected = max(2, min(8, os.cpu_count() or 4))+ assert proxy.anthropic_pre_upstream_concurrency == expected+ assert proxy.anthropic_pre_upstream_sem is not None+ assert proxy.anthropic_pre_upstream_sem._value == expected