Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Fix Codex WS proxy (Issue #86), LiteLLM metrics, and backend forwarding
chopratejasdc1f3ff8217b
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex c2c81514c..8cc57e3b1 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -662,9 +662,19 @@ class LiteLLMBackend(Backend):)kwargs["messages"].insert(0, {"role": "system", "content": system_text})- # AWS region for Bedrock- if self.provider == "bedrock" and self.region:- kwargs["aws_region_name"] = self.region+ # Provider-specific region config+ if self.region:+ if self.provider == "bedrock":+ kwargs["aws_region_name"] = self.region+ elif self.provider in ("vertex_ai", "vertex_ai_beta"):+ kwargs["vertex_location"] = self.region++ # Forward API key from request headers if present+ auth_header = headers.get("authorization", headers.get("Authorization", ""))+ if auth_header.startswith("Bearer "):+ kwargs["api_key"] = auth_header[7:]+ elif headers.get("x-api-key"):+ kwargs["api_key"] = headers["x-api-key"]logger.debug(f"LiteLLM request: model={litellm_model}")@@ -752,8 +762,19 @@ class LiteLLMBackend(Backend):)kwargs["messages"].insert(0, {"role": "system", "content": system_text})- if self.provider == "bedrock" and self.region:- kwargs["aws_region_name"] = self.region+ # Provider-specific region config+ if self.region:+ if self.provider == "bedrock":+ kwargs["aws_region_name"] = self.region+ elif self.provider in ("vertex_ai", "vertex_ai_beta"):+ kwargs["vertex_location"] = self.region++ # Forward API key from request headers if present+ auth_header = headers.get("authorization", headers.get("Authorization", ""))+ if auth_header.startswith("Bearer "):+ kwargs["api_key"] = auth_header[7:]+ elif headers.get("x-api-key"):+ kwargs["api_key"] = headers["x-api-key"]msg_id = f"msg_{uuid.uuid4().hex[:24]}"@@ -961,12 +982,19 @@ class LiteLLMBackend(Backend):if param in body:kwargs[param] = body[param]- # Provider-specific config- if self.provider == "bedrock" and self.region:- kwargs["aws_region_name"] = self.region- elif self.provider == "databricks":- # Databricks uses env vars for auth- pass+ # Provider-specific region config+ if self.region:+ if self.provider == "bedrock":+ kwargs["aws_region_name"] = self.region+ elif self.provider in ("vertex_ai", "vertex_ai_beta"):+ kwargs["vertex_location"] = self.region++ # Forward API key from request headers if present+ auth_header = headers.get("authorization", headers.get("Authorization", ""))+ if auth_header.startswith("Bearer "):+ kwargs["api_key"] = auth_header[7:]+ elif headers.get("x-api-key"):+ kwargs["api_key"] = headers["x-api-key"]logger.debug(f"LiteLLM OpenAI request: model={litellm_model}")@@ -1086,8 +1114,19 @@ class LiteLLMBackend(Backend):if "stream_options" in body:kwargs["stream_options"] = body["stream_options"]- if self.provider == "bedrock" and self.region:- kwargs["aws_region_name"] = self.region+ # Provider-specific region config+ if self.region:+ if self.provider == "bedrock":+ kwargs["aws_region_name"] = self.region+ elif self.provider in ("vertex_ai", "vertex_ai_beta"):+ kwargs["vertex_location"] = self.region++ # Forward API key from request headers if present+ auth_header = headers.get("authorization", headers.get("Authorization", ""))+ if auth_header.startswith("Bearer "):+ kwargs["api_key"] = auth_header[7:]+ elif headers.get("x-api-key"):+ kwargs["api_key"] = headers["x-api-key"]response = await acompletion(**kwargs)diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 06355b721..b8b9b2c8c 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -250,13 +250,12 @@ def proxy(if license_key:license_status = f"MANAGED (key={license_key[:8]}...)"- effective_region = bedrock_region or region- backend_status = "Anthropic (direct API)"+ anthropic_url = config.anthropic_api_url or "https://api.anthropic.com"+ openai_url = config.openai_api_url or "https://api.openai.com"backend_section = ""if config.backend == "anyllm" or config.backend.startswith("anyllm-"):# any-llm backend- backend_status = f"{effective_anyllm_provider.title()} via any-llm"backend_section = """Set credentials for your provider (e.g., OPENAI_API_KEY, MISTRAL_API_KEY)Providers: https://mozilla-ai.github.io/any-llm/providers/@@ -268,14 +267,6 @@ def proxy(provider = config.backend.replace("litellm-", "")provider_config = get_provider_config(provider)- # Build backend status- if provider_config.uses_region:- backend_status = (- f"{provider_config.display_name} via LiteLLM (region={effective_region})"- )- else:- backend_status = f"{provider_config.display_name} via LiteLLM"-# Build usage instructions from provider configenv_vars_str = (", ".join(provider_config.env_vars) if provider_config.env_vars else "See docs"@@ -314,7 +305,6 @@ Memory (Multi-Provider):Starting proxy server...URL: http://{config.host}:{config.port}- Backend: {backend_status}Mode: {config.mode}Optimization: {"ENABLED" if config.optimize else "DISABLED"}Caching: {"ENABLED" if config.cache_enabled else "DISABLED"}@@ -322,18 +312,19 @@ Starting proxy server...Memory: {memory_status}License: {license_status}{backend_section}-Usage with Claude Code:- ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude+Routing:+ /v1/messages → {anthropic_url}+ /v1/chat/completions → {openai_url}+ /v1/responses → {openai_url} (HTTP + WebSocket)-Usage with OpenAI-compatible clients:- OPENAI_BASE_URL=http://{config.host}:{config.port}/v1 your-app+Usage:+ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude+ Codex / OpenAI: OPENAI_BASE_URL=http://{config.host}:{config.port}/v1 your-app{memory_section}Endpoints:GET /health Health checkGET /stats Detailed statisticsGET /metrics Prometheus metrics- POST /v1/messages Anthropic API- POST /v1/chat/completions OpenAI APIPress Ctrl+C to stop.""")diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.pyindex 1ddebdb8e..d6e2ffd7e 100644--- a/headroom/cli/wrap.py+++ b/headroom/cli/wrap.py@@ -54,7 +54,14 @@ def _get_log_path() -> Path:return log_dir / "proxy.log"-def _start_proxy(port: int, *, learn: bool = False) -> subprocess.Popen:+def _start_proxy(+ port: int,+ *,+ learn: bool = False,+ backend: str | None = None,+ anyllm_provider: str | None = None,+ region: str | None = None,+) -> subprocess.Popen:"""Start Headroom proxy as a background subprocess.Logs are written to ~/.headroom/logs/proxy.log to avoid pipe buffer@@ -72,6 +79,19 @@ def _start_proxy(port: int, *, learn: bool = False) -> subprocess.Popen:if learn:cmd.append("--learn")+ # Forward backend configuration to proxy subprocess+ _backend = backend or os.environ.get("HEADROOM_BACKEND")+ if _backend:+ cmd.extend(["--backend", _backend])++ _anyllm = anyllm_provider or os.environ.get("HEADROOM_ANYLLM_PROVIDER")+ if _anyllm:+ cmd.extend(["--anyllm-provider", _anyllm])++ _region = region or os.environ.get("HEADROOM_REGION")+ if _region:+ cmd.extend(["--region", _region])+log_path = _get_log_path()log_file = open(log_path, "a") # noqa: SIM115@@ -233,7 +253,15 @@ def _inject_rtk_instructions(file_path: Path, verbose: bool = False) -> bool:return True-def _ensure_proxy(port: int, no_proxy: bool, *, learn: bool = False) -> subprocess.Popen | None:+def _ensure_proxy(+ port: int,+ no_proxy: bool,+ *,+ learn: bool = False,+ backend: str | None = None,+ anyllm_provider: str | None = None,+ region: str | None = None,+) -> subprocess.Popen | None:"""Start or verify proxy. Returns process handle if we started it."""if not no_proxy:if _check_proxy(port):@@ -242,7 +270,13 @@ def _ensure_proxy(port: int, no_proxy: bool, *, learn: bool = False) -> subproceelse:click.echo(f" Starting Headroom proxy on port {port}...")try:- proc = _start_proxy(port, learn=learn)+ proc = _start_proxy(+ port,+ learn=learn,+ backend=backend,+ anyllm_provider=anyllm_provider,+ region=region,+ )click.echo(f" Proxy ready on http://127.0.0.1:{port}")return procexcept RuntimeError as e:@@ -303,6 +337,9 @@ def _launch_tool(env_vars_display: list[str],*,learn: bool = False,+ backend: str | None = None,+ anyllm_provider: str | None = None,+ region: str | None = None,) -> None:"""Common logic: start proxy, launch tool, clean up."""proxy_holder: list[subprocess.Popen | None] = [None]@@ -318,7 +355,14 @@ def _launch_tool(click.echo(" ╚═══════════════════════════════════════════════╝")click.echo()- proxy_holder[0] = _ensure_proxy(port, no_proxy, learn=learn)+ proxy_holder[0] = _ensure_proxy(+ port,+ no_proxy,+ learn=learn,+ backend=backend,+ anyllm_provider=anyllm_provider,+ region=region,+ )click.echo()click.echo(f" Launching {tool_label} (API routed through Headroom)...")@@ -449,10 +493,31 @@ def claude(@click.option("--learn", is_flag=True, help="Enable live traffic learning (patterns saved to AGENTS.md)")+ "--backend",+ default=None,+ help="API backend for the proxy: 'anthropic', 'anyllm', 'litellm-vertex', etc. (env: HEADROOM_BACKEND)",+)+ "--anyllm-provider",+ default=None,+ help="Provider for any-llm backend: openai, mistral, groq, etc. (env: HEADROOM_ANYLLM_PROVIDER)",+)+ "--region", default=None, help="Cloud region for Bedrock/Vertex (env: HEADROOM_REGION)"+)@click.option("--verbose", "-v", is_flag=True, help="Verbose output")@click.argument("codex_args", nargs=-1, type=click.UNPROCESSED)def codex(- port: int, no_rtk: bool, no_proxy: bool, learn: bool, verbose: bool, codex_args: tuple+ port: int,+ no_rtk: bool,+ no_proxy: bool,+ learn: bool,+ backend: str | None,+ anyllm_provider: str | None,+ region: str | None,+ verbose: bool,+ codex_args: tuple,) -> None:"""Launch OpenAI Codex CLI through Headroom proxy.@@ -467,6 +532,7 @@ def codex(headroom wrap codex -- "fix the bug" # Pass prompt to codexheadroom wrap codex --no-rtk # Skip rtk setupheadroom wrap codex --port 9999 # Custom proxy port+ headroom wrap codex --backend anyllm --anyllm-provider groq"""codex_bin = shutil.which("codex")if not codex_bin:@@ -499,6 +565,9 @@ def codex(tool_label="CODEX",env_vars_display=[f"OPENAI_BASE_URL=http://127.0.0.1:{port}/v1"],learn=learn,+ backend=backend,+ anyllm_provider=anyllm_provider,+ region=region,)@@ -512,10 +581,23 @@ def codex(@click.option("--no-rtk", is_flag=True, help="Skip rtk installation and conventions injection")@click.option("--no-proxy", is_flag=True, help="Skip proxy startup (use existing proxy)")@click.option("--learn", is_flag=True, help="Enable live traffic learning")+ "--backend", default=None, help="API backend: 'anthropic', 'anyllm', 'litellm-vertex', etc."+)[email protected]("--anyllm-provider", default=None, help="Provider for any-llm backend")[email protected]("--region", default=None, help="Cloud region for Bedrock/Vertex")@click.option("--verbose", "-v", is_flag=True, help="Verbose output")@click.argument("aider_args", nargs=-1, type=click.UNPROCESSED)def aider(- port: int, no_rtk: bool, no_proxy: bool, learn: bool, verbose: bool, aider_args: tuple+ port: int,+ no_rtk: bool,+ no_proxy: bool,+ learn: bool,+ backend: str | None,+ anyllm_provider: str | None,+ region: str | None,+ verbose: bool,+ aider_args: tuple,) -> None:"""Launch aider through Headroom proxy.@@ -530,6 +612,7 @@ def aider(headroom wrap aider -- --model gpt-4o # Use GPT-4oheadroom wrap aider -- --model claude-sonnet-4 # Use Claudeheadroom wrap aider --no-rtk # Skip rtk setup+ headroom wrap aider --backend litellm-vertex --region us-central1"""aider_bin = shutil.which("aider")if not aider_bin:@@ -562,6 +645,9 @@ def aider(f"ANTHROPIC_BASE_URL=http://127.0.0.1:{port}",],learn=learn,+ backend=backend,+ anyllm_provider=anyllm_provider,+ region=region,)diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex a168579c9..98064f195 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -2653,8 +2653,11 @@ class HeadroomProxy:usage = backend_response.body.get("usage", {})output_tokens = usage.get("output_tokens", 0)+ _backend_name = (+ self.anthropic_backend.name if self.anthropic_backend else "anthropic"+ )await self.metrics.record_request(- provider="bedrock",+ provider=_backend_name,model=model,input_tokens=optimized_tokens,output_tokens=output_tokens,@@ -2674,7 +2677,7 @@ class HeadroomProxy:RequestLog(request_id=request_id,timestamp=datetime.now().isoformat(),- provider="bedrock",+ provider=_backend_name,model=model,input_tokens_original=original_tokens,input_tokens_optimized=optimized_tokens,@@ -4869,8 +4872,11 @@ class HeadroomProxy:total_latency = (time.time() - start_time) * 1000output_tokens = stream_state["output_tokens"]+ _backend_name = (+ self.anthropic_backend.name if self.anthropic_backend else "anthropic"+ )await self.metrics.record_request(- provider="bedrock",+ provider=_backend_name,model=model,input_tokens=optimized_tokens,output_tokens=output_tokens,@@ -4891,7 +4897,7 @@ class HeadroomProxy:RequestLog(request_id=request_id,timestamp=datetime.now().isoformat(),- provider="bedrock",+ provider=_backend_name,model=model,input_tokens_original=original_tokens,input_tokens_optimized=optimized_tokens,@@ -6264,6 +6270,14 @@ class HeadroomProxy:body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)+ # /v1/responses is OpenAI-specific (Codex) — always routes direct.+ # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.+ if self.anthropic_backend is not None:+ logger.debug(+ f"[{request_id}] /v1/responses always routes to OpenAI direct "+ f"(backend '{self.anthropic_backend.name}' not used for Responses API)"+ )+url = f"{self.OPENAI_API_URL}/v1/responses"try:@@ -6277,7 +6291,7 @@ class HeadroomProxy:model,request_id,original_tokens,- original_tokens,+ optimized_tokens,tokens_saved,transforms_applied,tags,@@ -6360,31 +6374,46 @@ class HeadroomProxy:)return- await websocket.accept()request_id = await self._next_request_id()# Forward client headers to upstream, adding required OpenAI-Beta headerws_headers = dict(websocket.headers)+ # Extract subprotocol from client — this is an application-level negotiation+ # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).+ # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.+ client_subprotocols: list[str] = []+ raw_protocol = ws_headers.get("sec-websocket-protocol", "")+ if raw_protocol:+ client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]++ # Accept client connection with the requested subprotocol+ if client_subprotocols:+ await websocket.accept(subprotocol=client_subprotocols[0])+ else:+ await websocket.accept()+# Build upstream WebSocket URL (http→ws, https→wss)base = self.OPENAI_API_URLws_base = base.replace("https://", "wss://").replace("http://", "ws://")upstream_url = f"{ws_base}/v1/responses"- # Forward all relevant headers (auth, org, project, beta, etc.)- # Skip hop-by-hop headers that shouldn't be forwarded+ # Forward all client headers except hop-by-hop / per-connection headers.+ # These are WebSocket handshake mechanics that the `websockets` library+ # generates fresh for the upstream connection — forwarding them would conflict.+ # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is._skip_headers = frozenset({- "host",- "connection",- "upgrade",- "sec-websocket-key",- "sec-websocket-version",- "sec-websocket-extensions",- "sec-websocket-accept",- "sec-websocket-protocol",- "content-length",- "transfer-encoding",+ "host", # must match upstream, not local proxy+ "connection", # hop-by-hop+ "upgrade", # hop-by-hop+ "sec-websocket-key", # per-connection cryptographic nonce+ "sec-websocket-version", # protocol version (websockets lib sets this)+ "sec-websocket-extensions", # per-connection negotiation+ "sec-websocket-accept", # server-side only+ "sec-websocket-protocol", # handled via subprotocols param below+ "content-length", # hop-by-hop+ "transfer-encoding", # hop-by-hop})upstream_headers: dict[str, str] = {}@@ -6392,10 +6421,30 @@ class HeadroomProxy:if k.lower() not in _skip_headers:upstream_headers[k] = v+ # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.+ # Safety net for clients that don't forward auth headers via WebSocket upgrade.+ _has_auth = "authorization" in {k.lower() for k in upstream_headers}+ if not _has_auth:+ api_key = os.environ.get("OPENAI_API_KEY")+ if api_key:+ upstream_headers["Authorization"] = f"Bearer {api_key}"+ logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")+ else:+ logger.warning(+ f"[{request_id}] WS: no Authorization header from client and "+ f"OPENAI_API_KEY not set — upstream will likely reject"+ )+# Ensure the required beta header is present — OpenAI returns 500 without itif "openai-beta" not in {k.lower() for k in upstream_headers}:upstream_headers["OpenAI-Beta"] = "responses-api=v1"+ logger.debug(+ f"[{request_id}] WS upstream headers: "+ f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "+ f"subprotocols={client_subprotocols}"+ )+try:# Receive the first message from client (the response.create request)first_msg_raw = await websocket.receive_text()@@ -6479,6 +6528,11 @@ class HeadroomProxy:async with websockets.connect(upstream_url,additional_headers=upstream_headers,+ subprotocols=(+ [websockets.Subprotocol(p) for p in client_subprotocols]+ if client_subprotocols and hasattr(websockets, "Subprotocol")+ else client_subprotocols or None+ ),ssl=ssl_ctx if upstream_url.startswith("wss://") else None,) as upstream:# Send (potentially compressed) first message@@ -6490,16 +6544,28 @@ class HeadroomProxy:while True:msg = await websocket.receive_text()await upstream.send(msg)- except Exception:+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ logger.debug(+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"+ )with contextlib.suppress(Exception):await upstream.close()async def _upstream_to_client() -> None:try:async for msg in upstream:- await websocket.send_text(msg if isinstance(msg, str) else msg.decode())- except Exception:- pass+ if isinstance(msg, str):+ await websocket.send_text(msg)+ elif isinstance(msg, bytes):+ await websocket.send_bytes(msg)+ else:+ await websocket.send_text(str(msg))+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ logger.debug(+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"+ )finally:with contextlib.suppress(Exception):await websocket.close()@@ -6524,7 +6590,19 @@ class HeadroomProxy:except Exception as e:if "WebSocketDisconnect" not in type(e).__name__:- logger.error(f"[{request_id}] WS proxy error: {e}")+ # Extract response body from websockets InvalidStatus for better debugging+ error_detail = str(e)+ if hasattr(e, "response"):+ try:+ resp = e.response+ body_bytes = getattr(resp, "body", None) or b""+ if body_bytes:+ error_detail += (+ f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"+ )+ except Exception:+ pass+ logger.error(f"[{request_id}] WS proxy error: {error_detail}")with contextlib.suppress(Exception):await websocket.close(code=1011, reason=str(e)[:120])