Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: resolve rebased ci regressions

JerrettDavis470bb6cfb9e2
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
index 734d6b89e..afc9d8b26 100644
--- a/.claude-plugin/marketplace.json
+++ b/.claude-plugin/marketplace.json
@@ -1,30 +1,30 @@
-{
- "name": "headroom-marketplace",
- "owner": {
- "name": "Headroom Contributors"
- },
- "metadata": {
- "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
- "version": "0.9.2"
- },
- "plugins": [
- {
- "name": "headroom",
- "source": "./plugins/headroom-agent-hooks",
- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "version": "0.9.2",
- "author": {
- "name": "Headroom Contributors",
- "url": "https://github.com/chopratejas/headroom"
- },
- "homepage": "https://github.com/chopratejas/headroom",
- "repository": "https://github.com/chopratejas/headroom",
- "keywords": [
- "headroom",
- "hooks",
- "claude-code",
- "copilot-cli"
- ]
- }
- ]
-}
+{
+ "name": "headroom-marketplace",
+ "owner": {
+ "name": "Headroom Contributors"
+ },
+ "metadata": {
+ "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
+ "version": "0.10.0"
+ },
+ "plugins": [
+ {
+ "name": "headroom",
+ "source": "./plugins/headroom-agent-hooks",
+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
+ "version": "0.10.0",
+ "author": {
+ "name": "Headroom Contributors",
+ "url": "https://github.com/chopratejas/headroom"
+ },
+ "homepage": "https://github.com/chopratejas/headroom",
+ "repository": "https://github.com/chopratejas/headroom",
+ "keywords": [
+ "headroom",
+ "hooks",
+ "claude-code",
+ "copilot-cli"
+ ]
+ }
+ ]
+}
diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json
index 734d6b89e..afc9d8b26 100644
--- a/.github/plugin/marketplace.json
+++ b/.github/plugin/marketplace.json
@@ -1,30 +1,30 @@
-{
- "name": "headroom-marketplace",
- "owner": {
- "name": "Headroom Contributors"
- },
- "metadata": {
- "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
- "version": "0.9.2"
- },
- "plugins": [
- {
- "name": "headroom",
- "source": "./plugins/headroom-agent-hooks",
- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "version": "0.9.2",
- "author": {
- "name": "Headroom Contributors",
- "url": "https://github.com/chopratejas/headroom"
- },
- "homepage": "https://github.com/chopratejas/headroom",
- "repository": "https://github.com/chopratejas/headroom",
- "keywords": [
- "headroom",
- "hooks",
- "claude-code",
- "copilot-cli"
- ]
- }
- ]
-}
+{
+ "name": "headroom-marketplace",
+ "owner": {
+ "name": "Headroom Contributors"
+ },
+ "metadata": {
+ "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
+ "version": "0.10.0"
+ },
+ "plugins": [
+ {
+ "name": "headroom",
+ "source": "./plugins/headroom-agent-hooks",
+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
+ "version": "0.10.0",
+ "author": {
+ "name": "Headroom Contributors",
+ "url": "https://github.com/chopratejas/headroom"
+ },
+ "homepage": "https://github.com/chopratejas/headroom",
+ "repository": "https://github.com/chopratejas/headroom",
+ "keywords": [
+ "headroom",
+ "hooks",
+ "claude-code",
+ "copilot-cli"
+ ]
+ }
+ ]
+}
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 49938c1ae..41a422842 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -1,2746 +1,2746 @@
-"""OpenAI handler mixin for HeadroomProxy.
-
-Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import base64
-import contextlib
-import copy
-import json
-import logging
-import os
-import time
-import uuid
-from datetime import datetime
-from typing import TYPE_CHECKING, Any
-
-from headroom.proxy.helpers import jitter_delay_ms
-from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
-from headroom.proxy.ws_session_registry import (
- TerminationCause,
- WebSocketSessionRegistry,
- WSSessionHandle,
-)
-
-if TYPE_CHECKING:
- from fastapi import Request, WebSocket
- from fastapi.responses import JSONResponse, Response, StreamingResponse
-
-import httpx
-
-from headroom.copilot_auth import apply_copilot_api_auth
-from headroom.pipeline import PipelineStage, summarize_routing_markers
-
-logger = logging.getLogger("headroom.proxy")
-
-
-# Interactive Responses turns are latency-sensitive. Fail open quickly rather
-# than holding the session hostage on memory lookup.
-RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0
-
-# Cap the wait for the first client frame after the WS handshake completes.
-# A zombie or malicious client that accepts the upgrade but never sends the
-# first response.create frame would otherwise hold a slot indefinitely and
-# starve the session registry. 60 s is generous for real clients (Codex
-# typically sends the first frame within a few hundred milliseconds of the
-# accept) but short enough to bound the damage from a hung peer.
-WS_FIRST_FRAME_TIMEOUT_SECONDS = 60.0
-
-
-def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:
- """Best-effort decode of an OpenAI OAuth bearer token payload.
-
- OpenClaw's Codex OAuth flow may forward only the bearer token after the
- provider base URL is overridden. In that case the explicit
- ``ChatGPT-Account-ID`` header can be missing even though the JWT still
- carries the account id we need to route to the ChatGPT Codex backend.
- """
- auth = headers.get("authorization") or headers.get("Authorization")
- if not auth:
- return None
-
- scheme, _, token = auth.partition(" ")
- if scheme.lower() != "bearer" or token.count(".") < 2:
- return None
-
- payload = token.split(".", 2)[1]
- payload += "=" * (-len(payload) % 4)
- # Intentionally no signature verification here: this is only a best-effort
- # routing hint extractor. Upstream still performs the actual auth/authz checks.
- try:
- decoded = base64.urlsafe_b64decode(payload.encode("ascii"))
- data = json.loads(decoded.decode("utf-8"))
- except (ValueError, UnicodeDecodeError):
- return None
-
- return data if isinstance(data, dict) else None
-
-
-def _resolve_codex_routing_headers(headers: dict[str, str]) -> tuple[dict[str, str], bool]:
- """Resolve ChatGPT Codex routing hints from explicit headers or OAuth JWT."""
- resolved = dict(headers)
- lower_lookup = {k.lower(): k for k in resolved}
-
- if "chatgpt-account-id" in lower_lookup:
- return resolved, True
-
- payload = _decode_openai_bearer_payload(resolved)
- auth_claims = payload.get("https://api.openai.com/auth") if isinstance(payload, dict) else None
- account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None
- if isinstance(account_id, str) and account_id.strip():
- resolved["ChatGPT-Account-ID"] = account_id.strip()
- return resolved, True
-
- return resolved, False
-
-
-class OpenAIHandlerMixin:
- """Mixin providing OpenAI API handler methods for HeadroomProxy."""
-
- @staticmethod
- def _strict_previous_turn_frozen_count(
- messages: list[dict[str, Any]],
- base_frozen_count: int,
- ) -> int:
- """Freeze all prior turns in cache mode; only final user turn is mutable."""
- if not messages:
- return base_frozen_count
- final_idx = len(messages) - 1
- if messages[final_idx].get("role") == "user":
- return max(base_frozen_count, final_idx)
- return len(messages)
-
- @staticmethod
- def _restore_frozen_prefix(
- original_messages: list[dict[str, Any]],
- candidate_messages: list[dict[str, Any]],
- *,
- frozen_message_count: int,
- ) -> tuple[list[dict[str, Any]], int]:
- """Force frozen prefix bytes to match original request exactly."""
- if frozen_message_count <= 0 or not original_messages:
- return candidate_messages, 0
-
- frozen = min(frozen_message_count, len(original_messages))
- restored = list(candidate_messages)
-
- if len(restored) < frozen:
- return list(original_messages[:frozen]) + restored, frozen
-
- changed = 0
- for idx in range(frozen):
- if restored[idx] != original_messages[idx]:
- restored[idx] = original_messages[idx]
- changed += 1
- return restored, changed
-
- async def handle_openai_chat(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle OpenAI /v1/chat/completions endpoint."""
- if not hasattr(self, "pipeline_extensions"):
- from headroom.pipeline import PipelineExtensionManager
-
- self.pipeline_extensions = PipelineExtensionManager(discover=False)
-
- from fastapi import HTTPException
- from fastapi.responses import JSONResponse, Response
-
- from headroom.ccr import CCRToolInjector
- from headroom.proxy.helpers import (
- COMPRESSION_TIMEOUT_SECONDS,
- MAX_MESSAGE_ARRAY_LENGTH,
- MAX_REQUEST_BODY_SIZE,
- _read_request_json,
- )
- from headroom.proxy.modes import is_cache_mode, is_token_mode
- from headroom.tokenizers import get_tokenizer
- from headroom.utils import extract_user_query
-
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "type": "invalid_request_error",
- "code": "request_too_large",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
- model = body.get("model", "unknown")
- messages = body.get("messages", [])
- original_client_messages = copy.deepcopy(messages)
- input_event = self.pipeline_extensions.emit(
- PipelineStage.INPUT_RECEIVED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=messages,
- tools=body.get("tools"),
- metadata={"path": "/v1/chat/completions", "stream": body.get("stream", False)},
- )
- if input_event.messages is not None:
- messages = input_event.messages
- original_client_messages = copy.deepcopy(messages)
- if input_event.tools is not None:
- body["tools"] = input_event.tools
-
- # Validate message array size
- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Message array too large ({len(messages)} messages). "
- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
- "type": "invalid_request_error",
- "code": "invalid_request",
- }
- },
- )
-
- stream = body.get("stream", False)
-
- # Bypass: skip ALL compression for explicit opt-out
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- if _bypass:
- logger.info(f"[{request_id}] Bypass: skipping compression (header)")
-
- # Image compression: tile alignment + ML-based technique routing
- if self.config.image_optimize and messages and not _bypass:
- from headroom.proxy.helpers import _get_image_compressor
-
- compressor = _get_image_compressor()
- if compressor and compressor.has_images(messages):
- messages = compressor.compress(messages, provider="openai")
- if compressor.last_result:
- logger.info(
- f"[{request_id}] Image: {compressor.last_result.technique.value} "
- f"({compressor.last_result.savings_percent:.0f}% saved, "
- f"{compressor.last_result.original_tokens} → "
- f"{compressor.last_result.compressed_tokens} tokens)"
- )
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- # Strip accept-encoding so httpx negotiates its own encoding.
- # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
- # if httpx lacks brotli support the response body is undecipherable → 502.
- headers.pop("accept-encoding", None)
- tags = self._extract_tags(headers)
-
- # Memory: Get user ID when memory is enabled
- memory_user_id: str | None = None
- if self.memory_handler:
- memory_user_id = headers.get(
- "x-headroom-user-id",
- os.environ.get("USER", os.environ.get("USERNAME", "default")),
- )
-
- # Rate limiting
- if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited(provider="openai")
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Check cache
- if self.cache and not stream:
- cached = await self.cache.get(messages, model)
- if cached:
- self.pipeline_extensions.emit(
- PipelineStage.INPUT_CACHED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=messages,
- metadata={"cache_hit": True, "path": "/v1/chat/completions"},
- )
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0, # Savings already counted when response was cached
- latency_ms=(time.time() - start_time) * 1000,
- cached=True,
- )
-
- # Remove compression headers from cached response
- response_headers = dict(cached.response_headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(content=cached.response_body, headers=response_headers)
-
- # Token counting
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Hook: pre_compress
- _hook_biases = None
- if self.config.hooks:
- from headroom.hooks import CompressContext
-
- _hook_ctx = CompressContext(model=model, provider="openai")
- try:
- messages = self.config.hooks.pre_compress(messages, _hook_ctx)
- _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
- except Exception as e:
- logger.debug(f"[{request_id}] Hook error: {e}")
-
- # Optimization
- transforms_applied = []
- pipeline_timing: dict[str, float] = {}
- waste_signals_dict: dict[str, int] | None = None
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- # Get prefix cache tracker for this session
- openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
- openai_prefix_tracker = self.session_tracker_store.get_or_create(
- openai_session_id, "openai"
- )
- openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
- if is_cache_mode(self.config.mode):
- openai_frozen_count = self._strict_previous_turn_frozen_count(
- original_client_messages,
- openai_frozen_count,
- )
-
- _compression_failed = False
- original_messages = messages # Preserve for 400-retry fallback
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
- if self.config.optimize and messages and not _bypass and _license_ok:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
-
- if is_token_mode(self.config.mode):
- comp_cache = self._get_compression_cache(openai_session_id)
-
- # Zone 1: Swap cached compressed versions
- working_messages = comp_cache.apply_cached(messages)
-
- # Re-freeze boundary
- openai_frozen_count = comp_cache.compute_frozen_count(messages)
-
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=working_messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(working_messages),
- frozen_message_count=openai_frozen_count,
- biases=_hook_biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != working_messages:
- comp_cache.update_from_result(messages, result.messages)
-
- # Always use pipeline result in token mode
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- # Keep original_tokens as the REAL original (pre-Zone-1-swap)
- # so tokens_saved captures both Zone 1 + Zone 2 savings.
- optimized_tokens = result.tokens_after
- else:
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- frozen_message_count=openai_frozen_count,
- biases=_hook_biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
-
- if result.waste_signals:
- waste_signals_dict = result.waste_signals.to_dict()
- except Exception as e:
- logger.warning(f"Optimization failed: {e}")
- # Flag compression failure for observability
- _compression_failed = True
-
- # Guard: if "optimization" inflated tokens, revert to originals
- if optimized_tokens > original_tokens:
- logger.warning(
- f"[{request_id}] Optimization inflated tokens "
- f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
- )
- optimized_messages = original_messages
- optimized_tokens = original_tokens
- transforms_applied = []
-
- tokens_saved = original_tokens - optimized_tokens
- optimization_latency = (time.time() - start_time) * 1000
-
- routing_markers = summarize_routing_markers(transforms_applied)
- if routing_markers:
- routed_event = self.pipeline_extensions.emit(
- PipelineStage.INPUT_ROUTED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=optimized_messages,
- metadata={
- "routing_markers": routing_markers,
- "transforms_applied": transforms_applied,
- },
- )
- if routed_event.messages is not None:
- optimized_messages = routed_event.messages
- optimized_tokens = tokenizer.count_messages(optimized_messages)
- tokens_saved = original_tokens - optimized_tokens
-
- compressed_event = self.pipeline_extensions.emit(
- PipelineStage.INPUT_COMPRESSED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=optimized_messages,
- metadata={
- "tokens_before": original_tokens,
- "tokens_after": optimized_tokens,
- "transforms_applied": transforms_applied,
- },
- )
- if compressed_event.messages is not None:
- optimized_messages = compressed_event.messages
- optimized_tokens = tokenizer.count_messages(optimized_messages)
- tokens_saved = original_tokens - optimized_tokens
-
- # Hook: post_compress
- if self.config.hooks and tokens_saved > 0:
- from headroom.hooks import CompressEvent
-
- try:
- self.config.hooks.post_compress(
- CompressEvent(
- tokens_before=original_tokens,
- tokens_after=optimized_tokens,
- tokens_saved=tokens_saved,
- compression_ratio=tokens_saved / original_tokens
- if original_tokens > 0
- else 0,
- transforms_applied=transforms_applied,
- model=model,
- provider="openai",
- )
- )
- except Exception as e:
- logger.debug(f"[{request_id}] post_compress hook error: {e}")
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = body.get("tools")
- _original_tools = tools # Preserve for diagnostic / future retry
- if (
- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
- ) and not _bypass:
- injector = CCRToolInjector(
- provider="openai",
- inject_tool=self.config.ccr_inject_tool,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- optimized_messages, tools, was_injected = injector.process_request(
- optimized_messages, tools
- )
-
- if injector.has_compressed_content:
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
- )
- else:
- logger.debug(
- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
- )
-
- # Query Echo: disabled — hurts prefix caching in long conversations.
- if is_cache_mode(self.config.mode):
- optimized_messages, restored_count = self._restore_frozen_prefix(
- original_client_messages,
- optimized_messages,
- frozen_message_count=openai_frozen_count,
- )
- if restored_count > 0:
- logger.warning(
- f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
- "to preserve cache stability (openai)"
- )
-
- # Memory: inject context and tools for OpenAI requests
- memory_context_injected = False
- memory_tools_injected = False
- if self.memory_handler and memory_user_id:
- try:
- # Inject memory context (search similar memories, add as system message)
- if self.memory_handler.config.inject_context:
- memory_context = await self.memory_handler.search_and_format_context(
- memory_user_id, optimized_messages
- )
- if memory_context:
- # Prepend as system message for OpenAI format
- optimized_messages = [
- {"role": "system", "content": memory_context},
- *optimized_messages,
- ]
- memory_context_injected = True
- logger.info(
- f"[{request_id}] Memory: Injected {len(memory_context)} chars "
- f"of context for user {memory_user_id}"
- )
-
- # Inject memory tools
- if self.memory_handler.config.inject_tools:
- tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "openai")
- if mem_tools_injected:
- memory_tools_injected = True
- logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")
- except Exception as e:
- logger.warning(f"[{request_id}] Memory injection failed: {e}")
-
- if memory_context_injected or memory_tools_injected:
- remembered_event = self.pipeline_extensions.emit(
- PipelineStage.INPUT_REMEMBERED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=optimized_messages,
- tools=tools,
- metadata={
- "memory_context_injected": memory_context_injected,
- "memory_tools_injected": memory_tools_injected,
- },
- )
- if remembered_event.messages is not None:
- optimized_messages = remembered_event.messages
- if remembered_event.tools is not None:
- tools = remembered_event.tools
-
- body["messages"] = optimized_messages
- if tools is not None:
- body["tools"] = tools
-
- presend_event = self.pipeline_extensions.emit(
- PipelineStage.PRE_SEND,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=optimized_messages,
- tools=tools,
- headers=headers,
- metadata={"path": "/v1/chat/completions", "stream": stream},
- )
- if presend_event.messages is not None:
- optimized_messages = presend_event.messages
- body["messages"] = optimized_messages
- if presend_event.tools is not None:
- tools = presend_event.tools
- body["tools"] = tools
- if presend_event.headers is not None:
- headers = presend_event.headers
- optimized_tokens = tokenizer.count_messages(body["messages"])
- tokens_saved = original_tokens - optimized_tokens
-
- # Route through LiteLLM/any-llm backend if configured
- if self.anthropic_backend is not None:
- try:
- if stream:
- self.pipeline_extensions.emit(
- PipelineStage.POST_SEND,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=body["messages"],
- tools=tools,
- metadata={"path": "/v1/chat/completions", "stream": True},
- )
- # Streaming: use stream_openai_message() → SSE events
- return await self._stream_openai_via_backend(
- body,
- headers,
- model,
- request_id,
- start_time,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- )
- else:
- # Non-streaming: use send_openai_message() → JSON
- backend_response = await self.anthropic_backend.send_openai_message(
- body, headers
- )
- self.pipeline_extensions.emit(
- PipelineStage.POST_SEND,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=body["messages"],
- tools=tools,
- response=backend_response.body,
- metadata={
- "path": "/v1/chat/completions",
- "stream": False,
- "status_code": backend_response.status_code,
- },
- )
- self.pipeline_extensions.emit(
- PipelineStage.RESPONSE_RECEIVED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- response=backend_response.body,
- metadata={
- "path": "/v1/chat/completions",
- "stream": False,
- "status_code": backend_response.status_code,
- },
- )
-
- if backend_response.error:
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
-
- # Track metrics
- total_latency = (time.time() - start_time) * 1000
- usage = backend_response.body.get("usage", {})
- output_tokens = usage.get("completion_tokens", 0)
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
-
- await self.metrics.record_request(
- provider=self.anthropic_backend.name,
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- cached=False,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"
- )
-
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
- except Exception as e:
- logger.error(f"[{request_id}] Backend error: {e}")
- return JSONResponse(
- status_code=500,
- content={
- "error": {
- "message": str(e),
- "type": "api_error",
- "code": "backend_error",
- }
- },
- )
-
- # Direct OpenAI API (no backend configured)
- url = f"{self.OPENAI_API_URL}/v1/chat/completions"
-
- try:
- if stream:
- # Inject stream_options to get usage stats in streaming response
- # This allows accurate token counting instead of byte-based estimation
- if "stream_options" not in body:
- body["stream_options"] = {"include_usage": True}
- elif isinstance(body.get("stream_options"), dict):
- body["stream_options"]["include_usage"] = True
-
- self.pipeline_extensions.emit(
- PipelineStage.POST_SEND,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=body["messages"],
- tools=tools,
- metadata={"path": "/v1/chat/completions", "stream": True},
- )
- return await self._stream_response(
- url,
- headers,
- body,
- "openai",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- prefix_tracker=openai_prefix_tracker,
- )
- else:
- headers = await apply_copilot_api_auth(headers, url=url)
- response = await self._retry_request("POST", url, headers, body)
- self.pipeline_extensions.emit(
- PipelineStage.POST_SEND,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- messages=body["messages"],
- tools=tools,
- response=response,
- metadata={
- "path": "/v1/chat/completions",
- "stream": False,
- "status_code": response.status_code,
- },
- )
- self.pipeline_extensions.emit(
- PipelineStage.RESPONSE_RECEIVED,
- operation="proxy.request",
- request_id=request_id,
- provider="openai",
- model=model,
- response=response,
- metadata={
- "path": "/v1/chat/completions",
- "stream": False,
- "status_code": response.status_code,
- },
- )
-
- # Full diagnostic dump on upstream errors (OpenAI handler)
- if response.status_code >= 400:
- try:
- err_body = response.json()
- err_msg = err_body.get("error", {}).get("message", "")
- err_type = err_body.get("error", {}).get("type", "")
- except Exception:
- err_body = {"raw": response.text[:2000]}
- err_msg = str(response.text[:500])
- err_type = "parse_error"
-
- logger.warning(
- f"[{request_id}] UPSTREAM_ERROR "
- f"status={response.status_code} "
- f"error_type={err_type} "
- f"error_msg={err_msg!r} "
- f"model={model} "
- f"compressed={'yes' if transforms_applied else 'no'} "
- f"transforms={transforms_applied} "
- f"original_tokens={original_tokens} "
- f"optimized_tokens={optimized_tokens} "
- f"message_count={len(body.get('messages', []))} "
- f"stream={stream}"
- )
-
- try:
- from headroom import paths as _hr_paths
-
- debug_dir = _hr_paths.debug_400_dir()
- debug_dir.mkdir(parents=True, exist_ok=True)
- ts = datetime.now().strftime("%Y%m%d_%H%M%S")
- debug_file = debug_dir / f"{ts}_{request_id}.json"
-
- safe_headers = {}
- for k, v in headers.items():
- if k.lower() in ("x-api-key", "authorization"):
- safe_headers[k] = v[:12] + "..." if v else ""
- else:
- safe_headers[k] = v
-
- debug_payload = {
- "request_id": request_id,
- "timestamp": datetime.now().isoformat(),
- "status_code": response.status_code,
- "error_response": err_body,
- "model": model,
- "stream": stream,
- "headers": safe_headers,
- "compression": {
- "was_compressed": bool(transforms_applied),
- "transforms": transforms_applied,
- "original_tokens": original_tokens,
- "optimized_tokens": optimized_tokens,
- "tokens_saved": tokens_saved,
- "compression_failed": _compression_failed,
- },
- "tools_sent": body.get("tools"),
- "tool_count": len(body.get("tools") or []),
- "original_tool_count": len(_original_tools or []),
- "messages_sent": body.get("messages"),
- "message_count": len(body.get("messages", [])),
- "original_messages": (
- original_messages
- if original_messages is not body.get("messages")
- else "__same_as_sent__"
- ),
- "original_message_count": len(original_messages),
- "system_prompt": body.get("system"),
- }
-
- with open(debug_file, "w") as f:
- json.dump(debug_payload, f, indent=2, default=str)
-
- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
- except Exception as dump_err:
- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
-
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = optimized_tokens # fallback
- output_tokens = 0
- cache_read_tokens = 0
- resp_json = None
- try:
- resp_json = response.json()
- usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
- output_tokens = usage.get("completion_tokens", 0)
- # OpenAI returns cached_tokens in prompt_tokens_details
- # These are charged at 50% of the input price
- prompt_details = usage.get("prompt_tokens_details") or {}
- cache_read_tokens = prompt_details.get("cached_tokens", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
- )
-
- # Update prefix cache tracker for next turn
- openai_prefix_tracker.update_from_response(
- cache_read_tokens=cache_read_tokens,
- cache_write_tokens=0, # OpenAI doesn't report write tokens
- messages=optimized_messages,
- )
-
- # OpenAI has no write penalty — uncached = total - cached
- uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(
- model,
- tokens_saved,
- optimized_tokens,
- cache_read_tokens=cache_read_tokens,
- uncached_tokens=uncached_input_tokens,
- )
-
- # Memory: handle memory tool calls in OpenAI Chat Completions response.
- # After executing tools, send a continuation request so the model
- # can produce a final user-facing response (not just tool_calls).
- if (
- self.memory_handler
- and memory_user_id
- and resp_json
- and response.status_code == 200
- and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
- ):
- try:
- tool_results = await self.memory_handler.handle_memory_tool_calls(
- resp_json, memory_user_id, "openai"
- )
- if tool_results:
- # Build continuation: original messages + assistant tool_calls + tool results
- assistant_msg = resp_json.get("choices", [{}])[0].get("message", {})
- continuation_messages = list(optimized_messages)
- continuation_messages.append(assistant_msg)
- continuation_messages.extend(tool_results)
-
- continuation_body = {
- **body,
- "messages": continuation_messages,
- }
-
- cont_response = await self._retry_request(
- "POST", url, headers, continuation_body
- )
- if cont_response.status_code == 200:
- resp_json = cont_response.json()
- response = cont_response
-
- logger.info(
- f"[{request_id}] Memory: Handled {len(tool_results)} "
- f"tool call(s) with continuation for user {memory_user_id}"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Memory tool handling failed: {e}")
-
- # Cache
- if self.cache and response.status_code == 200:
- await self.cache.set(
- messages,
- model,
- response.content,
- dict(response.headers),
- tokens_saved,
- )
-
- # Capture Codex rate-limit window data from response headers
- from headroom.subscription.codex_rate_limits import (
- get_codex_rate_limit_state,
- )
-
- get_codex_rate_limit_state().update_from_headers(dict(response.headers))
-
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- waste_signals=waste_signals_dict,
- cache_read_tokens=cache_read_tokens,
- uncached_input_tokens=uncached_input_tokens,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens)"
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Inject Headroom compression metrics (for SaaS metering)
- response_headers["x-headroom-tokens-before"] = str(original_tokens)
- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
- response_headers["x-headroom-model"] = model
- if transforms_applied:
- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
- if cache_read_tokens > 0:
- response_headers["x-headroom-cached"] = "true"
- if _compression_failed:
- response_headers["x-headroom-compression-failed"] = "true"
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed(provider="openai")
- # Log full error details internally for debugging
- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
- # Return sanitized error message to client (don't expose internal details)
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "type": "server_error",
- "code": "proxy_error",
- }
- },
- )
-
- async def handle_openai_responses(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle OpenAI /v1/responses endpoint (new Responses API).
-
- The Responses API differs from /v1/chat/completions:
- - Input: `input` (string or array) instead of `messages`
- - System: `instructions` instead of system message
- - Output: `output[]` array instead of `choices[].message`
- - State: `previous_response_id` for multi-turn
- - Built-in tools: web_search, file_search, code_interpreter
- """
- from fastapi import HTTPException
- from fastapi.responses import JSONResponse, Response
-
- from headroom.proxy.helpers import (
- COMPRESSION_TIMEOUT_SECONDS,
- MAX_REQUEST_BODY_SIZE,
- _read_request_json,
- )
- from headroom.proxy.responses_converter import (
- messages_to_responses_items,
- responses_items_to_messages,
- )
- from headroom.tokenizers import get_tokenizer
- from headroom.utils import extract_user_query
-
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "type": "invalid_request_error",
- "code": "request_too_large",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
-
- model = body.get("model", "unknown")
- stream = body.get("stream", False)
-
- # Convert Responses API input to messages format for optimization.
- # The Responses API uses a different item model (function_call,
- # function_call_output, reasoning as top-level items) — we convert to
- # Chat Completions messages for the pipeline, then convert back.
-
- input_data = body.get("input", "")
- instructions = body.get("instructions")
- previous_response_id = body.get("previous_response_id")
-
- messages: list[dict[str, Any]] = []
- original_items: list[dict[str, Any]] | None = None
- preserved_indices: list[int] = []
-
- if instructions:
- messages.append({"role": "system", "content": instructions})
-
- if isinstance(input_data, str):
- messages.append({"role": "user", "content": input_data})
- elif isinstance(input_data, list):
- original_items = input_data
- converted, preserved_indices = responses_items_to_messages(input_data)
- messages.extend(converted)
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- # Strip accept-encoding so httpx negotiates its own encoding.
- # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
- # if httpx lacks brotli support the response body is undecipherable → 502.
- headers.pop("accept-encoding", None)
- tags = self._extract_tags(headers)
-
- # Memory: Get user ID when memory is enabled
- memory_user_id: str | None = None
- if self.memory_handler:
- memory_user_id = headers.get(
- "x-headroom-user-id",
- os.environ.get("USER", os.environ.get("USERNAME", "default")),
- )
-
- # Rate limiting
- if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited(provider="openai")
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Token counting on converted messages
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Optimize: convert items → compress → convert back
- tokens_saved = 0
- transforms_applied: list[str] = []
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- _should_compress = (
- self.config.optimize
- and original_items is not None
- and not previous_response_id
- and not _bypass
- and len(messages) > 1
- )
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
-
- if _should_compress and _license_ok:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
- except Exception as e:
- logger.warning(f"[{request_id}] Responses API optimization failed: {e}")
-
- # Guard: if "optimization" inflated tokens, revert to originals
- if optimized_tokens > original_tokens:
- logger.warning(
- f"[{request_id}] Optimization inflated tokens "
- f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
- )
- optimized_messages = messages
- optimized_tokens = original_tokens
- transforms_applied = []
-
- tokens_saved = original_tokens - optimized_tokens
- optimization_latency = (time.time() - start_time) * 1000
-
- # Convert compressed messages back to Responses API items
- if optimized_messages is not messages and original_items is not None:
- opt_msgs = optimized_messages
- # Strip system message (instructions) — it's separate in Responses API
- if instructions and opt_msgs and opt_msgs[0].get("role") == "system":
- body["instructions"] = opt_msgs[0]["content"]
- opt_msgs = opt_msgs[1:]
-
- body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)
-
- # Memory: inject context and tools for Responses API requests
- if self.memory_handler and memory_user_id:
- try:
- # Inject memory context into instructions (Responses API system prompt)
- if self.memory_handler.config.inject_context:
- try:
- memory_context = await asyncio.wait_for(
- self.memory_handler.search_and_format_context(
- memory_user_id, optimized_messages
- ),
- timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- memory_context = None
- logger.info(
- f"[{request_id}] Memory context lookup exceeded "
- f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"
- )
- if memory_context:
- existing_instructions = body.get("instructions") or ""
- if existing_instructions:
- body["instructions"] = f"{existing_instructions}\n\n{memory_context}"
- else:
- body["instructions"] = memory_context
- logger.info(
- f"[{request_id}] Memory: Injected {len(memory_context)} chars "
- f"of context into instructions for user {memory_user_id}"
- )
-
- # Inject memory tools (Responses API format)
- if self.memory_handler.config.inject_tools:
- resp_tools = body.get("tools") or []
- resp_tools, mem_tools_injected = self.memory_handler.inject_tools(
- resp_tools, "openai"
- )
- if mem_tools_injected:
- # Convert Chat Completions format to Responses API format
- converted_tools = []
- for t in resp_tools:
- if t.get("type") == "function" and "function" in t:
- fn = t["function"]
- converted_tools.append(
- {
- "type": "function",
- "name": fn.get("name"),
- "description": fn.get("description", ""),
- "parameters": fn.get("parameters", {}),
- }
- )
- else:
- converted_tools.append(t)
- body["tools"] = converted_tools
- logger.info(
- f"[{request_id}] Memory: Injected memory tools (openai/responses)"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Memory injection failed (responses): {e}")
-
- # /v1/responses is OpenAI-specific (Codex) — always routes direct.
- # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.
- if self.anthropic_backend is not None:
- logger.debug(
- f"[{request_id}] /v1/responses always routes to OpenAI direct "
- f"(backend '{self.anthropic_backend.name}' not used for Responses API)"
- )
-
- headers, is_chatgpt_auth = _resolve_codex_routing_headers(headers)
-
- # Route to correct endpoint based on auth mode.
- # ChatGPT session auth (codex login) uses chatgpt.com, not api.openai.com.
- if is_chatgpt_auth:
- url = "https://chatgpt.com/backend-api/codex/responses"
- else:
- url = f"{self.OPENAI_API_URL}/v1/responses"
-
- try:
- if stream:
- # Streaming for Responses API uses semantic events
- return await self._stream_response(
- url,
- headers,
- body,
- "openai",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- memory_user_id=memory_user_id,
- )
- else:
- headers = await apply_copilot_api_auth(headers, url=url)
- response = await self._retry_request("POST", url, headers, body)
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = original_tokens # fallback
- output_tokens = 0
- try:
- resp_json = response.json()
- usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("input_tokens", original_tokens)
- output_tokens = usage.get("output_tokens", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"
- )
-
- # Memory: handle memory tool calls in Responses API response
- if (
- self.memory_handler
- and memory_user_id
- and resp_json
- and response.status_code == 200
- and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
- ):
- try:
- # Extract function_call items from output
- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
-
- output_items = resp_json.get("output", [])
- memory_fc_items = [
- item
- for item in output_items
- if isinstance(item, dict)
- and item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ]
-
- # Execute memory tool calls
- tool_outputs: list[dict[str, Any]] = []
- for fc in memory_fc_items:
- call_id = fc.get("call_id", fc.get("id", ""))
- name = fc.get("name", "")
- args_str = fc.get("arguments", "{}")
- try:
- args = json.loads(args_str)
- except json.JSONDecodeError:
- args = {}
-
- await self.memory_handler._ensure_initialized()
- if self.memory_handler._backend:
- result = await self.memory_handler._execute_memory_tool(
- name, args, memory_user_id, "openai"
- )
- else:
- result = json.dumps({"error": "Memory backend not initialized"})
-
- tool_outputs.append(
- {
- "type": "function_call_output",
- "call_id": call_id,
- "output": result,
- }
- )
-
- if tool_outputs:
- # Make continuation request with tool results
- response_id = resp_json.get("id")
- continuation_body = {
- "model": model,
- "input": tool_outputs,
- }
- if response_id:
- continuation_body["previous_response_id"] = response_id
- existing_tools = body.get("tools")
- if existing_tools:
- continuation_body["tools"] = existing_tools
-
- cont_response = await self._retry_request(
- "POST", url, headers, continuation_body
- )
- resp_json = cont_response.json()
- response = cont_response
- logger.info(
- f"[{request_id}] Memory: Handled {len(tool_outputs)} "
- f"tool call(s) with continuation for user {memory_user_id} (responses)"
- )
- except Exception as e:
- logger.warning(
- f"[{request_id}] Memory tool handling failed (responses): {e}"
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)
-
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- )
-
- logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")
-
- # Capture Codex rate-limit window data from response headers
- from headroom.subscription.codex_rate_limits import (
- get_codex_rate_limit_state,
- )
-
- get_codex_rate_limit_state().update_from_headers(dict(response.headers))
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed(provider="openai")
- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "type": "server_error",
- "code": "proxy_error",
- }
- },
- )
-
- async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:
- """WebSocket proxy for /v1/responses (Codex gpt-5.4+).
-
- Newer Codex versions use WebSocket instead of HTTP POST for the
- Responses API. This handler:
- 1. Accepts the client WebSocket
- 2. Receives the first message (``response.create`` request)
- 3. Compresses the ``input`` array using the existing pipeline
- 4. Opens an upstream WebSocket to OpenAI
- 5. Sends the compressed request upstream
- 6. Relays all subsequent messages bidirectionally
- """
- from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
- from headroom.tokenizers import get_tokenizer
- from headroom.utils import extract_user_query
-
- try:
- import websockets
- except ImportError:
- await websocket.accept()
- await websocket.close(
- code=1011,
- reason="websockets package not installed. pip install websockets",
- )
- return
-
- request_id = await self._next_request_id()
- session_id = uuid.uuid4().hex
-
- # Stage-timer — captures per-stage durations for the structured
- # log emitted on session close. Unit 2 instrumentation.
- stage_timer = StageTimer()
- session_started_at = time.perf_counter()
-
- # Unit 3: initialize registry variables *before* accept so the
- # outermost ``finally`` can rely on them existing even if
- # registration itself fails for some reason.
- ws_sessions: WebSocketSessionRegistry | None = getattr(self, "ws_sessions", None)
- session_handle: WSSessionHandle | None = None
- termination_cause: TerminationCause = "unknown"
-
- # Forward client headers to upstream, adding required OpenAI-Beta header
- ws_headers = dict(websocket.headers)
-
- # Extract subprotocol from client — this is an application-level negotiation
- # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
- # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.
- client_subprotocols: list[str] = []
- raw_protocol = ws_headers.get("sec-websocket-protocol", "")
- if raw_protocol:
- client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]
-
- # Accept client connection with the requested subprotocol
- async with stage_timer.measure("accept"):
- if client_subprotocols:
- await websocket.accept(subprotocol=client_subprotocols[0])
- else:
- await websocket.accept()
-
- # --- Unit 3: register the session as soon as accept succeeds ---
- client_addr: str | None = None
- client_info = getattr(websocket, "client", None)
- if client_info is not None:
- host = getattr(client_info, "host", None)
- port = getattr(client_info, "port", None)
- if host is not None and port is not None:
- client_addr = f"{host}:{port}"
- elif host is not None:
- client_addr = str(host)
- if ws_sessions is not None:
- session_handle = WSSessionHandle(
- session_id=session_id,
- request_id=request_id,
- client_addr=client_addr,
- upstream_url=None, # set below once upstream_url is computed
- )
- ws_sessions.register(session_handle)
- metrics = getattr(self, "metrics", None)
- if metrics is not None and hasattr(metrics, "inc_active_ws_sessions"):
- try:
- metrics.inc_active_ws_sessions()
- except Exception: # pragma: no cover - defensive
- pass
-
- # Forward all client headers except hop-by-hop / per-connection headers.
- # These are WebSocket handshake mechanics that the `websockets` library
- # generates fresh for the upstream connection — forwarding them would conflict.
- # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.
- _skip_headers = frozenset(
- {
- "host", # must match upstream, not local proxy
- "connection", # hop-by-hop
- "upgrade", # hop-by-hop
- "sec-websocket-key", # per-connection cryptographic nonce
- "sec-websocket-version", # protocol version (websockets lib sets this)
- "sec-websocket-extensions", # per-connection negotiation
- "sec-websocket-accept", # server-side only
- "sec-websocket-protocol", # handled via subprotocols param below
- "content-length", # hop-by-hop
- "transfer-encoding", # hop-by-hop
- }
- )
- upstream_headers: dict[str, str] = {}
- for k, v in ws_headers.items():
- if k.lower() not in _skip_headers:
- upstream_headers[k] = v
-
- upstream_headers, is_chatgpt_auth = _resolve_codex_routing_headers(upstream_headers)
- _lower_headers = {k.lower(): v for k, v in upstream_headers.items()}
-
- # Build upstream WebSocket URL based on auth mode
- if is_chatgpt_auth:
- # ChatGPT session auth → route to chatgpt.com backend
- upstream_url = "wss://chatgpt.com/backend-api/codex/responses"
- logger.debug(
- f"[{request_id}] WS: ChatGPT session auth detected, routing to chatgpt.com"
- )
- else:
- # API key auth → route to configured OpenAI API URL
- base = self.OPENAI_API_URL
- ws_base = base.replace("https://", "wss://").replace("http://", "ws://")
- upstream_url = f"{ws_base}/v1/responses"
-
- # Unit 3: attach the resolved upstream URL to the session handle.
- if session_handle is not None:
- session_handle.upstream_url = upstream_url
-
- # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.
- # Safety net for clients that don't forward auth headers via WebSocket upgrade.
- if "authorization" not in _lower_headers:
- api_key = os.environ.get("OPENAI_API_KEY")
- if api_key:
- upstream_headers["Authorization"] = f"Bearer {api_key}"
- logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")
- else:
- logger.warning(
- f"[{request_id}] WS: no Authorization header from client and "
- f"OPENAI_API_KEY not set — upstream will likely reject"
- )
-
- upstream_headers = await apply_copilot_api_auth(upstream_headers, url=upstream_url)
-
- # Ensure the required beta header is present — OpenAI returns 500 without it.
- # Codex sends `responses_websockets=2026-02-06`; only inject if missing entirely.
- if "openai-beta" not in _lower_headers:
- upstream_headers["OpenAI-Beta"] = "responses_websockets=2026-02-06"
-
- logger.debug(
- f"[{request_id}] WS upstream headers: "
- f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "
- f"subprotocols={client_subprotocols}"
- )
-
- try:
- # Receive the first message from client (the response.create request).
- # Bound the wait with WS_FIRST_FRAME_TIMEOUT_SECONDS so a zombie
- # client that opens the WS but never sends a frame cannot hold a
- # session slot indefinitely. The StageTimer measurement still
- # captures the elapsed time up to the timeout so operators can
- # see the slow-client pattern in the stage-timings log.
- try:
- async with stage_timer.measure("first_client_frame"):
- first_msg_raw = await asyncio.wait_for(
- websocket.receive_text(),
- timeout=WS_FIRST_FRAME_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- logger.info(
- f"[{request_id}] WS first-frame timeout after "
- f"{WS_FIRST_FRAME_TIMEOUT_SECONDS:.0f}s; closing session "
- f"{session_id} (no client data)"
- )
- termination_cause = "client_timeout"
- with contextlib.suppress(Exception):
- # 1001 (going away): server is cleanly terminating a slow
- # client, not an internal error.
- await websocket.close(code=1001, reason="first-frame timeout")
- # Exit the outer try so the session-lifecycle ``finally`` runs
- # deregister / metrics / stage-timings emission as usual.
- return
-
- # --- Optional: compress the input in the first message ---
- body: dict[str, Any] = {}
- try:
- body = json.loads(first_msg_raw)
- tokens_saved = 0
- ws_request_body = body.get("response", body)
- input_data = (
- ws_request_body.get("input") if isinstance(ws_request_body, dict) else None
- )
-
- should_compress = (
- self.config.optimize
- and isinstance(input_data, list)
- and len(input_data) > 1
- and not (
- ws_request_body.get("previous_response_id")
- if isinstance(ws_request_body, dict)
- else None
- )
- )
- if should_compress:
- try:
- from headroom.proxy.responses_converter import (
- messages_to_responses_items,
- responses_items_to_messages,
- )
-
- model = ws_request_body.get("model", "gpt-4o")
- converted, preserved = responses_items_to_messages(input_data)
-
- messages: list[dict[str, Any]] = []
- instructions = ws_request_body.get("instructions")
- if instructions:
- messages.append({"role": "system", "content": instructions})
- messages.extend(converted)
-
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- context_limit = self.openai_provider.get_context_limit(model)
- async with stage_timer.measure("compression"):
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- opt = result.messages
- if instructions and opt and opt[0].get("role") == "system":
- ws_request_body["instructions"] = opt[0]["content"]
- opt = opt[1:]
- if result.tokens_after <= original_tokens:
- ws_request_body["input"] = messages_to_responses_items(
- opt, input_data, preserved
- )
- else:
- logger.warning(
- f"[{request_id}] WS optimization inflated tokens "
- f"({original_tokens} -> {result.tokens_after}), reverting"
- )
- tokens_saved = max(0, original_tokens - result.tokens_after)
- if "response" in body and isinstance(body["response"], dict):
- body["response"] = ws_request_body
- else:
- body = ws_request_body
- first_msg_raw = json.dumps(body)
- logger.info(
- f"[{request_id}] WS /v1/responses compressed: "
- f"saved {tokens_saved} tokens"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] WS compression failed: {e}")
-
- except json.JSONDecodeError:
- # Not JSON — pass through as-is
- tokens_saved = 0
-
- # --- Memory: inject context, tools, and instructions ---
- memory_user_id: str | None = None
- if self.memory_handler and body:
- memory_user_id = ws_headers.get(
- "x-headroom-user-id",
- os.environ.get("USER", os.environ.get("USERNAME", "default")),
- )
- try:
- # Unwrap response.create envelope to access the response body
- ws_response_body = body.get("response", body)
-
- # Debug: log what Codex sends so we can see the full tool list
- existing_tool_names = [
- t.get("name") or t.get("function", {}).get("name", "?")
- for t in (ws_response_body.get("tools") or [])
- ]
- instr_preview = (ws_response_body.get("instructions") or "")[:200]
- logger.info(
- f"[{request_id}] WS Memory: Codex tools={existing_tool_names}, "
- f"instructions_len={len(ws_response_body.get('instructions') or '')}, "
- f"instructions_preview={instr_preview!r}"
- )
-
- # Inject memory context into instructions
- if self.memory_handler.config.inject_context:
- ws_input = ws_response_body.get("input", "")
- ws_instructions = ws_response_body.get("instructions")
- ws_msgs: list[dict[str, Any]] = []
- if ws_instructions:
- ws_msgs.append({"role": "system", "content": ws_instructions})
- if isinstance(ws_input, str) and ws_input:
- ws_msgs.append({"role": "user", "content": ws_input})
- elif isinstance(ws_input, list):
- from headroom.proxy.responses_converter import (
- responses_items_to_messages,
- )
-
- converted_msgs, _ = responses_items_to_messages(ws_input)
- ws_msgs.extend(converted_msgs)
-
- try:
- async with stage_timer.measure("memory_context"):
- memory_context = await asyncio.wait_for(
- self.memory_handler.search_and_format_context(
- memory_user_id, ws_msgs
- ),
- timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- memory_context = None
- logger.info(
- f"[{request_id}] WS Memory: Context lookup exceeded "
- f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "
- f"continuing without it"
- )
- if memory_context:
- existing = ws_response_body.get("instructions") or ""
- if existing:
- ws_response_body["instructions"] = f"{existing}\n\n{memory_context}"
- else:
- ws_response_body["instructions"] = memory_context
- logger.info(
- f"[{request_id}] WS Memory: Injected {len(memory_context)} chars "
- f"of context into instructions"
- )
-
- # Inject memory tools (Responses API format)
- if self.memory_handler.config.inject_tools:
- ws_tools = ws_response_body.get("tools") or []
- ws_tools, mem_injected = self.memory_handler.inject_tools(
- ws_tools, "openai"
- )
- if mem_injected:
- converted_tools = []
- for t in ws_tools:
- if t.get("type") == "function" and "function" in t:
- fn = t["function"]
- converted_tools.append(
- {
- "type": "function",
- "name": fn.get("name"),
- "description": fn.get("description", ""),
- "parameters": fn.get("parameters", {}),
- }
- )
- else:
- converted_tools.append(t)
- ws_response_body["tools"] = converted_tools
-
- # Add memory instruction so the model uses
- # memory tools as persistent cross-session knowledge.
- mem_instruction = (
- "\n\n## Memory\n"
- "You have persistent memory via memory_search and "
- "memory_save tools. Memory stores knowledge across "
- "sessions — user info, project details, org context, "
- "decisions, architecture, conventions, anything worth "
- "remembering.\n\n"
- "- ALWAYS call memory_search BEFORE searching files "
- "when the user asks a question that could be answered "
- "from prior knowledge.\n"
- "- Call memory_save to store important facts, decisions, "
- "or context that would be useful in future sessions.\n"
- "- Memory is your first source of truth for anything "
- "not visible in the current conversation."
- )
- existing_instr = ws_response_body.get("instructions") or ""
- ws_response_body["instructions"] = existing_instr + mem_instruction
- logger.info(
- f"[{request_id}] WS Memory: Injected memory tools + instruction"
- )
-
- # Write back into envelope if it was wrapped
- if "response" in body and isinstance(body["response"], dict):
- body["response"] = ws_response_body
- else:
- body = ws_response_body
-
- first_msg_raw = json.dumps(body)
- except Exception as e:
- logger.warning(f"[{request_id}] WS Memory injection failed: {e}")
-
- # --- Connect to upstream OpenAI WebSocket ---
- logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")
-
- # Use ssl=True to let the websockets library handle SSL natively.
- # Manual ssl.create_default_context() + certifi doesn't load the
- # Windows system cert store, causing HTTP 500 on wss:// connections.
- use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
-
- ws_connected = False
- ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
- ws_last_err: Exception | None = None
- _upstream_connect_started = time.perf_counter()
- _upstream_connect_recorded = False
- _upstream_first_event_started: float | None = None
-
- for ws_attempt in range(ws_connect_attempts):
- try:
- async with websockets.connect(
- upstream_url,
- additional_headers=upstream_headers,
- subprotocols=(
- [websockets.Subprotocol(p) for p in client_subprotocols]
- if client_subprotocols and hasattr(websockets, "Subprotocol")
- else client_subprotocols or None
- ),
- ssl=use_ssl,
- open_timeout=max(30, self.config.connect_timeout_seconds * 3),
- close_timeout=10,
- ping_interval=20,
- ping_timeout=20,
- ) as upstream:
- ws_connected = True
- if not _upstream_connect_recorded:
- stage_timer.record(
- "upstream_connect",
- (time.perf_counter() - _upstream_connect_started) * 1000.0,
- )
- _upstream_connect_recorded = True
- _upstream_first_event_started = time.perf_counter()
- await upstream.send(first_msg_raw)
-
- # Unit 3: flag the upstream side flips on seeing
- # ``response.completed`` so the outer cause
- # classifier can prefer it over the raw
- # "upstream iterator ended" default.
- response_completed_seen = False
- # Captures the first exception surfaced by the
- # inner relay ``except`` blocks so the outer
- # classifier can still tell ``upstream_error``
- # from ``upstream_disconnect`` / ``response_completed``
- # even though the halves swallow and log.
- upstream_relay_error: BaseException | None = None
- client_relay_error: BaseException | None = None
-
- async def _client_to_upstream() -> None:
- nonlocal client_relay_error
- try:
- while True:
- msg = await websocket.receive_text()
- await upstream.send(msg)
- except asyncio.CancelledError:
- # Explicit cancel from the outer
- # orchestrator — re-raise so
- # ``t.cancelled()`` and ``t.exception()``
- # behave correctly in the caller.
- raise
- except Exception as relay_err:
- # Surface real errors to the classifier
- # without re-raising (existing fork
- # behavior: log and return so the
- # partner task can be cancelled
- # deterministically).
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- client_relay_error = relay_err
- logger.debug(
- f"[{request_id}] WS client→upstream relay ended: {relay_err}"
- )
- with contextlib.suppress(Exception):
- await upstream.close()
-
- async def _upstream_to_client() -> None:
- """Relay upstream→client with transparent memory tool handling.
-
- Uses a buffer-then-decide approach:
- 1. Buffer events until first output item arrives
- 2. If first output is a memory tool → suppress entire response,
- execute tools silently, send continuation upstream
- 3. If first output is non-memory → flush buffer, stream normally
- 4. Continuation response events are relayed to Codex seamlessly
-
- This prevents orphaned response.created events from confusing Codex.
- """
- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
-
- # Unit 3: surface response.completed observation
- # to the outer scope so the termination-cause
- # classifier can prefer ``response_completed``
- # over ``upstream_disconnect``.
- nonlocal response_completed_seen
- nonlocal upstream_relay_error
-
- memory_enabled = bool(self.memory_handler and memory_user_id)
-
- # Per-response state (reset after each response.completed)
- event_buffer: list[str] = []
- decided = False
- suppress_response = False
- pending_fcs: list[dict[str, Any]] = []
- resp_id: str | None = None
-
- def _reset() -> None:
- nonlocal decided, suppress_response, resp_id
- event_buffer.clear()
- decided = False
- suppress_response = False
- pending_fcs.clear()
- resp_id = None
-
- # The retry-loop variable is safe to close over here:
- # ``_upstream_to_client`` is defined and awaited within
- # a single iteration and never escapes.
- _first_event_started_at = _upstream_first_event_started # noqa: B023
-
- try:
- async for msg in upstream:
- if (
- _first_event_started_at is not None
- and "upstream_first_event" not in stage_timer
- ):
- stage_timer.record(
- "upstream_first_event",
- (time.perf_counter() - _first_event_started_at)
- * 1000.0,
- )
- if isinstance(msg, bytes):
- await websocket.send_bytes(msg)
- continue
- msg_str = msg if isinstance(msg, str) else str(msg)
-
- if not memory_enabled:
- await websocket.send_text(msg_str)
- continue
-
- # Parse event
- try:
- event = json.loads(msg_str)
- except (json.JSONDecodeError, TypeError):
- await websocket.send_text(msg_str)
- continue
-
- event_type = event.get("type", "")
-
- # --- Phase 1: Buffer until first output item ---
- if not decided:
- event_buffer.append(msg_str)
-
- if event_type == "response.output_item.added":
- item = event.get("item", {})
- if (
- item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ):
- # Memory tool first → suppress entire response
- suppress_response = True
- decided = True
- event_buffer.clear()
- logger.info(
- f"[{request_id}] WS Memory: Detected "
- f"{item.get('name')} — suppressing response"
- )
- else:
- # Non-memory first → flush buffer, pass through
- decided = True
- for buf in event_buffer:
- await websocket.send_text(buf)
- event_buffer.clear()
-
- elif event_type == "response.completed":
- # No output items at all — flush
- decided = True
- for buf in event_buffer:
- await websocket.send_text(buf)
- event_buffer.clear()
- _reset()
- response_completed_seen = True
-
- continue
-
- # --- Phase 2a: Suppress mode (memory response) ---
- if suppress_response:
- if event_type == "response.output_item.done":
- item = event.get("item", {})
- if (
- item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ):
- pending_fcs.append(item)
-
- elif event_type == "response.completed":
- response_completed_seen = True
- resp = event.get("response", {})
- resp_id = resp.get("id")
-
- if pending_fcs:
- logger.info(
- f"[{request_id}] WS Memory: Executing "
- f"{len(pending_fcs)} tool(s) transparently"
- )
-
- # Execute memory tool calls
- tool_outputs: list[dict[str, Any]] = []
- for fc in pending_fcs:
- call_id = fc.get("call_id", fc.get("id", ""))
- fc_name = fc.get("name", "")
- args_str = fc.get("arguments", "{}")
- try:
- fc_args = json.loads(args_str)
- except json.JSONDecodeError:
- fc_args = {}
-
- await self.memory_handler._ensure_initialized()
- if self.memory_handler._backend:
- result = await self.memory_handler._execute_memory_tool(
- fc_name,
- fc_args,
- memory_user_id,
- "openai",
- )
- else:
- result = json.dumps(
- {"error": "backend not ready"}
- )
-
- tool_outputs.append(
- {
- "type": "function_call_output",
- "call_id": call_id,
- "output": result,
- }
- )
- logger.info(
- f"[{request_id}] WS Memory: Executed "
- f"{fc_name} for user {memory_user_id}"
- )
-
- # Send continuation upstream
- cont: dict[str, Any] = {
- "type": "response.create",
- "response": {"input": tool_outputs},
- }
- if resp_id:
- cont["response"]["previous_response_id"] = (
- resp_id
- )
- await upstream.send(json.dumps(cont))
- logger.info(
- f"[{request_id}] WS Memory: Sent continuation "
- f"with {len(tool_outputs)} result(s)"
- )
-
- _reset()
- # All events suppressed in this mode
- continue
-
- # --- Phase 2b: Pass-through mode ---
- await websocket.send_text(msg_str)
-
- except asyncio.CancelledError:
- raise
- except Exception as relay_err:
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- # Capture for the outer classifier
- # so ``upstream_error`` can be
- # distinguished from a clean
- # upstream disconnect.
- upstream_relay_error = relay_err
- logger.debug(
- f"[{request_id}] WS upstream→client relay ended: {relay_err}"
- )
- finally:
- with contextlib.suppress(Exception):
- await websocket.close()
-
- # --- Unit 3: deterministic relay-task cancellation ---
- # Spawn each half as a named task so we can:
- # (a) attach them to the session registry for
- # ``/debug/ws-sessions``,
- # (b) cancel the survivor explicitly when the
- # first one exits, and
- # (c) classify the termination cause for the
- # duration histogram.
- client_task = asyncio.create_task(
- _client_to_upstream(),
- name=f"codex-ws-c2u-{session_id}",
- )
- upstream_task = asyncio.create_task(
- _upstream_to_client(),
- name=f"codex-ws-u2c-{session_id}",
- )
- relay_tasks = [client_task, upstream_task]
- if ws_sessions is not None:
- ws_sessions.attach_tasks(session_id, relay_tasks)
- metrics_for_tasks = getattr(self, "metrics", None)
- if metrics_for_tasks is not None and hasattr(
- metrics_for_tasks, "inc_active_relay_tasks"
- ):
- try:
- metrics_for_tasks.inc_active_relay_tasks(len(relay_tasks))
- except Exception: # pragma: no cover - defensive
- pass
-
- try:
- done, pending = await asyncio.wait(
- {client_task, upstream_task},
- return_when=asyncio.FIRST_COMPLETED,
- )
- # Cancel the survivor so we don't leak the
- # partner task. Suppress the CancelledError
- # we just raised ourselves — any *other*
- # exception from the cancelled task is
- # already logged inside its own try/except.
- for t in pending:
- t.cancel()
- if pending:
- with contextlib.suppress(asyncio.CancelledError):
- await asyncio.gather(*pending, return_exceptions=True)
-
- # Classify termination cause from whichever
- # task completed first. ``CancelledError``
- # can show up on the "done" side if the
- # handler itself was cancelled from outside
- # (e.g. server shutdown).
- for t in done:
- exc = None
- # Cancelled tasks raise CancelledError from
- # .exception(); surface it explicitly so the
- # downstream ``isinstance(exc, CancelledError)``
- # branches actually run. For any other
- # unexpected state (``InvalidStateError`` if
- # the task somehow isn't done — shouldn't
- # happen post-gather but defensive), we
- # suppress and leave ``exc`` as ``None``.
- if t.cancelled():
- exc = asyncio.CancelledError()
- else:
- with contextlib.suppress(asyncio.InvalidStateError):
- exc = t.exception()
- task_name = t.get_name() or ""
- if t is client_task:
- if client_relay_error is not None:
- termination_cause = "client_error"
- elif exc is None:
- termination_cause = "client_disconnect"
- elif isinstance(exc, asyncio.CancelledError):
- termination_cause = "client_disconnect"
- else:
- # Distinguish legitimate client
- # disconnect exceptions from
- # real errors: WebSocketDisconnect
- # is a normal client exit.
- if "WebSocketDisconnect" in type(exc).__name__:
- termination_cause = "client_disconnect"
- else:
- termination_cause = "client_error"
- elif t is upstream_task:
- if upstream_relay_error is not None:
- termination_cause = "upstream_error"
- logger.debug(
- f"[{request_id}] WS relay {task_name} "
- f"raised: {upstream_relay_error!r}"
- )
- elif exc is None:
- termination_cause = (
- "response_completed"
- if response_completed_seen
- else "upstream_disconnect"
- )
- elif isinstance(exc, asyncio.CancelledError):
- termination_cause = "upstream_disconnect"
- else:
- termination_cause = "upstream_error"
- logger.debug(
- f"[{request_id}] WS relay {task_name} raised: {exc!r}"
- )
- finally:
- # In case anything above raised before the
- # cancel-and-await loop ran.
- for t in relay_tasks:
- if not t.done():
- t.cancel()
- with contextlib.suppress(asyncio.CancelledError):
- await asyncio.gather(*relay_tasks, return_exceptions=True)
-
- logger.info(
- f"[{request_id}] WS /v1/responses completed "
- f"(tokens_saved={tokens_saved}, cause={termination_cause})"
- )
- break
- except Exception as ws_err:
- if ws_connected:
- # WS was established but broke mid-stream — re-raise
- raise
-
- ws_last_err = ws_err
- if ws_attempt >= ws_connect_attempts - 1:
- break
-
- delay_with_jitter = jitter_delay_ms(
- self.config.retry_base_delay_ms,
- self.config.retry_max_delay_ms,
- ws_attempt,
- )
- logger.warning(
- f"[{request_id}] WS upstream connect failed "
- f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "
- f"retrying in {delay_with_jitter:.0f}ms"
- )
- await asyncio.sleep(delay_with_jitter / 1000)
-
- if not ws_connected:
- # WS upgrade failed (HTTP 500 from OpenAI is common).
- # Fall back to HTTP POST streaming and relay SSE events
- # back over the client WebSocket transparently.
- ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")
- _ws_detail = str(ws_err)
- if hasattr(ws_err, "response"):
- resp_body = getattr(getattr(ws_err, "response", None), "body", b"")
- if resp_body:
- _ws_detail += f" | {resp_body[:300].decode('utf-8', errors='replace')}"
- logger.warning(
- f"[{request_id}] WS upstream failed ({_ws_detail}), "
- f"falling back to HTTP POST streaming"
- )
- await self._ws_http_fallback(
- websocket, body, first_msg_raw, upstream_headers, request_id
- )
-
- # Record metrics
- if tokens_saved > 0:
- model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"
- await self.metrics.record_request(
- provider="openai",
- model=model_name,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=tokens_saved,
- latency_ms=0,
- )
-
- except Exception as e:
- if "WebSocketDisconnect" in type(e).__name__:
- # Unit 3: client dropped the socket before or during
- # relay. The registry classifier may already have set
- # ``client_disconnect`` via the relay task exit path;
- # preserve that, otherwise set it here.
- if termination_cause == "unknown":
- termination_cause = "client_disconnect"
- else:
- # Extract response body from websockets InvalidStatus for better debugging
- error_detail = str(e)
- if hasattr(e, "response"):
- try:
- resp = e.response
- body_bytes = getattr(resp, "body", None) or b""
- if body_bytes:
- error_detail += (
- f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"
- )
- except Exception:
- pass
- logger.error(f"[{request_id}] WS proxy error: {error_detail}")
- if termination_cause == "unknown":
- termination_cause = "client_error"
- with contextlib.suppress(Exception):
- await websocket.close(code=1011, reason=str(e)[:120])
- finally:
- # Unit 2: emit structured per-session stage timings.
- stage_timer.record(
- "total_session",
- (time.perf_counter() - session_started_at) * 1000.0,
- )
- # Unit 3: deregister the session before (or independently
- # of) the stage-timings log so a failure there cannot leak
- # the registry entry. ``deregister`` is idempotent, so a
- # session that never registered is a no-op.
- if ws_sessions is not None and session_handle is not None:
- # Use deregister_and_count so the handle pop and the
- # relay-task count are read atomically inside the
- # registry. Capturing ``len(session_handle.relay_tasks)``
- # separately before ``deregister`` would risk drift if
- # the registry's bookkeeping ever changes.
- _deregistered, released_tasks = ws_sessions.deregister_and_count(
- session_id, cause=termination_cause
- )
- session_duration_ms = (time.perf_counter() - session_started_at) * 1000.0
- metrics_for_close = getattr(self, "metrics", None)
- if metrics_for_close is not None:
- with contextlib.suppress(Exception):
- if hasattr(metrics_for_close, "dec_active_ws_sessions"):
- metrics_for_close.dec_active_ws_sessions()
- if released_tasks and hasattr(metrics_for_close, "dec_active_relay_tasks"):
- metrics_for_close.dec_active_relay_tasks(released_tasks)
- if hasattr(metrics_for_close, "record_ws_session_duration"):
- metrics_for_close.record_ws_session_duration(
- session_duration_ms, termination_cause
- )
- await emit_stage_timings_log(
- path="openai_responses_ws",
- request_id=request_id,
- session_id=session_id,
- stage_timer=stage_timer,
- expected_stages=(
- "accept",
- "first_client_frame",
- "upstream_connect",
- "upstream_first_event",
- "memory_context",
- "compression",
- "total_session",
- ),
- metrics=getattr(self, "metrics", None),
- )
-
- async def _ws_http_fallback(
- self,
- websocket: WebSocket,
- body: dict[str, Any],
- first_msg_raw: str,
- upstream_headers: dict[str, str],
- request_id: str,
- ) -> None:
- """Fall back to HTTP POST streaming when upstream WS fails.
-
- Converts the WS ``response.create`` message to an HTTP POST to
- ``/v1/responses?stream=true``, reads SSE events, and relays each
- ``data:`` line as a WS text message to the client. This makes
- Codex work immediately instead of exhausting its WS retry budget.
- """
- # Route to correct endpoint based on auth mode
- _lower = {k.lower() for k in upstream_headers}
- if "chatgpt-account-id" in _lower:
- http_url = "https://chatgpt.com/backend-api/codex/responses"
- else:
- http_url = f"{self.OPENAI_API_URL}/v1/responses"
-
- # Build HTTP body from the WS response.create payload.
- # WS messages use {"type": "response.create", "response": {...}} wrapper.
- # The HTTP POST endpoint expects the inner response object directly.
- http_body: dict[str, Any]
- try:
- parsed = json.loads(first_msg_raw) if isinstance(first_msg_raw, str) else body
- except (json.JSONDecodeError, TypeError):
- parsed = body
-
- # Normalize WebSocket response.create payload into the HTTP request body.
- # Codex may send either:
- # 1. {"type":"response.create","response":{...}}
- # 2. {"type":"response.create", ...response fields...}
- if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):
- http_body = dict(parsed["response"])
- elif isinstance(parsed, dict):
- http_body = dict(parsed)
- if http_body.get("type") == "response.create":
- http_body.pop("type", None)
- else:
- http_body = body if isinstance(body, dict) else {}
-
- # Some clients include response-ish metadata that the HTTP endpoint rejects.
- if http_body.get("type") in {"response.create", "response"}:
- http_body.pop("type", None)
-
- # Ensure streaming is enabled so we get SSE events
- http_body["stream"] = True
-
- # Build HTTP headers from the upstream headers (already stripped of WS
- # hop-by-hop headers by the caller).
- http_headers = dict(upstream_headers)
- http_headers["content-type"] = "application/json"
-
- logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")
-
- try:
- retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
- for http_attempt in range(retry_attempts):
- try:
- async with self.http_client.stream(
- "POST",
- http_url,
- headers=http_headers,
- json=http_body,
- timeout=120.0,
- ) as response:
- if response.status_code != 200:
- error_body = b""
- async for chunk in response.aiter_bytes():
- error_body += chunk
- if len(error_body) > 2000:
- break
- error_text = error_body.decode("utf-8", errors="replace")
- logger.error(
- f"[{request_id}] WS→HTTP fallback got {response.status_code}: "
- f"{error_text[:500]}"
- )
- # Send error as WS message so client sees it
- error_event = {
- "type": "error",
- "error": {
- "type": "server_error",
- "message": f"Upstream returned {response.status_code}",
- },
- }
- await websocket.send_text(json.dumps(error_event))
- return
-
- # Relay SSE events as WS text messages
- buffer = ""
- async for chunk in response.aiter_text():
- buffer += chunk
- while "\n" in buffer:
- line, buffer = buffer.split("\n", 1)
- line = line.strip()
- if not line:
- continue
- if line.startswith("data: "):
- data = line[6:]
- if data == "[DONE]":
- continue
- try:
- await websocket.send_text(data)
- except Exception:
- return
- elif line.startswith("event: "):
- # SSE event type — skip, the data line contains the type
- continue
-
- # Flush any remaining data in buffer
- for line in buffer.strip().splitlines():
- line = line.strip()
- if line.startswith("data: ") and line[6:] != "[DONE]":
- with contextlib.suppress(Exception):
- await websocket.send_text(line[6:])
- return
- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:
- if http_attempt >= retry_attempts - 1:
- raise
-
- delay_with_jitter = jitter_delay_ms(
- self.config.retry_base_delay_ms,
- self.config.retry_max_delay_ms,
- http_attempt,
- )
- logger.warning(
- f"[{request_id}] WS→HTTP fallback connect failed "
- f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "
- f"retrying in {delay_with_jitter:.0f}ms"
- )
- await asyncio.sleep(delay_with_jitter / 1000)
-
- except Exception as http_err:
- logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")
- error_event = {
- "type": "error",
- "error": {
- "type": "server_error",
- "message": f"HTTP fallback failed: {http_err!s}"[:200],
- },
- }
- with contextlib.suppress(Exception):
- await websocket.send_text(json.dumps(error_event))
- finally:
- with contextlib.suppress(Exception):
- await websocket.close()
-
- async def handle_compress(self, request: Request) -> JSONResponse:
- """Compress messages without calling an LLM.
-
- POST /v1/compress
- Body: {"messages": [...], "model": "...", "config": {}}
- Returns compressed messages + metrics.
- """
- from fastapi.responses import JSONResponse
-
- from headroom.proxy.helpers import _read_request_json
-
- # Check bypass header
- if request.headers.get("x-headroom-bypass", "").lower() == "true":
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={"error": f"Invalid request body: {e!s}"},
- )
- messages = body.get("messages", [])
- return JSONResponse(
- {
- "messages": messages,
- "tokens_before": 0,
- "tokens_after": 0,
- "tokens_saved": 0,
- "compression_ratio": 1.0,
- "transforms_applied": [],
- "ccr_hashes": [],
- }
- )
-
- try:
- body = await _read_request_json(request)
- except Exception:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Invalid JSON in request body.",
- }
- },
- )
-
- messages = body.get("messages")
- model = body.get("model")
-
- if messages is None:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Missing required field: messages",
- }
- },
- )
-
- if model is None:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Missing required field: model",
- }
- },
- )
-
- if not messages:
- return JSONResponse(
- {
- "messages": [],
- "tokens_before": 0,
- "tokens_after": 0,
- "tokens_saved": 0,
- "compression_ratio": 1.0,
- "transforms_applied": [],
- "ccr_hashes": [],
- }
- )
-
- try:
- # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)
- # Allow optional token_budget to override model's context limit
- # (used by OpenClaw compact() and other callers that need tighter budgets)
- token_budget = body.get("token_budget")
- context_limit = (
- token_budget
- if token_budget and isinstance(token_budget, int)
- else self.openai_provider.get_context_limit(model)
- )
- # Extract CompressConfig options from request body
- compress_config = body.get("config", {})
- compress_user_messages = compress_config.get("compress_user_messages", False)
- target_ratio = compress_config.get("target_ratio")
- protect_recent = compress_config.get("protect_recent")
- protect_analysis_context = compress_config.get("protect_analysis_context")
-
- pipeline_kwargs: dict = {"model_limit": context_limit}
- if compress_user_messages:
- pipeline_kwargs["compress_user_messages"] = True
- if target_ratio is not None:
- pipeline_kwargs["target_ratio"] = float(target_ratio)
- if protect_recent is not None:
- pipeline_kwargs["protect_recent"] = int(protect_recent)
- if protect_analysis_context is not None:
- pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)
-
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- **pipeline_kwargs,
- )
-
- return JSONResponse(
- {
- "messages": result.messages,
- "tokens_before": result.tokens_before,
- "tokens_after": result.tokens_after,
- "tokens_saved": result.tokens_before - result.tokens_after,
- "compression_ratio": (
- result.tokens_after / result.tokens_before
- if result.tokens_before > 0
- else 1.0
- ),
- "transforms_applied": result.transforms_applied,
- "transforms_summary": result.transforms_summary,
- "ccr_hashes": result.markers_inserted,
- }
- )
- except Exception as e:
- logger.exception("Compression failed: %s", e)
- return JSONResponse(
- status_code=503,
- content={
- "error": {
- "type": "compression_error",
- "message": str(e),
- }
- },
- )
-
- async def handle_passthrough(
- self,
- request: Request,
- base_url: str,
- endpoint_name: str | None = None,
- provider: str | None = None,
- ) -> Response:
- """Pass through request unchanged.
-
- Args:
- request: The incoming request
- base_url: The upstream API base URL
- endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")
- provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")
- """
- from fastapi.responses import Response
-
- start_time = time.time()
- path = request.url.path
- url = f"{base_url}{path}"
-
- # Preserve query string parameters
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("accept-encoding", None)
-
- body = await request.body()
-
- headers = await apply_copilot_api_auth(headers, url=url)
- response = await self.http_client.request( # type: ignore[union-attr]
- method=request.method,
- url=url,
- headers=headers,
- content=body,
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Track stats for passthrough requests
- if endpoint_name and provider:
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider=provider,
- model=f"passthrough:{endpoint_name}",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- cached=False,
- )
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- async def handle_databricks_invocations(
- self,
- request: Request,
- model: str,
- ) -> Response | StreamingResponse:
- """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.
-
- This enables using the Databricks CLI directly with Headroom:
- databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'
-
- The request/response format is identical to OpenAI chat completions,
- so we inject the model from the path and delegate to handle_openai_chat.
- """
- from fastapi.responses import JSONResponse
-
- from headroom.proxy.helpers import _read_request_json
-
- request_id = await self._next_request_id()
-
- try:
- body = await _read_request_json(request)
- except Exception as e:
- logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid JSON: {e}",
- "type": "invalid_request_error",
- }
- },
- )
-
- # Inject model from path into body (Databricks CLI passes model in URL, not body)
- body["model"] = model
-
- logger.info(f"[{request_id}] Databricks invocation: model={model}")
-
- # Create a new request with the modified body
- # We reuse the OpenAI chat handler since the format is identical
- from starlette.requests import Request as StarletteRequest
-
- # Build new scope with the body already parsed
- scope = dict(request.scope)
-
- # Create a simple receive function that returns our modified body
- body_bytes = json.dumps(body).encode()
-
- async def receive():
- return {"type": "http.request", "body": body_bytes}
-
- modified_request = StarletteRequest(scope, receive)
-
- # Delegate to the OpenAI chat handler (same format)
- return await self.handle_openai_chat(modified_request)
+"""OpenAI handler mixin for HeadroomProxy.
+
+Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import contextlib
+import copy
+import json
+import logging
+import os
+import time
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from headroom.proxy.helpers import jitter_delay_ms
+from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
+from headroom.proxy.ws_session_registry import (
+ TerminationCause,
+ WebSocketSessionRegistry,
+ WSSessionHandle,
+)
+
+if TYPE_CHECKING:
+ from fastapi import Request, WebSocket
+ from fastapi.responses import JSONResponse, Response, StreamingResponse
+
+import httpx
+
+from headroom.copilot_auth import apply_copilot_api_auth
+from headroom.pipeline import PipelineStage, summarize_routing_markers
+
+logger = logging.getLogger("headroom.proxy")
+
+
+# Interactive Responses turns are latency-sensitive. Fail open quickly rather
+# than holding the session hostage on memory lookup.
+RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0
+
+# Cap the wait for the first client frame after the WS handshake completes.
+# A zombie or malicious client that accepts the upgrade but never sends the
+# first response.create frame would otherwise hold a slot indefinitely and
+# starve the session registry. 60 s is generous for real clients (Codex
+# typically sends the first frame within a few hundred milliseconds of the
+# accept) but short enough to bound the damage from a hung peer.
+WS_FIRST_FRAME_TIMEOUT_SECONDS = 60.0
+
+
+def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:
+ """Best-effort decode of an OpenAI OAuth bearer token payload.
+
+ OpenClaw's Codex OAuth flow may forward only the bearer token after the
+ provider base URL is overridden. In that case the explicit
+ ``ChatGPT-Account-ID`` header can be missing even though the JWT still
+ carries the account id we need to route to the ChatGPT Codex backend.
+ """
+ auth = headers.get("authorization") or headers.get("Authorization")
+ if not auth:
+ return None
+
+ scheme, _, token = auth.partition(" ")
+ if scheme.lower() != "bearer" or token.count(".") < 2:
+ return None
+
+ payload = token.split(".", 2)[1]
+ payload += "=" * (-len(payload) % 4)
+ # Intentionally no signature verification here: this is only a best-effort
+ # routing hint extractor. Upstream still performs the actual auth/authz checks.
+ try:
+ decoded = base64.urlsafe_b64decode(payload.encode("ascii"))
+ data = json.loads(decoded.decode("utf-8"))
+ except (ValueError, UnicodeDecodeError):
+ return None
+
+ return data if isinstance(data, dict) else None
+
+
+def _resolve_codex_routing_headers(headers: dict[str, str]) -> tuple[dict[str, str], bool]:
+ """Resolve ChatGPT Codex routing hints from explicit headers or OAuth JWT."""
+ resolved = dict(headers)
+ lower_lookup = {k.lower(): k for k in resolved}
+
+ if "chatgpt-account-id" in lower_lookup:
+ return resolved, True
+
+ payload = _decode_openai_bearer_payload(resolved)
+ auth_claims = payload.get("https://api.openai.com/auth") if isinstance(payload, dict) else None
+ account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None
+ if isinstance(account_id, str) and account_id.strip():
+ resolved["ChatGPT-Account-ID"] = account_id.strip()
+ return resolved, True
+
+ return resolved, False
+
+
+class OpenAIHandlerMixin:
+ """Mixin providing OpenAI API handler methods for HeadroomProxy."""
+
+ @staticmethod
+ def _strict_previous_turn_frozen_count(
+ messages: list[dict[str, Any]],
+ base_frozen_count: int,
+ ) -> int:
+ """Freeze all prior turns in cache mode; only final user turn is mutable."""
+ if not messages:
+ return base_frozen_count
+ final_idx = len(messages) - 1
+ if messages[final_idx].get("role") == "user":
+ return max(base_frozen_count, final_idx)
+ return len(messages)
+
+ @staticmethod
+ def _restore_frozen_prefix(
+ original_messages: list[dict[str, Any]],
+ candidate_messages: list[dict[str, Any]],
+ *,
+ frozen_message_count: int,
+ ) -> tuple[list[dict[str, Any]], int]:
+ """Force frozen prefix bytes to match original request exactly."""
+ if frozen_message_count <= 0 or not original_messages:
+ return candidate_messages, 0
+
+ frozen = min(frozen_message_count, len(original_messages))
+ restored = list(candidate_messages)
+
+ if len(restored) < frozen:
+ return list(original_messages[:frozen]) + restored, frozen
+
+ changed = 0
+ for idx in range(frozen):
+ if restored[idx] != original_messages[idx]:
+ restored[idx] = original_messages[idx]
+ changed += 1
+ return restored, changed
+
+ async def handle_openai_chat(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle OpenAI /v1/chat/completions endpoint."""
+ if not hasattr(self, "pipeline_extensions"):
+ from headroom.pipeline import PipelineExtensionManager
+
+ self.pipeline_extensions = PipelineExtensionManager(discover=False)
+
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.ccr import CCRToolInjector
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS,
+ MAX_MESSAGE_ARRAY_LENGTH,
+ MAX_REQUEST_BODY_SIZE,
+ _read_request_json,
+ )
+ from headroom.proxy.modes import is_cache_mode, is_token_mode
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+ model = body.get("model", "unknown")
+ messages = body.get("messages", [])
+ original_client_messages = copy.deepcopy(messages)
+ input_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=messages,
+ tools=body.get("tools"),
+ metadata={"path": "/v1/chat/completions", "stream": body.get("stream", False)},
+ )
+ if input_event.messages is not None:
+ messages = input_event.messages
+ original_client_messages = copy.deepcopy(messages)
+ if input_event.tools is not None:
+ body["tools"] = input_event.tools
+
+ # Validate message array size
+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Message array too large ({len(messages)} messages). "
+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
+ "type": "invalid_request_error",
+ "code": "invalid_request",
+ }
+ },
+ )
+
+ stream = body.get("stream", False)
+
+ # Bypass: skip ALL compression for explicit opt-out
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ if _bypass:
+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")
+
+ # Image compression: tile alignment + ML-based technique routing
+ if self.config.image_optimize and messages and not _bypass:
+ from headroom.proxy.helpers import _get_image_compressor
+
+ compressor = _get_image_compressor()
+ if compressor and compressor.has_images(messages):
+ messages = compressor.compress(messages, provider="openai")
+ if compressor.last_result:
+ logger.info(
+ f"[{request_id}] Image: {compressor.last_result.technique.value} "
+ f"({compressor.last_result.savings_percent:.0f}% saved, "
+ f"{compressor.last_result.original_tokens} → "
+ f"{compressor.last_result.compressed_tokens} tokens)"
+ )
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ # Strip accept-encoding so httpx negotiates its own encoding.
+ # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
+ # if httpx lacks brotli support the response body is undecipherable → 502.
+ headers.pop("accept-encoding", None)
+ tags = self._extract_tags(headers)
+
+ # Memory: Get user ID when memory is enabled
+ memory_user_id: str | None = None
+ if self.memory_handler:
+ memory_user_id = headers.get(
+ "x-headroom-user-id",
+ os.environ.get("USER", os.environ.get("USERNAME", "default")),
+ )
+
+ # Rate limiting
+ if self.rate_limiter:
+ rate_key = headers.get("authorization", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider="openai")
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Check cache
+ if self.cache and not stream:
+ cached = await self.cache.get(messages, model)
+ if cached:
+ self.pipeline_extensions.emit(
+ PipelineStage.INPUT_CACHED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=messages,
+ metadata={"cache_hit": True, "path": "/v1/chat/completions"},
+ )
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0, # Savings already counted when response was cached
+ latency_ms=(time.time() - start_time) * 1000,
+ cached=True,
+ )
+
+ # Remove compression headers from cached response
+ response_headers = dict(cached.response_headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(content=cached.response_body, headers=response_headers)
+
+ # Token counting
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Hook: pre_compress
+ _hook_biases = None
+ if self.config.hooks:
+ from headroom.hooks import CompressContext
+
+ _hook_ctx = CompressContext(model=model, provider="openai")
+ try:
+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)
+ _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
+ except Exception as e:
+ logger.debug(f"[{request_id}] Hook error: {e}")
+
+ # Optimization
+ transforms_applied = []
+ pipeline_timing: dict[str, float] = {}
+ waste_signals_dict: dict[str, int] | None = None
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ # Get prefix cache tracker for this session
+ openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
+ openai_prefix_tracker = self.session_tracker_store.get_or_create(
+ openai_session_id, "openai"
+ )
+ openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
+ if is_cache_mode(self.config.mode):
+ openai_frozen_count = self._strict_previous_turn_frozen_count(
+ original_client_messages,
+ openai_frozen_count,
+ )
+
+ _compression_failed = False
+ original_messages = messages # Preserve for 400-retry fallback
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+ if self.config.optimize and messages and not _bypass and _license_ok:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+
+ if is_token_mode(self.config.mode):
+ comp_cache = self._get_compression_cache(openai_session_id)
+
+ # Zone 1: Swap cached compressed versions
+ working_messages = comp_cache.apply_cached(messages)
+
+ # Re-freeze boundary
+ openai_frozen_count = comp_cache.compute_frozen_count(messages)
+
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=working_messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(working_messages),
+ frozen_message_count=openai_frozen_count,
+ biases=_hook_biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != working_messages:
+ comp_cache.update_from_result(messages, result.messages)
+
+ # Always use pipeline result in token mode
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)
+ # so tokens_saved captures both Zone 1 + Zone 2 savings.
+ optimized_tokens = result.tokens_after
+ else:
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ frozen_message_count=openai_frozen_count,
+ biases=_hook_biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+
+ if result.waste_signals:
+ waste_signals_dict = result.waste_signals.to_dict()
+ except Exception as e:
+ logger.warning(f"Optimization failed: {e}")
+ # Flag compression failure for observability
+ _compression_failed = True
+
+ # Guard: if "optimization" inflated tokens, revert to originals
+ if optimized_tokens > original_tokens:
+ logger.warning(
+ f"[{request_id}] Optimization inflated tokens "
+ f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
+ )
+ optimized_messages = original_messages
+ optimized_tokens = original_tokens
+ transforms_applied = []
+
+ tokens_saved = original_tokens - optimized_tokens
+ optimization_latency = (time.time() - start_time) * 1000
+
+ routing_markers = summarize_routing_markers(transforms_applied)
+ if routing_markers:
+ routed_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_ROUTED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "routing_markers": routing_markers,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if routed_event.messages is not None:
+ optimized_messages = routed_event.messages
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ tokens_saved = original_tokens - optimized_tokens
+
+ compressed_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_COMPRESSED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "tokens_before": original_tokens,
+ "tokens_after": optimized_tokens,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if compressed_event.messages is not None:
+ optimized_messages = compressed_event.messages
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ tokens_saved = original_tokens - optimized_tokens
+
+ # Hook: post_compress
+ if self.config.hooks and tokens_saved > 0:
+ from headroom.hooks import CompressEvent
+
+ try:
+ self.config.hooks.post_compress(
+ CompressEvent(
+ tokens_before=original_tokens,
+ tokens_after=optimized_tokens,
+ tokens_saved=tokens_saved,
+ compression_ratio=tokens_saved / original_tokens
+ if original_tokens > 0
+ else 0,
+ transforms_applied=transforms_applied,
+ model=model,
+ provider="openai",
+ )
+ )
+ except Exception as e:
+ logger.debug(f"[{request_id}] post_compress hook error: {e}")
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = body.get("tools")
+ _original_tools = tools # Preserve for diagnostic / future retry
+ if (
+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
+ ) and not _bypass:
+ injector = CCRToolInjector(
+ provider="openai",
+ inject_tool=self.config.ccr_inject_tool,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ optimized_messages, tools, was_injected = injector.process_request(
+ optimized_messages, tools
+ )
+
+ if injector.has_compressed_content:
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
+ )
+ else:
+ logger.debug(
+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
+ )
+
+ # Query Echo: disabled — hurts prefix caching in long conversations.
+ if is_cache_mode(self.config.mode):
+ optimized_messages, restored_count = self._restore_frozen_prefix(
+ original_client_messages,
+ optimized_messages,
+ frozen_message_count=openai_frozen_count,
+ )
+ if restored_count > 0:
+ logger.warning(
+ f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
+ "to preserve cache stability (openai)"
+ )
+
+ # Memory: inject context and tools for OpenAI requests
+ memory_context_injected = False
+ memory_tools_injected = False
+ if self.memory_handler and memory_user_id:
+ try:
+ # Inject memory context (search similar memories, add as system message)
+ if self.memory_handler.config.inject_context:
+ memory_context = await self.memory_handler.search_and_format_context(
+ memory_user_id, optimized_messages
+ )
+ if memory_context:
+ # Prepend as system message for OpenAI format
+ optimized_messages = [
+ {"role": "system", "content": memory_context},
+ *optimized_messages,
+ ]
+ memory_context_injected = True
+ logger.info(
+ f"[{request_id}] Memory: Injected {len(memory_context)} chars "
+ f"of context for user {memory_user_id}"
+ )
+
+ # Inject memory tools
+ if self.memory_handler.config.inject_tools:
+ tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "openai")
+ if mem_tools_injected:
+ memory_tools_injected = True
+ logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory injection failed: {e}")
+
+ if memory_context_injected or memory_tools_injected:
+ remembered_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_REMEMBERED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ tools=tools,
+ metadata={
+ "memory_context_injected": memory_context_injected,
+ "memory_tools_injected": memory_tools_injected,
+ },
+ )
+ if remembered_event.messages is not None:
+ optimized_messages = remembered_event.messages
+ if remembered_event.tools is not None:
+ tools = remembered_event.tools
+
+ body["messages"] = optimized_messages
+ if tools is not None:
+ body["tools"] = tools
+
+ presend_event = self.pipeline_extensions.emit(
+ PipelineStage.PRE_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ tools=tools,
+ headers=headers,
+ metadata={"path": "/v1/chat/completions", "stream": stream},
+ )
+ if presend_event.messages is not None:
+ optimized_messages = presend_event.messages
+ body["messages"] = optimized_messages
+ if presend_event.tools is not None:
+ tools = presend_event.tools
+ body["tools"] = tools
+ if presend_event.headers is not None:
+ headers = presend_event.headers
+ optimized_tokens = tokenizer.count_messages(body["messages"])
+ tokens_saved = original_tokens - optimized_tokens
+
+ # Route through LiteLLM/any-llm backend if configured
+ if self.anthropic_backend is not None:
+ try:
+ if stream:
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ metadata={"path": "/v1/chat/completions", "stream": True},
+ )
+ # Streaming: use stream_openai_message() → SSE events
+ return await self._stream_openai_via_backend(
+ body,
+ headers,
+ model,
+ request_id,
+ start_time,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+ else:
+ # Non-streaming: use send_openai_message() → JSON
+ backend_response = await self.anthropic_backend.send_openai_message(
+ body, headers
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ response=backend_response.body,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": backend_response.status_code,
+ },
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ response=backend_response.body,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": backend_response.status_code,
+ },
+ )
+
+ if backend_response.error:
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+
+ # Track metrics
+ total_latency = (time.time() - start_time) * 1000
+ usage = backend_response.body.get("usage", {})
+ output_tokens = usage.get("completion_tokens", 0)
+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+
+ await self.metrics.record_request(
+ provider=self.anthropic_backend.name,
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ cached=False,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"
+ )
+
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+ except Exception as e:
+ logger.error(f"[{request_id}] Backend error: {e}")
+ return JSONResponse(
+ status_code=500,
+ content={
+ "error": {
+ "message": str(e),
+ "type": "api_error",
+ "code": "backend_error",
+ }
+ },
+ )
+
+ # Direct OpenAI API (no backend configured)
+ url = f"{self.OPENAI_API_URL}/v1/chat/completions"
+
+ try:
+ if stream:
+ # Inject stream_options to get usage stats in streaming response
+ # This allows accurate token counting instead of byte-based estimation
+ if "stream_options" not in body:
+ body["stream_options"] = {"include_usage": True}
+ elif isinstance(body.get("stream_options"), dict):
+ body["stream_options"]["include_usage"] = True
+
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ metadata={"path": "/v1/chat/completions", "stream": True},
+ )
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "openai",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ prefix_tracker=openai_prefix_tracker,
+ )
+ else:
+ headers = await apply_copilot_api_auth(headers, url=url)
+ response = await self._retry_request("POST", url, headers, body)
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ response=response,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": response.status_code,
+ },
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ response=response,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": response.status_code,
+ },
+ )
+
+ # Full diagnostic dump on upstream errors (OpenAI handler)
+ if response.status_code >= 400:
+ try:
+ err_body = response.json()
+ err_msg = err_body.get("error", {}).get("message", "")
+ err_type = err_body.get("error", {}).get("type", "")
+ except Exception:
+ err_body = {"raw": response.text[:2000]}
+ err_msg = str(response.text[:500])
+ err_type = "parse_error"
+
+ logger.warning(
+ f"[{request_id}] UPSTREAM_ERROR "
+ f"status={response.status_code} "
+ f"error_type={err_type} "
+ f"error_msg={err_msg!r} "
+ f"model={model} "
+ f"compressed={'yes' if transforms_applied else 'no'} "
+ f"transforms={transforms_applied} "
+ f"original_tokens={original_tokens} "
+ f"optimized_tokens={optimized_tokens} "
+ f"message_count={len(body.get('messages', []))} "
+ f"stream={stream}"
+ )
+
+ try:
+ from headroom import paths as _hr_paths
+
+ debug_dir = _hr_paths.debug_400_dir()
+ debug_dir.mkdir(parents=True, exist_ok=True)
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+ debug_file = debug_dir / f"{ts}_{request_id}.json"
+
+ safe_headers = {}
+ for k, v in headers.items():
+ if k.lower() in ("x-api-key", "authorization"):
+ safe_headers[k] = v[:12] + "..." if v else ""
+ else:
+ safe_headers[k] = v
+
+ debug_payload = {
+ "request_id": request_id,
+ "timestamp": datetime.now().isoformat(),
+ "status_code": response.status_code,
+ "error_response": err_body,
+ "model": model,
+ "stream": stream,
+ "headers": safe_headers,
+ "compression": {
+ "was_compressed": bool(transforms_applied),
+ "transforms": transforms_applied,
+ "original_tokens": original_tokens,
+ "optimized_tokens": optimized_tokens,
+ "tokens_saved": tokens_saved,
+ "compression_failed": _compression_failed,
+ },
+ "tools_sent": body.get("tools"),
+ "tool_count": len(body.get("tools") or []),
+ "original_tool_count": len(_original_tools or []),
+ "messages_sent": body.get("messages"),
+ "message_count": len(body.get("messages", [])),
+ "original_messages": (
+ original_messages
+ if original_messages is not body.get("messages")
+ else "__same_as_sent__"
+ ),
+ "original_message_count": len(original_messages),
+ "system_prompt": body.get("system"),
+ }
+
+ with open(debug_file, "w") as f:
+ json.dump(debug_payload, f, indent=2, default=str)
+
+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
+ except Exception as dump_err:
+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
+
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = optimized_tokens # fallback
+ output_tokens = 0
+ cache_read_tokens = 0
+ resp_json = None
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usage", {})
+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+ output_tokens = usage.get("completion_tokens", 0)
+ # OpenAI returns cached_tokens in prompt_tokens_details
+ # These are charged at 50% of the input price
+ prompt_details = usage.get("prompt_tokens_details") or {}
+ cache_read_tokens = prompt_details.get("cached_tokens", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
+ )
+
+ # Update prefix cache tracker for next turn
+ openai_prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=0, # OpenAI doesn't report write tokens
+ messages=optimized_messages,
+ )
+
+ # OpenAI has no write penalty — uncached = total - cached
+ uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(
+ model,
+ tokens_saved,
+ optimized_tokens,
+ cache_read_tokens=cache_read_tokens,
+ uncached_tokens=uncached_input_tokens,
+ )
+
+ # Memory: handle memory tool calls in OpenAI Chat Completions response.
+ # After executing tools, send a continuation request so the model
+ # can produce a final user-facing response (not just tool_calls).
+ if (
+ self.memory_handler
+ and memory_user_id
+ and resp_json
+ and response.status_code == 200
+ and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
+ ):
+ try:
+ tool_results = await self.memory_handler.handle_memory_tool_calls(
+ resp_json, memory_user_id, "openai"
+ )
+ if tool_results:
+ # Build continuation: original messages + assistant tool_calls + tool results
+ assistant_msg = resp_json.get("choices", [{}])[0].get("message", {})
+ continuation_messages = list(optimized_messages)
+ continuation_messages.append(assistant_msg)
+ continuation_messages.extend(tool_results)
+
+ continuation_body = {
+ **body,
+ "messages": continuation_messages,
+ }
+
+ cont_response = await self._retry_request(
+ "POST", url, headers, continuation_body
+ )
+ if cont_response.status_code == 200:
+ resp_json = cont_response.json()
+ response = cont_response
+
+ logger.info(
+ f"[{request_id}] Memory: Handled {len(tool_results)} "
+ f"tool call(s) with continuation for user {memory_user_id}"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory tool handling failed: {e}")
+
+ # Cache
+ if self.cache and response.status_code == 200:
+ await self.cache.set(
+ messages,
+ model,
+ response.content,
+ dict(response.headers),
+ tokens_saved,
+ )
+
+ # Capture Codex rate-limit window data from response headers
+ from headroom.subscription.codex_rate_limits import (
+ get_codex_rate_limit_state,
+ )
+
+ get_codex_rate_limit_state().update_from_headers(dict(response.headers))
+
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ waste_signals=waste_signals_dict,
+ cache_read_tokens=cache_read_tokens,
+ uncached_input_tokens=uncached_input_tokens,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens)"
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Inject Headroom compression metrics (for SaaS metering)
+ response_headers["x-headroom-tokens-before"] = str(original_tokens)
+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
+ response_headers["x-headroom-model"] = model
+ if transforms_applied:
+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
+ if cache_read_tokens > 0:
+ response_headers["x-headroom-cached"] = "true"
+ if _compression_failed:
+ response_headers["x-headroom-compression-failed"] = "true"
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed(provider="openai")
+ # Log full error details internally for debugging
+ logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
+ # Return sanitized error message to client (don't expose internal details)
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "type": "server_error",
+ "code": "proxy_error",
+ }
+ },
+ )
+
+ async def handle_openai_responses(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle OpenAI /v1/responses endpoint (new Responses API).
+
+ The Responses API differs from /v1/chat/completions:
+ - Input: `input` (string or array) instead of `messages`
+ - System: `instructions` instead of system message
+ - Output: `output[]` array instead of `choices[].message`
+ - State: `previous_response_id` for multi-turn
+ - Built-in tools: web_search, file_search, code_interpreter
+ """
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS,
+ MAX_REQUEST_BODY_SIZE,
+ _read_request_json,
+ )
+ from headroom.proxy.responses_converter import (
+ messages_to_responses_items,
+ responses_items_to_messages,
+ )
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+
+ model = body.get("model", "unknown")
+ stream = body.get("stream", False)
+
+ # Convert Responses API input to messages format for optimization.
+ # The Responses API uses a different item model (function_call,
+ # function_call_output, reasoning as top-level items) — we convert to
+ # Chat Completions messages for the pipeline, then convert back.
+
+ input_data = body.get("input", "")
+ instructions = body.get("instructions")
+ previous_response_id = body.get("previous_response_id")
+
+ messages: list[dict[str, Any]] = []
+ original_items: list[dict[str, Any]] | None = None
+ preserved_indices: list[int] = []
+
+ if instructions:
+ messages.append({"role": "system", "content": instructions})
+
+ if isinstance(input_data, str):
+ messages.append({"role": "user", "content": input_data})
+ elif isinstance(input_data, list):
+ original_items = input_data
+ converted, preserved_indices = responses_items_to_messages(input_data)
+ messages.extend(converted)
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ # Strip accept-encoding so httpx negotiates its own encoding.
+ # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
+ # if httpx lacks brotli support the response body is undecipherable → 502.
+ headers.pop("accept-encoding", None)
+ tags = self._extract_tags(headers)
+
+ # Memory: Get user ID when memory is enabled
+ memory_user_id: str | None = None
+ if self.memory_handler:
+ memory_user_id = headers.get(
+ "x-headroom-user-id",
+ os.environ.get("USER", os.environ.get("USERNAME", "default")),
+ )
+
+ # Rate limiting
+ if self.rate_limiter:
+ rate_key = headers.get("authorization", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider="openai")
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Token counting on converted messages
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Optimize: convert items → compress → convert back
+ tokens_saved = 0
+ transforms_applied: list[str] = []
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ _should_compress = (
+ self.config.optimize
+ and original_items is not None
+ and not previous_response_id
+ and not _bypass
+ and len(messages) > 1
+ )
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+
+ if _should_compress and _license_ok:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+ except Exception as e:
+ logger.warning(f"[{request_id}] Responses API optimization failed: {e}")
+
+ # Guard: if "optimization" inflated tokens, revert to originals
+ if optimized_tokens > original_tokens:
+ logger.warning(
+ f"[{request_id}] Optimization inflated tokens "
+ f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
+ )
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+ transforms_applied = []
+
+ tokens_saved = original_tokens - optimized_tokens
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Convert compressed messages back to Responses API items
+ if optimized_messages is not messages and original_items is not None:
+ opt_msgs = optimized_messages
+ # Strip system message (instructions) — it's separate in Responses API
+ if instructions and opt_msgs and opt_msgs[0].get("role") == "system":
+ body["instructions"] = opt_msgs[0]["content"]
+ opt_msgs = opt_msgs[1:]
+
+ body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)
+
+ # Memory: inject context and tools for Responses API requests
+ if self.memory_handler and memory_user_id:
+ try:
+ # Inject memory context into instructions (Responses API system prompt)
+ if self.memory_handler.config.inject_context:
+ try:
+ memory_context = await asyncio.wait_for(
+ self.memory_handler.search_and_format_context(
+ memory_user_id, optimized_messages
+ ),
+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ memory_context = None
+ logger.info(
+ f"[{request_id}] Memory context lookup exceeded "
+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"
+ )
+ if memory_context:
+ existing_instructions = body.get("instructions") or ""
+ if existing_instructions:
+ body["instructions"] = f"{existing_instructions}\n\n{memory_context}"
+ else:
+ body["instructions"] = memory_context
+ logger.info(
+ f"[{request_id}] Memory: Injected {len(memory_context)} chars "
+ f"of context into instructions for user {memory_user_id}"
+ )
+
+ # Inject memory tools (Responses API format)
+ if self.memory_handler.config.inject_tools:
+ resp_tools = body.get("tools") or []
+ resp_tools, mem_tools_injected = self.memory_handler.inject_tools(
+ resp_tools, "openai"
+ )
+ if mem_tools_injected:
+ # Convert Chat Completions format to Responses API format
+ converted_tools = []
+ for t in resp_tools:
+ if t.get("type") == "function" and "function" in t:
+ fn = t["function"]
+ converted_tools.append(
+ {
+ "type": "function",
+ "name": fn.get("name"),
+ "description": fn.get("description", ""),
+ "parameters": fn.get("parameters", {}),
+ }
+ )
+ else:
+ converted_tools.append(t)
+ body["tools"] = converted_tools
+ logger.info(
+ f"[{request_id}] Memory: Injected memory tools (openai/responses)"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory injection failed (responses): {e}")
+
+ # /v1/responses is OpenAI-specific (Codex) — always routes direct.
+ # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.
+ if self.anthropic_backend is not None:
+ logger.debug(
+ f"[{request_id}] /v1/responses always routes to OpenAI direct "
+ f"(backend '{self.anthropic_backend.name}' not used for Responses API)"
+ )
+
+ headers, is_chatgpt_auth = _resolve_codex_routing_headers(headers)
+
+ # Route to correct endpoint based on auth mode.
+ # ChatGPT session auth (codex login) uses chatgpt.com, not api.openai.com.
+ if is_chatgpt_auth:
+ url = "https://chatgpt.com/backend-api/codex/responses"
+ else:
+ url = f"{self.OPENAI_API_URL}/v1/responses"
+
+ try:
+ if stream:
+ # Streaming for Responses API uses semantic events
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "openai",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ memory_user_id=memory_user_id,
+ )
+ else:
+ headers = await apply_copilot_api_auth(headers, url=url)
+ response = await self._retry_request("POST", url, headers, body)
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = original_tokens # fallback
+ output_tokens = 0
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usage", {})
+ total_input_tokens = usage.get("input_tokens", original_tokens)
+ output_tokens = usage.get("output_tokens", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"
+ )
+
+ # Memory: handle memory tool calls in Responses API response
+ if (
+ self.memory_handler
+ and memory_user_id
+ and resp_json
+ and response.status_code == 200
+ and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
+ ):
+ try:
+ # Extract function_call items from output
+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
+
+ output_items = resp_json.get("output", [])
+ memory_fc_items = [
+ item
+ for item in output_items
+ if isinstance(item, dict)
+ and item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ]
+
+ # Execute memory tool calls
+ tool_outputs: list[dict[str, Any]] = []
+ for fc in memory_fc_items:
+ call_id = fc.get("call_id", fc.get("id", ""))
+ name = fc.get("name", "")
+ args_str = fc.get("arguments", "{}")
+ try:
+ args = json.loads(args_str)
+ except json.JSONDecodeError:
+ args = {}
+
+ await self.memory_handler._ensure_initialized()
+ if self.memory_handler._backend:
+ result = await self.memory_handler._execute_memory_tool(
+ name, args, memory_user_id, "openai"
+ )
+ else:
+ result = json.dumps({"error": "Memory backend not initialized"})
+
+ tool_outputs.append(
+ {
+ "type": "function_call_output",
+ "call_id": call_id,
+ "output": result,
+ }
+ )
+
+ if tool_outputs:
+ # Make continuation request with tool results
+ response_id = resp_json.get("id")
+ continuation_body = {
+ "model": model,
+ "input": tool_outputs,
+ }
+ if response_id:
+ continuation_body["previous_response_id"] = response_id
+ existing_tools = body.get("tools")
+ if existing_tools:
+ continuation_body["tools"] = existing_tools
+
+ cont_response = await self._retry_request(
+ "POST", url, headers, continuation_body
+ )
+ resp_json = cont_response.json()
+ response = cont_response
+ logger.info(
+ f"[{request_id}] Memory: Handled {len(tool_outputs)} "
+ f"tool call(s) with continuation for user {memory_user_id} (responses)"
+ )
+ except Exception as e:
+ logger.warning(
+ f"[{request_id}] Memory tool handling failed (responses): {e}"
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)
+
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ )
+
+ logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")
+
+ # Capture Codex rate-limit window data from response headers
+ from headroom.subscription.codex_rate_limits import (
+ get_codex_rate_limit_state,
+ )
+
+ get_codex_rate_limit_state().update_from_headers(dict(response.headers))
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed(provider="openai")
+ logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "type": "server_error",
+ "code": "proxy_error",
+ }
+ },
+ )
+
+ async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:
+ """WebSocket proxy for /v1/responses (Codex gpt-5.4+).
+
+ Newer Codex versions use WebSocket instead of HTTP POST for the
+ Responses API. This handler:
+ 1. Accepts the client WebSocket
+ 2. Receives the first message (``response.create`` request)
+ 3. Compresses the ``input`` array using the existing pipeline
+ 4. Opens an upstream WebSocket to OpenAI
+ 5. Sends the compressed request upstream
+ 6. Relays all subsequent messages bidirectionally
+ """
+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ try:
+ import websockets
+ except ImportError:
+ await websocket.accept()
+ await websocket.close(
+ code=1011,
+ reason="websockets package not installed. pip install websockets",
+ )
+ return
+
+ request_id = await self._next_request_id()
+ session_id = uuid.uuid4().hex
+
+ # Stage-timer — captures per-stage durations for the structured
+ # log emitted on session close. Unit 2 instrumentation.
+ stage_timer = StageTimer()
+ session_started_at = time.perf_counter()
+
+ # Unit 3: initialize registry variables *before* accept so the
+ # outermost ``finally`` can rely on them existing even if
+ # registration itself fails for some reason.
+ ws_sessions: WebSocketSessionRegistry | None = getattr(self, "ws_sessions", None)
+ session_handle: WSSessionHandle | None = None
+ termination_cause: TerminationCause = "unknown"
+
+ # Forward client headers to upstream, adding required OpenAI-Beta header
+ ws_headers = dict(websocket.headers)
+
+ # Extract subprotocol from client — this is an application-level negotiation
+ # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
+ # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.
+ client_subprotocols: list[str] = []
+ raw_protocol = ws_headers.get("sec-websocket-protocol", "")
+ if raw_protocol:
+ client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]
+
+ # Accept client connection with the requested subprotocol
+ async with stage_timer.measure("accept"):
+ if client_subprotocols:
+ await websocket.accept(subprotocol=client_subprotocols[0])
+ else:
+ await websocket.accept()
+
+ # --- Unit 3: register the session as soon as accept succeeds ---
+ client_addr: str | None = None
+ client_info = getattr(websocket, "client", None)
+ if client_info is not None:
+ host = getattr(client_info, "host", None)
+ port = getattr(client_info, "port", None)
+ if host is not None and port is not None:
+ client_addr = f"{host}:{port}"
+ elif host is not None:
+ client_addr = str(host)
+ if ws_sessions is not None:
+ session_handle = WSSessionHandle(
+ session_id=session_id,
+ request_id=request_id,
+ client_addr=client_addr,
+ upstream_url=None, # set below once upstream_url is computed
+ )
+ ws_sessions.register(session_handle)
+ metrics = getattr(self, "metrics", None)
+ if metrics is not None and hasattr(metrics, "inc_active_ws_sessions"):
+ try:
+ metrics.inc_active_ws_sessions()
+ except Exception: # pragma: no cover - defensive
+ pass
+
+ # Forward all client headers except hop-by-hop / per-connection headers.
+ # These are WebSocket handshake mechanics that the `websockets` library
+ # generates fresh for the upstream connection — forwarding them would conflict.
+ # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.
+ _skip_headers = frozenset(
+ {
+ "host", # must match upstream, not local proxy
+ "connection", # hop-by-hop
+ "upgrade", # hop-by-hop
+ "sec-websocket-key", # per-connection cryptographic nonce
+ "sec-websocket-version", # protocol version (websockets lib sets this)
+ "sec-websocket-extensions", # per-connection negotiation
+ "sec-websocket-accept", # server-side only
+ "sec-websocket-protocol", # handled via subprotocols param below
+ "content-length", # hop-by-hop
+ "transfer-encoding", # hop-by-hop
+ }
+ )
+ upstream_headers: dict[str, str] = {}
+ for k, v in ws_headers.items():
+ if k.lower() not in _skip_headers:
+ upstream_headers[k] = v
+
+ upstream_headers, is_chatgpt_auth = _resolve_codex_routing_headers(upstream_headers)
+ _lower_headers = {k.lower(): v for k, v in upstream_headers.items()}
+
+ # Build upstream WebSocket URL based on auth mode
+ if is_chatgpt_auth:
+ # ChatGPT session auth → route to chatgpt.com backend
+ upstream_url = "wss://chatgpt.com/backend-api/codex/responses"
+ logger.debug(
+ f"[{request_id}] WS: ChatGPT session auth detected, routing to chatgpt.com"
+ )
+ else:
+ # API key auth → route to configured OpenAI API URL
+ base = self.OPENAI_API_URL
+ ws_base = base.replace("https://", "wss://").replace("http://", "ws://")
+ upstream_url = f"{ws_base}/v1/responses"
+
+ # Unit 3: attach the resolved upstream URL to the session handle.
+ if session_handle is not None:
+ session_handle.upstream_url = upstream_url
+
+ # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.
+ # Safety net for clients that don't forward auth headers via WebSocket upgrade.
+ if "authorization" not in _lower_headers:
+ api_key = os.environ.get("OPENAI_API_KEY")
+ if api_key:
+ upstream_headers["Authorization"] = f"Bearer {api_key}"
+ logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")
+ else:
+ logger.warning(
+ f"[{request_id}] WS: no Authorization header from client and "
+ f"OPENAI_API_KEY not set — upstream will likely reject"
+ )
+
+ upstream_headers = await apply_copilot_api_auth(upstream_headers, url=upstream_url)
+
+ # Ensure the required beta header is present — OpenAI returns 500 without it.
+ # Codex sends `responses_websockets=2026-02-06`; only inject if missing entirely.
+ if "openai-beta" not in _lower_headers:
+ upstream_headers["OpenAI-Beta"] = "responses_websockets=2026-02-06"
+
+ logger.debug(
+ f"[{request_id}] WS upstream headers: "
+ f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "
+ f"subprotocols={client_subprotocols}"
+ )
+
+ try:
+ # Receive the first message from client (the response.create request).
+ # Bound the wait with WS_FIRST_FRAME_TIMEOUT_SECONDS so a zombie
+ # client that opens the WS but never sends a frame cannot hold a
+ # session slot indefinitely. The StageTimer measurement still
+ # captures the elapsed time up to the timeout so operators can
+ # see the slow-client pattern in the stage-timings log.
+ try:
+ async with stage_timer.measure("first_client_frame"):
+ first_msg_raw = await asyncio.wait_for(
+ websocket.receive_text(),
+ timeout=WS_FIRST_FRAME_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ logger.info(
+ f"[{request_id}] WS first-frame timeout after "
+ f"{WS_FIRST_FRAME_TIMEOUT_SECONDS:.0f}s; closing session "
+ f"{session_id} (no client data)"
+ )
+ termination_cause = "client_timeout"
+ with contextlib.suppress(Exception):
+ # 1001 (going away): server is cleanly terminating a slow
+ # client, not an internal error.
+ await websocket.close(code=1001, reason="first-frame timeout")
+ # Exit the outer try so the session-lifecycle ``finally`` runs
+ # deregister / metrics / stage-timings emission as usual.
+ return
+
+ # --- Optional: compress the input in the first message ---
+ body: dict[str, Any] = {}
+ try:
+ body = json.loads(first_msg_raw)
+ tokens_saved = 0
+ ws_request_body = body.get("response", body)
+ input_data = (
+ ws_request_body.get("input") if isinstance(ws_request_body, dict) else None
+ )
+
+ should_compress = (
+ self.config.optimize
+ and isinstance(input_data, list)
+ and len(input_data) > 1
+ and not (
+ ws_request_body.get("previous_response_id")
+ if isinstance(ws_request_body, dict)
+ else None
+ )
+ )
+ if should_compress:
+ try:
+ from headroom.proxy.responses_converter import (
+ messages_to_responses_items,
+ responses_items_to_messages,
+ )
+
+ model = ws_request_body.get("model", "gpt-4o")
+ converted, preserved = responses_items_to_messages(input_data)
+
+ messages: list[dict[str, Any]] = []
+ instructions = ws_request_body.get("instructions")
+ if instructions:
+ messages.append({"role": "system", "content": instructions})
+ messages.extend(converted)
+
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ context_limit = self.openai_provider.get_context_limit(model)
+ async with stage_timer.measure("compression"):
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ opt = result.messages
+ if instructions and opt and opt[0].get("role") == "system":
+ ws_request_body["instructions"] = opt[0]["content"]
+ opt = opt[1:]
+ if result.tokens_after <= original_tokens:
+ ws_request_body["input"] = messages_to_responses_items(
+ opt, input_data, preserved
+ )
+ else:
+ logger.warning(
+ f"[{request_id}] WS optimization inflated tokens "
+ f"({original_tokens} -> {result.tokens_after}), reverting"
+ )
+ tokens_saved = max(0, original_tokens - result.tokens_after)
+ if "response" in body and isinstance(body["response"], dict):
+ body["response"] = ws_request_body
+ else:
+ body = ws_request_body
+ first_msg_raw = json.dumps(body)
+ logger.info(
+ f"[{request_id}] WS /v1/responses compressed: "
+ f"saved {tokens_saved} tokens"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] WS compression failed: {e}")
+
+ except json.JSONDecodeError:
+ # Not JSON — pass through as-is
+ tokens_saved = 0
+
+ # --- Memory: inject context, tools, and instructions ---
+ memory_user_id: str | None = None
+ if self.memory_handler and body:
+ memory_user_id = ws_headers.get(
+ "x-headroom-user-id",
+ os.environ.get("USER", os.environ.get("USERNAME", "default")),
+ )
+ try:
+ # Unwrap response.create envelope to access the response body
+ ws_response_body = body.get("response", body)
+
+ # Debug: log what Codex sends so we can see the full tool list
+ existing_tool_names = [
+ t.get("name") or t.get("function", {}).get("name", "?")
+ for t in (ws_response_body.get("tools") or [])
+ ]
+ instr_preview = (ws_response_body.get("instructions") or "")[:200]
+ logger.info(
+ f"[{request_id}] WS Memory: Codex tools={existing_tool_names}, "
+ f"instructions_len={len(ws_response_body.get('instructions') or '')}, "
+ f"instructions_preview={instr_preview!r}"
+ )
+
+ # Inject memory context into instructions
+ if self.memory_handler.config.inject_context:
+ ws_input = ws_response_body.get("input", "")
+ ws_instructions = ws_response_body.get("instructions")
+ ws_msgs: list[dict[str, Any]] = []
+ if ws_instructions:
+ ws_msgs.append({"role": "system", "content": ws_instructions})
+ if isinstance(ws_input, str) and ws_input:
+ ws_msgs.append({"role": "user", "content": ws_input})
+ elif isinstance(ws_input, list):
+ from headroom.proxy.responses_converter import (
+ responses_items_to_messages,
+ )
+
+ converted_msgs, _ = responses_items_to_messages(ws_input)
+ ws_msgs.extend(converted_msgs)
+
+ try:
+ async with stage_timer.measure("memory_context"):
+ memory_context = await asyncio.wait_for(
+ self.memory_handler.search_and_format_context(
+ memory_user_id, ws_msgs
+ ),
+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ memory_context = None
+ logger.info(
+ f"[{request_id}] WS Memory: Context lookup exceeded "
+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "
+ f"continuing without it"
+ )
+ if memory_context:
+ existing = ws_response_body.get("instructions") or ""
+ if existing:
+ ws_response_body["instructions"] = f"{existing}\n\n{memory_context}"
+ else:
+ ws_response_body["instructions"] = memory_context
+ logger.info(
+ f"[{request_id}] WS Memory: Injected {len(memory_context)} chars "
+ f"of context into instructions"
+ )
+
+ # Inject memory tools (Responses API format)
+ if self.memory_handler.config.inject_tools:
+ ws_tools = ws_response_body.get("tools") or []
+ ws_tools, mem_injected = self.memory_handler.inject_tools(
+ ws_tools, "openai"
+ )
+ if mem_injected:
+ converted_tools = []
+ for t in ws_tools:
+ if t.get("type") == "function" and "function" in t:
+ fn = t["function"]
+ converted_tools.append(
+ {
+ "type": "function",
+ "name": fn.get("name"),
+ "description": fn.get("description", ""),
+ "parameters": fn.get("parameters", {}),
+ }
+ )
+ else:
+ converted_tools.append(t)
+ ws_response_body["tools"] = converted_tools
+
+ # Add memory instruction so the model uses
+ # memory tools as persistent cross-session knowledge.
+ mem_instruction = (
+ "\n\n## Memory\n"
+ "You have persistent memory via memory_search and "
+ "memory_save tools. Memory stores knowledge across "
+ "sessions — user info, project details, org context, "
+ "decisions, architecture, conventions, anything worth "
+ "remembering.\n\n"
+ "- ALWAYS call memory_search BEFORE searching files "
+ "when the user asks a question that could be answered "
+ "from prior knowledge.\n"
+ "- Call memory_save to store important facts, decisions, "
+ "or context that would be useful in future sessions.\n"
+ "- Memory is your first source of truth for anything "
+ "not visible in the current conversation."
+ )
+ existing_instr = ws_response_body.get("instructions") or ""
+ ws_response_body["instructions"] = existing_instr + mem_instruction
+ logger.info(
+ f"[{request_id}] WS Memory: Injected memory tools + instruction"
+ )
+
+ # Write back into envelope if it was wrapped
+ if "response" in body and isinstance(body["response"], dict):
+ body["response"] = ws_response_body
+ else:
+ body = ws_response_body
+
+ first_msg_raw = json.dumps(body)
+ except Exception as e:
+ logger.warning(f"[{request_id}] WS Memory injection failed: {e}")
+
+ # --- Connect to upstream OpenAI WebSocket ---
+ logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")
+
+ # Use ssl=True to let the websockets library handle SSL natively.
+ # Manual ssl.create_default_context() + certifi doesn't load the
+ # Windows system cert store, causing HTTP 500 on wss:// connections.
+ use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
+
+ ws_connected = False
+ ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ ws_last_err: Exception | None = None
+ _upstream_connect_started = time.perf_counter()
+ _upstream_connect_recorded = False
+ _upstream_first_event_started: float | None = None
+
+ for ws_attempt in range(ws_connect_attempts):
+ try:
+ async with websockets.connect(
+ upstream_url,
+ additional_headers=upstream_headers,
+ subprotocols=(
+ [websockets.Subprotocol(p) for p in client_subprotocols]
+ if client_subprotocols and hasattr(websockets, "Subprotocol")
+ else client_subprotocols or None
+ ),
+ ssl=use_ssl,
+ open_timeout=max(30, self.config.connect_timeout_seconds * 3),
+ close_timeout=10,
+ ping_interval=20,
+ ping_timeout=20,
+ ) as upstream:
+ ws_connected = True
+ if not _upstream_connect_recorded:
+ stage_timer.record(
+ "upstream_connect",
+ (time.perf_counter() - _upstream_connect_started) * 1000.0,
+ )
+ _upstream_connect_recorded = True
+ _upstream_first_event_started = time.perf_counter()
+ await upstream.send(first_msg_raw)
+
+ # Unit 3: flag the upstream side flips on seeing
+ # ``response.completed`` so the outer cause
+ # classifier can prefer it over the raw
+ # "upstream iterator ended" default.
+ response_completed_seen = False
+ # Captures the first exception surfaced by the
+ # inner relay ``except`` blocks so the outer
+ # classifier can still tell ``upstream_error``
+ # from ``upstream_disconnect`` / ``response_completed``
+ # even though the halves swallow and log.
+ upstream_relay_error: BaseException | None = None
+ client_relay_error: BaseException | None = None
+
+ async def _client_to_upstream() -> None:
+ nonlocal client_relay_error
+ try:
+ while True:
+ msg = await websocket.receive_text()
+ await upstream.send(msg)
+ except asyncio.CancelledError:
+ # Explicit cancel from the outer
+ # orchestrator — re-raise so
+ # ``t.cancelled()`` and ``t.exception()``
+ # behave correctly in the caller.
+ raise
+ except Exception as relay_err:
+ # Surface real errors to the classifier
+ # without re-raising (existing fork
+ # behavior: log and return so the
+ # partner task can be cancelled
+ # deterministically).
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ client_relay_error = relay_err
+ logger.debug(
+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"
+ )
+ with contextlib.suppress(Exception):
+ await upstream.close()
+
+ async def _upstream_to_client() -> None:
+ """Relay upstream→client with transparent memory tool handling.
+
+ Uses a buffer-then-decide approach:
+ 1. Buffer events until first output item arrives
+ 2. If first output is a memory tool → suppress entire response,
+ execute tools silently, send continuation upstream
+ 3. If first output is non-memory → flush buffer, stream normally
+ 4. Continuation response events are relayed to Codex seamlessly
+
+ This prevents orphaned response.created events from confusing Codex.
+ """
+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
+
+ # Unit 3: surface response.completed observation
+ # to the outer scope so the termination-cause
+ # classifier can prefer ``response_completed``
+ # over ``upstream_disconnect``.
+ nonlocal response_completed_seen
+ nonlocal upstream_relay_error
+
+ memory_enabled = bool(self.memory_handler and memory_user_id)
+
+ # Per-response state (reset after each response.completed)
+ event_buffer: list[str] = []
+ decided = False
+ suppress_response = False
+ pending_fcs: list[dict[str, Any]] = []
+ resp_id: str | None = None
+
+ def _reset() -> None:
+ nonlocal decided, suppress_response, resp_id
+ event_buffer.clear()
+ decided = False
+ suppress_response = False
+ pending_fcs.clear()
+ resp_id = None
+
+ # The retry-loop variable is safe to close over here:
+ # ``_upstream_to_client`` is defined and awaited within
+ # a single iteration and never escapes.
+ _first_event_started_at = _upstream_first_event_started # noqa: B023
+
+ try:
+ async for msg in upstream:
+ if (
+ _first_event_started_at is not None
+ and "upstream_first_event" not in stage_timer
+ ):
+ stage_timer.record(
+ "upstream_first_event",
+ (time.perf_counter() - _first_event_started_at)
+ * 1000.0,
+ )
+ if isinstance(msg, bytes):
+ await websocket.send_bytes(msg)
+ continue
+ msg_str = msg if isinstance(msg, str) else str(msg)
+
+ if not memory_enabled:
+ await websocket.send_text(msg_str)
+ continue
+
+ # Parse event
+ try:
+ event = json.loads(msg_str)
+ except (json.JSONDecodeError, TypeError):
+ await websocket.send_text(msg_str)
+ continue
+
+ event_type = event.get("type", "")
+
+ # --- Phase 1: Buffer until first output item ---
+ if not decided:
+ event_buffer.append(msg_str)
+
+ if event_type == "response.output_item.added":
+ item = event.get("item", {})
+ if (
+ item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ):
+ # Memory tool first → suppress entire response
+ suppress_response = True
+ decided = True
+ event_buffer.clear()
+ logger.info(
+ f"[{request_id}] WS Memory: Detected "
+ f"{item.get('name')} — suppressing response"
+ )
+ else:
+ # Non-memory first → flush buffer, pass through
+ decided = True
+ for buf in event_buffer:
+ await websocket.send_text(buf)
+ event_buffer.clear()
+
+ elif event_type == "response.completed":
+ # No output items at all — flush
+ decided = True
+ for buf in event_buffer:
+ await websocket.send_text(buf)
+ event_buffer.clear()
+ _reset()
+ response_completed_seen = True
+
+ continue
+
+ # --- Phase 2a: Suppress mode (memory response) ---
+ if suppress_response:
+ if event_type == "response.output_item.done":
+ item = event.get("item", {})
+ if (
+ item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ):
+ pending_fcs.append(item)
+
+ elif event_type == "response.completed":
+ response_completed_seen = True
+ resp = event.get("response", {})
+ resp_id = resp.get("id")
+
+ if pending_fcs:
+ logger.info(
+ f"[{request_id}] WS Memory: Executing "
+ f"{len(pending_fcs)} tool(s) transparently"
+ )
+
+ # Execute memory tool calls
+ tool_outputs: list[dict[str, Any]] = []
+ for fc in pending_fcs:
+ call_id = fc.get("call_id", fc.get("id", ""))
+ fc_name = fc.get("name", "")
+ args_str = fc.get("arguments", "{}")
+ try:
+ fc_args = json.loads(args_str)
+ except json.JSONDecodeError:
+ fc_args = {}
+
+ await self.memory_handler._ensure_initialized()
+ if self.memory_handler._backend:
+ result = await self.memory_handler._execute_memory_tool(
+ fc_name,
+ fc_args,
+ memory_user_id,
+ "openai",
+ )
+ else:
+ result = json.dumps(
+ {"error": "backend not ready"}
+ )
+
+ tool_outputs.append(
+ {
+ "type": "function_call_output",
+ "call_id": call_id,
+ "output": result,
+ }
+ )
+ logger.info(
+ f"[{request_id}] WS Memory: Executed "
+ f"{fc_name} for user {memory_user_id}"
+ )
+
+ # Send continuation upstream
+ cont: dict[str, Any] = {
+ "type": "response.create",
+ "response": {"input": tool_outputs},
+ }
+ if resp_id:
+ cont["response"]["previous_response_id"] = (
+ resp_id
+ )
+ await upstream.send(json.dumps(cont))
+ logger.info(
+ f"[{request_id}] WS Memory: Sent continuation "
+ f"with {len(tool_outputs)} result(s)"
+ )
+
+ _reset()
+ # All events suppressed in this mode
+ continue
+
+ # --- Phase 2b: Pass-through mode ---
+ await websocket.send_text(msg_str)
+
+ except asyncio.CancelledError:
+ raise
+ except Exception as relay_err:
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ # Capture for the outer classifier
+ # so ``upstream_error`` can be
+ # distinguished from a clean
+ # upstream disconnect.
+ upstream_relay_error = relay_err
+ logger.debug(
+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"
+ )
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close()
+
+ # --- Unit 3: deterministic relay-task cancellation ---
+ # Spawn each half as a named task so we can:
+ # (a) attach them to the session registry for
+ # ``/debug/ws-sessions``,
+ # (b) cancel the survivor explicitly when the
+ # first one exits, and
+ # (c) classify the termination cause for the
+ # duration histogram.
+ client_task = asyncio.create_task(
+ _client_to_upstream(),
+ name=f"codex-ws-c2u-{session_id}",
+ )
+ upstream_task = asyncio.create_task(
+ _upstream_to_client(),
+ name=f"codex-ws-u2c-{session_id}",
+ )
+ relay_tasks = [client_task, upstream_task]
+ if ws_sessions is not None:
+ ws_sessions.attach_tasks(session_id, relay_tasks)
+ metrics_for_tasks = getattr(self, "metrics", None)
+ if metrics_for_tasks is not None and hasattr(
+ metrics_for_tasks, "inc_active_relay_tasks"
+ ):
+ try:
+ metrics_for_tasks.inc_active_relay_tasks(len(relay_tasks))
+ except Exception: # pragma: no cover - defensive
+ pass
+
+ try:
+ done, pending = await asyncio.wait(
+ {client_task, upstream_task},
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ # Cancel the survivor so we don't leak the
+ # partner task. Suppress the CancelledError
+ # we just raised ourselves — any *other*
+ # exception from the cancelled task is
+ # already logged inside its own try/except.
+ for t in pending:
+ t.cancel()
+ if pending:
+ with contextlib.suppress(asyncio.CancelledError):
+ await asyncio.gather(*pending, return_exceptions=True)
+
+ # Classify termination cause from whichever
+ # task completed first. ``CancelledError``
+ # can show up on the "done" side if the
+ # handler itself was cancelled from outside
+ # (e.g. server shutdown).
+ for t in done:
+ exc = None
+ # Cancelled tasks raise CancelledError from
+ # .exception(); surface it explicitly so the
+ # downstream ``isinstance(exc, CancelledError)``
+ # branches actually run. For any other
+ # unexpected state (``InvalidStateError`` if
+ # the task somehow isn't done — shouldn't
+ # happen post-gather but defensive), we
+ # suppress and leave ``exc`` as ``None``.
+ if t.cancelled():
+ exc = asyncio.CancelledError()
+ else:
+ with contextlib.suppress(asyncio.InvalidStateError):
+ exc = t.exception()
+ task_name = t.get_name() or ""
+ if t is client_task:
+ if client_relay_error is not None:
+ termination_cause = "client_error"
+ elif exc is None:
+ termination_cause = "client_disconnect"
+ elif isinstance(exc, asyncio.CancelledError):
+ termination_cause = "client_disconnect"
+ else:
+ # Distinguish legitimate client
+ # disconnect exceptions from
+ # real errors: WebSocketDisconnect
+ # is a normal client exit.
+ if "WebSocketDisconnect" in type(exc).__name__:
+ termination_cause = "client_disconnect"
+ else:
+ termination_cause = "client_error"
+ elif t is upstream_task:
+ if upstream_relay_error is not None:
+ termination_cause = "upstream_error"
+ logger.debug(
+ f"[{request_id}] WS relay {task_name} "
+ f"raised: {upstream_relay_error!r}"
+ )
+ elif exc is None:
+ termination_cause = (
+ "response_completed"
+ if response_completed_seen
+ else "upstream_disconnect"
+ )
+ elif isinstance(exc, asyncio.CancelledError):
+ termination_cause = "upstream_disconnect"
+ else:
+ termination_cause = "upstream_error"
+ logger.debug(
+ f"[{request_id}] WS relay {task_name} raised: {exc!r}"
+ )
+ finally:
+ # In case anything above raised before the
+ # cancel-and-await loop ran.
+ for t in relay_tasks:
+ if not t.done():
+ t.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await asyncio.gather(*relay_tasks, return_exceptions=True)
+
+ logger.info(
+ f"[{request_id}] WS /v1/responses completed "
+ f"(tokens_saved={tokens_saved}, cause={termination_cause})"
+ )
+ break
+ except Exception as ws_err:
+ if ws_connected:
+ # WS was established but broke mid-stream — re-raise
+ raise
+
+ ws_last_err = ws_err
+ if ws_attempt >= ws_connect_attempts - 1:
+ break
+
+ delay_with_jitter = jitter_delay_ms(
+ self.config.retry_base_delay_ms,
+ self.config.retry_max_delay_ms,
+ ws_attempt,
+ )
+ logger.warning(
+ f"[{request_id}] WS upstream connect failed "
+ f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ if not ws_connected:
+ # WS upgrade failed (HTTP 500 from OpenAI is common).
+ # Fall back to HTTP POST streaming and relay SSE events
+ # back over the client WebSocket transparently.
+ ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")
+ _ws_detail = str(ws_err)
+ if hasattr(ws_err, "response"):
+ resp_body = getattr(getattr(ws_err, "response", None), "body", b"")
+ if resp_body:
+ _ws_detail += f" | {resp_body[:300].decode('utf-8', errors='replace')}"
+ logger.warning(
+ f"[{request_id}] WS upstream failed ({_ws_detail}), "
+ f"falling back to HTTP POST streaming"
+ )
+ await self._ws_http_fallback(
+ websocket, body, first_msg_raw, upstream_headers, request_id
+ )
+
+ # Record metrics
+ if tokens_saved > 0:
+ model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"
+ await self.metrics.record_request(
+ provider="openai",
+ model=model_name,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=tokens_saved,
+ latency_ms=0,
+ )
+
+ except Exception as e:
+ if "WebSocketDisconnect" in type(e).__name__:
+ # Unit 3: client dropped the socket before or during
+ # relay. The registry classifier may already have set
+ # ``client_disconnect`` via the relay task exit path;
+ # preserve that, otherwise set it here.
+ if termination_cause == "unknown":
+ termination_cause = "client_disconnect"
+ else:
+ # Extract response body from websockets InvalidStatus for better debugging
+ error_detail = str(e)
+ if hasattr(e, "response"):
+ try:
+ resp = e.response
+ body_bytes = getattr(resp, "body", None) or b""
+ if body_bytes:
+ error_detail += (
+ f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"
+ )
+ except Exception:
+ pass
+ logger.error(f"[{request_id}] WS proxy error: {error_detail}")
+ if termination_cause == "unknown":
+ termination_cause = "client_error"
+ with contextlib.suppress(Exception):
+ await websocket.close(code=1011, reason=str(e)[:120])
+ finally:
+ # Unit 2: emit structured per-session stage timings.
+ stage_timer.record(
+ "total_session",
+ (time.perf_counter() - session_started_at) * 1000.0,
+ )
+ # Unit 3: deregister the session before (or independently
+ # of) the stage-timings log so a failure there cannot leak
+ # the registry entry. ``deregister`` is idempotent, so a
+ # session that never registered is a no-op.
+ if ws_sessions is not None and session_handle is not None:
+ # Use deregister_and_count so the handle pop and the
+ # relay-task count are read atomically inside the
+ # registry. Capturing ``len(session_handle.relay_tasks)``
+ # separately before ``deregister`` would risk drift if
+ # the registry's bookkeeping ever changes.
+ _deregistered, released_tasks = ws_sessions.deregister_and_count(
+ session_id, cause=termination_cause
+ )
+ session_duration_ms = (time.perf_counter() - session_started_at) * 1000.0
+ metrics_for_close = getattr(self, "metrics", None)
+ if metrics_for_close is not None:
+ with contextlib.suppress(Exception):
+ if hasattr(metrics_for_close, "dec_active_ws_sessions"):
+ metrics_for_close.dec_active_ws_sessions()
+ if released_tasks and hasattr(metrics_for_close, "dec_active_relay_tasks"):
+ metrics_for_close.dec_active_relay_tasks(released_tasks)
+ if hasattr(metrics_for_close, "record_ws_session_duration"):
+ metrics_for_close.record_ws_session_duration(
+ session_duration_ms, termination_cause
+ )
+ await emit_stage_timings_log(
+ path="openai_responses_ws",
+ request_id=request_id,
+ session_id=session_id,
+ stage_timer=stage_timer,
+ expected_stages=(
+ "accept",
+ "first_client_frame",
+ "upstream_connect",
+ "upstream_first_event",
+ "memory_context",
+ "compression",
+ "total_session",
+ ),
+ metrics=getattr(self, "metrics", None),
+ )
+
+ async def _ws_http_fallback(
+ self,
+ websocket: WebSocket,
+ body: dict[str, Any],
+ first_msg_raw: str,
+ upstream_headers: dict[str, str],
+ request_id: str,
+ ) -> None:
+ """Fall back to HTTP POST streaming when upstream WS fails.
+
+ Converts the WS ``response.create`` message to an HTTP POST to
+ ``/v1/responses?stream=true``, reads SSE events, and relays each
+ ``data:`` line as a WS text message to the client. This makes
+ Codex work immediately instead of exhausting its WS retry budget.
+ """
+ # Route to correct endpoint based on auth mode
+ _lower = {k.lower() for k in upstream_headers}
+ if "chatgpt-account-id" in _lower:
+ http_url = "https://chatgpt.com/backend-api/codex/responses"
+ else:
+ http_url = f"{self.OPENAI_API_URL}/v1/responses"
+
+ # Build HTTP body from the WS response.create payload.
+ # WS messages use {"type": "response.create", "response": {...}} wrapper.
+ # The HTTP POST endpoint expects the inner response object directly.
+ http_body: dict[str, Any]
+ try:
+ parsed = json.loads(first_msg_raw) if isinstance(first_msg_raw, str) else body
+ except (json.JSONDecodeError, TypeError):
+ parsed = body
+
+ # Normalize WebSocket response.create payload into the HTTP request body.
+ # Codex may send either:
+ # 1. {"type":"response.create","response":{...}}
+ # 2. {"type":"response.create", ...response fields...}
+ if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):
+ http_body = dict(parsed["response"])
+ elif isinstance(parsed, dict):
+ http_body = dict(parsed)
+ if http_body.get("type") == "response.create":
+ http_body.pop("type", None)
+ else:
+ http_body = body if isinstance(body, dict) else {}
+
+ # Some clients include response-ish metadata that the HTTP endpoint rejects.
+ if http_body.get("type") in {"response.create", "response"}:
+ http_body.pop("type", None)
+
+ # Ensure streaming is enabled so we get SSE events
+ http_body["stream"] = True
+
+ # Build HTTP headers from the upstream headers (already stripped of WS
+ # hop-by-hop headers by the caller).
+ http_headers = dict(upstream_headers)
+ http_headers["content-type"] = "application/json"
+
+ logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")
+
+ try:
+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ for http_attempt in range(retry_attempts):
+ try:
+ async with self.http_client.stream(
+ "POST",
+ http_url,
+ headers=http_headers,
+ json=http_body,
+ timeout=120.0,
+ ) as response:
+ if response.status_code != 200:
+ error_body = b""
+ async for chunk in response.aiter_bytes():
+ error_body += chunk
+ if len(error_body) > 2000:
+ break
+ error_text = error_body.decode("utf-8", errors="replace")
+ logger.error(
+ f"[{request_id}] WS→HTTP fallback got {response.status_code}: "
+ f"{error_text[:500]}"
+ )
+ # Send error as WS message so client sees it
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "server_error",
+ "message": f"Upstream returned {response.status_code}",
+ },
+ }
+ await websocket.send_text(json.dumps(error_event))
+ return
+
+ # Relay SSE events as WS text messages
+ buffer = ""
+ async for chunk in response.aiter_text():
+ buffer += chunk
+ while "\n" in buffer:
+ line, buffer = buffer.split("\n", 1)
+ line = line.strip()
+ if not line:
+ continue
+ if line.startswith("data: "):
+ data = line[6:]
+ if data == "[DONE]":
+ continue
+ try:
+ await websocket.send_text(data)
+ except Exception:
+ return
+ elif line.startswith("event: "):
+ # SSE event type — skip, the data line contains the type
+ continue
+
+ # Flush any remaining data in buffer
+ for line in buffer.strip().splitlines():
+ line = line.strip()
+ if line.startswith("data: ") and line[6:] != "[DONE]":
+ with contextlib.suppress(Exception):
+ await websocket.send_text(line[6:])
+ return
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:
+ if http_attempt >= retry_attempts - 1:
+ raise
+
+ delay_with_jitter = jitter_delay_ms(
+ self.config.retry_base_delay_ms,
+ self.config.retry_max_delay_ms,
+ http_attempt,
+ )
+ logger.warning(
+ f"[{request_id}] WS→HTTP fallback connect failed "
+ f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ except Exception as http_err:
+ logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "server_error",
+ "message": f"HTTP fallback failed: {http_err!s}"[:200],
+ },
+ }
+ with contextlib.suppress(Exception):
+ await websocket.send_text(json.dumps(error_event))
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close()
+
+ async def handle_compress(self, request: Request) -> JSONResponse:
+ """Compress messages without calling an LLM.
+
+ POST /v1/compress
+ Body: {"messages": [...], "model": "...", "config": {}}
+ Returns compressed messages + metrics.
+ """
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+
+ # Check bypass header
+ if request.headers.get("x-headroom-bypass", "").lower() == "true":
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={"error": f"Invalid request body: {e!s}"},
+ )
+ messages = body.get("messages", [])
+ return JSONResponse(
+ {
+ "messages": messages,
+ "tokens_before": 0,
+ "tokens_after": 0,
+ "tokens_saved": 0,
+ "compression_ratio": 1.0,
+ "transforms_applied": [],
+ "ccr_hashes": [],
+ }
+ )
+
+ try:
+ body = await _read_request_json(request)
+ except Exception:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Invalid JSON in request body.",
+ }
+ },
+ )
+
+ messages = body.get("messages")
+ model = body.get("model")
+
+ if messages is None:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Missing required field: messages",
+ }
+ },
+ )
+
+ if model is None:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Missing required field: model",
+ }
+ },
+ )
+
+ if not messages:
+ return JSONResponse(
+ {
+ "messages": [],
+ "tokens_before": 0,
+ "tokens_after": 0,
+ "tokens_saved": 0,
+ "compression_ratio": 1.0,
+ "transforms_applied": [],
+ "ccr_hashes": [],
+ }
+ )
+
+ try:
+ # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)
+ # Allow optional token_budget to override model's context limit
+ # (used by OpenClaw compact() and other callers that need tighter budgets)
+ token_budget = body.get("token_budget")
+ context_limit = (
+ token_budget
+ if token_budget and isinstance(token_budget, int)
+ else self.openai_provider.get_context_limit(model)
+ )
+ # Extract CompressConfig options from request body
+ compress_config = body.get("config", {})
+ compress_user_messages = compress_config.get("compress_user_messages", False)
+ target_ratio = compress_config.get("target_ratio")
+ protect_recent = compress_config.get("protect_recent")
+ protect_analysis_context = compress_config.get("protect_analysis_context")
+
+ pipeline_kwargs: dict = {"model_limit": context_limit}
+ if compress_user_messages:
+ pipeline_kwargs["compress_user_messages"] = True
+ if target_ratio is not None:
+ pipeline_kwargs["target_ratio"] = float(target_ratio)
+ if protect_recent is not None:
+ pipeline_kwargs["protect_recent"] = int(protect_recent)
+ if protect_analysis_context is not None:
+ pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)
+
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ **pipeline_kwargs,
+ )
+
+ return JSONResponse(
+ {
+ "messages": result.messages,
+ "tokens_before": result.tokens_before,
+ "tokens_after": result.tokens_after,
+ "tokens_saved": result.tokens_before - result.tokens_after,
+ "compression_ratio": (
+ result.tokens_after / result.tokens_before
+ if result.tokens_before > 0
+ else 1.0
+ ),
+ "transforms_applied": result.transforms_applied,
+ "transforms_summary": result.transforms_summary,
+ "ccr_hashes": result.markers_inserted,
+ }
+ )
+ except Exception as e:
+ logger.exception("Compression failed: %s", e)
+ return JSONResponse(
+ status_code=503,
+ content={
+ "error": {
+ "type": "compression_error",
+ "message": str(e),
+ }
+ },
+ )
+
+ async def handle_passthrough(
+ self,
+ request: Request,
+ base_url: str,
+ endpoint_name: str | None = None,
+ provider: str | None = None,
+ ) -> Response:
+ """Pass through request unchanged.
+
+ Args:
+ request: The incoming request
+ base_url: The upstream API base URL
+ endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")
+ provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")
+ """
+ from fastapi.responses import Response
+
+ start_time = time.time()
+ path = request.url.path
+ url = f"{base_url}{path}"
+
+ # Preserve query string parameters
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("accept-encoding", None)
+
+ body = await request.body()
+
+ headers = await apply_copilot_api_auth(headers, url=url)
+ response = await self.http_client.request( # type: ignore[union-attr]
+ method=request.method,
+ url=url,
+ headers=headers,
+ content=body,
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Track stats for passthrough requests
+ if endpoint_name and provider:
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider=provider,
+ model=f"passthrough:{endpoint_name}",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ cached=False,
+ )
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def handle_databricks_invocations(
+ self,
+ request: Request,
+ model: str,
+ ) -> Response | StreamingResponse:
+ """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.
+
+ This enables using the Databricks CLI directly with Headroom:
+ databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'
+
+ The request/response format is identical to OpenAI chat completions,
+ so we inject the model from the path and delegate to handle_openai_chat.
+ """
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+
+ request_id = await self._next_request_id()
+
+ try:
+ body = await _read_request_json(request)
+ except Exception as e:
+ logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid JSON: {e}",
+ "type": "invalid_request_error",
+ }
+ },
+ )
+
+ # Inject model from path into body (Databricks CLI passes model in URL, not body)
+ body["model"] = model
+
+ logger.info(f"[{request_id}] Databricks invocation: model={model}")
+
+ # Create a new request with the modified body
+ # We reuse the OpenAI chat handler since the format is identical
+ from starlette.requests import Request as StarletteRequest
+
+ # Build new scope with the body already parsed
+ scope = dict(request.scope)
+
+ # Create a simple receive function that returns our modified body
+ body_bytes = json.dumps(body).encode()
+
+ async def receive():
+ return {"type": "http.request", "body": body_bytes}
+
+ modified_request = StarletteRequest(scope, receive)
+
+ # Delegate to the OpenAI chat handler (same format)
+ return await self.handle_openai_chat(modified_request)
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index cee782ecf..398210e8e 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -1,2577 +1,2578 @@
-"""Headroom Proxy Server - Production Ready.
-
-A full-featured LLM proxy with optimization, caching, rate limiting,
-and observability.
-
-Features:
-- Context optimization (SmartCrusher, CacheAligner, RollingWindow)
-- Semantic caching (save costs on repeated queries)
-- Rate limiting (token bucket)
-- Retry with exponential backoff
-- Cost tracking and budgets
-- Request tagging and metadata
-- Provider fallback
-- Prometheus metrics
-- Full request/response logging
-
-Usage:
- python -m headroom.proxy.server --port 8787
-
- # With Claude Code:
- ANTHROPIC_BASE_URL=http://localhost:8787 claude
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import logging
-import os
-import sys
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
-
-if TYPE_CHECKING:
- from ..backends.base import Backend
- from ..cache.compression_cache import CompressionCache
- from ..memory.tracker import MemoryTracker
-
-
-import httpx
-
-try:
- import uvicorn
- from fastapi import Depends, FastAPI, HTTPException, Request, Response
- from fastapi.middleware.cors import CORSMiddleware
- from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
-
- FASTAPI_AVAILABLE = True
-except ImportError:
- FASTAPI_AVAILABLE = False
-
-# Add parent to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent.parent))
-
-from headroom._version import __version__
-from headroom.cache.compression_feedback import get_compression_feedback
-from headroom.cache.compression_store import get_compression_store
-from headroom.ccr import (
- CCR_TOOL_NAME,
- # Batch processing
- CCRResponseHandler,
- CCRToolInjector,
- ContextTracker,
- ContextTrackerConfig,
- ResponseHandlerConfig,
- parse_tool_call,
-)
-from headroom.config import (
- CacheAlignerConfig,
- CCRConfig,
- IntelligentContextConfig,
- ReadLifecycleConfig,
- RollingWindowConfig,
- SmartCrusherConfig,
-)
-from headroom.dashboard import get_dashboard_html
-from headroom.observability import (
- LangfuseTracingConfig,
- OTelMetricsConfig,
- configure_langfuse_tracing,
- configure_otel_metrics,
- get_langfuse_tracing_status,
- get_otel_metrics_status,
- shutdown_headroom_tracing,
- shutdown_otel_metrics,
-)
-from headroom.pipeline import PipelineExtensionManager, PipelineStage
-from headroom.providers.proxy_routes import register_provider_routes
-from headroom.providers.registry import (
- DEFAULT_ANTHROPIC_API_URL,
- DEFAULT_CLOUDCODE_API_URL,
- DEFAULT_GEMINI_API_URL,
- DEFAULT_OPENAI_API_URL,
- build_proxy_provider_runtime,
- create_proxy_backend,
- format_backend_status,
-)
-
-# =============================================================================
-# Extracted modules (re-exported for backward compatibility)
-# =============================================================================
-from headroom.proxy.cost import (
- _CACHE_ECONOMICS, # noqa: F401
- CostTracker, # noqa: F401
- _summarize_transforms, # noqa: F401
- build_prefix_cache_stats, # noqa: F401
- build_session_summary, # noqa: F401
- merge_cost_stats, # noqa: F401
-)
-from headroom.proxy.helpers import (
- COMPRESSION_TIMEOUT_SECONDS, # noqa: F401
- MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401
- MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401
- MAX_REQUEST_BODY_SIZE, # noqa: F401
- MAX_SSE_BUFFER_SIZE, # noqa: F401
- _get_image_compressor, # noqa: F401
- _get_rtk_stats, # noqa: F401
- _read_request_json, # noqa: F401
- _setup_file_logging, # noqa: F401
- is_anthropic_auth, # noqa: F401
- jitter_delay_ms,
-)
-from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
-
-# Data models (extracted to headroom/proxy/models.py for maintainability)
-from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
-from headroom.proxy.modes import (
- PROXY_MODE_CACHE,
- PROXY_MODE_TOKEN,
- is_token_mode,
- normalize_proxy_mode,
-)
-from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
-from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
-from headroom.proxy.request_logger import RequestLogger # noqa: F401
-from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
-from headroom.proxy.warmup import WarmupRegistry
-from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
-from headroom.subscription.base import get_quota_registry, reset_quota_registry
-from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state
-from headroom.subscription.copilot_quota import get_copilot_quota_tracker
-from headroom.subscription.tracker import (
- configure_subscription_tracker,
- get_subscription_tracker,
-)
-from headroom.telemetry import get_telemetry_collector
-from headroom.telemetry.beacon import is_telemetry_enabled
-from headroom.telemetry.toin import get_toin
-from headroom.transforms import (
- CacheAligner,
- CodeAwareCompressor,
- CodeCompressorConfig,
- ContentRouter,
- ContentRouterConfig,
- IntelligentContextManager,
- RollingWindow,
- SmartCrusher,
- Transform,
- TransformPipeline,
- is_tree_sitter_available,
-)
-
-AnyLLMBackend: Any = None
-LiteLLMBackend: Any = None
-
-fcntl: Any = None
-try:
- import fcntl as _fcntl
-
- fcntl = _fcntl
- HAS_FCNTL = True
-except ImportError:
- HAS_FCNTL = False
-
-_build_prefix_cache_stats = build_prefix_cache_stats
-_build_session_summary = build_session_summary
-_merge_cost_stats = merge_cost_stats
-
-
-logging.basicConfig(
- level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-)
-logger = logging.getLogger("headroom.proxy")
-
-# Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis
-_setup_file_logging()
-
-
-# Compression pipeline timeout in seconds
-
-
-from headroom.proxy.handlers import ( # noqa: E402
- AnthropicHandlerMixin,
- BatchHandlerMixin,
- GeminiHandlerMixin,
- OpenAIHandlerMixin,
- StreamingMixin,
-)
-
-
-class HeadroomProxy(
- StreamingMixin,
- AnthropicHandlerMixin,
- OpenAIHandlerMixin,
- GeminiHandlerMixin,
- BatchHandlerMixin,
-):
- """Production-ready Headroom optimization proxy."""
-
- ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL
- OPENAI_API_URL = DEFAULT_OPENAI_API_URL
- GEMINI_API_URL = DEFAULT_GEMINI_API_URL
- CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL
-
- def __init__(self, config: ProxyConfig):
- self.config = config
- self.config.mode = normalize_proxy_mode(self.config.mode)
- self.pipeline_extensions = PipelineExtensionManager(
- hooks=config.hooks,
- extensions=config.pipeline_extensions,
- discover=config.discover_pipeline_extensions,
- )
-
- self.provider_runtime = build_proxy_provider_runtime(config)
- api_targets = self.provider_runtime.api_targets
-
- # Preserve the long-standing proxy compatibility surface while keeping
- # provider_runtime as the source of truth for resolved upstream targets.
- HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic
- HeadroomProxy.OPENAI_API_URL = api_targets.openai
- HeadroomProxy.GEMINI_API_URL = api_targets.gemini
- HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode
- self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")
- self.openai_provider = self.provider_runtime.pipeline_provider("openai")
-
- # Initialize transforms based on routing mode
- # Choose context manager: IntelligentContextManager (smart) or RollingWindow (legacy)
- context_manager: Transform # Can be either IntelligentContextManager or RollingWindow
- if config.intelligent_context:
- # Get TOIN instance for learned pattern integration
- toin = get_toin() if config.intelligent_context_scoring else None
- context_manager = IntelligentContextManager(
- config=IntelligentContextConfig(
- enabled=True,
- keep_system=True,
- keep_last_turns=config.keep_last_turns,
- use_importance_scoring=config.intelligent_context_scoring,
- toin_integration=config.intelligent_context_scoring,
- compress_threshold=0.10 if config.intelligent_context_compress_first else 0.0,
- ),
- toin=toin,
- )
- self._context_manager_status = "intelligent"
- else:
- context_manager = RollingWindow(
- RollingWindowConfig(
- enabled=True,
- keep_system=True,
- keep_last_turns=config.keep_last_turns,
- )
- )
- self._context_manager_status = "rolling_window"
-
- if config.smart_routing:
- # Smart routing: ContentRouter handles all content types intelligently
- # It lazy-loads compressors only when needed
- router_config = ContentRouterConfig(
- enable_code_aware=config.code_aware_enabled,
- tool_profiles=config.tool_profiles,
- read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),
- )
- # Token mode: allow compression of older excluded-tool results
- if is_token_mode(config.mode):
- router_config.protect_recent_reads_fraction = 0.3
- transforms = [
- CacheAligner(CacheAlignerConfig(enabled=False)),
- ContentRouter(router_config),
- context_manager,
- ]
- self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"
- else:
- # Legacy mode: sequential pipeline
- transforms = [
- CacheAligner(CacheAlignerConfig(enabled=False)),
- SmartCrusher(
- SmartCrusherConfig( # type: ignore[arg-type]
- enabled=True,
- min_tokens_to_crush=config.min_tokens_to_crush,
- max_items_after_crush=config.max_items_after_crush,
- ),
- ccr_config=CCRConfig(
- enabled=config.ccr_inject_tool,
- inject_retrieval_marker=config.ccr_inject_tool, # Add CCR markers
- ),
- ),
- context_manager,
- ]
- # Add CodeAware if enabled and available
- self._code_aware_status = self._setup_code_aware(config, transforms)
-
- self.anthropic_pipeline = TransformPipeline(
- transforms=transforms,
- provider=self.anthropic_provider,
- )
- self.openai_pipeline = TransformPipeline(
- transforms=transforms,
- provider=self.openai_provider,
- )
-
- # Initialize components
- self.cache = (
- SemanticCache(
- max_entries=config.cache_max_entries,
- ttl_seconds=config.cache_ttl_seconds,
- )
- if config.cache_enabled
- else None
- )
-
- self.rate_limiter = (
- TokenBucketRateLimiter(
- requests_per_minute=config.rate_limit_requests_per_minute,
- tokens_per_minute=config.rate_limit_tokens_per_minute,
- )
- if config.rate_limit_enabled
- else None
- )
-
- self.cost_tracker = (
- CostTracker(
- budget_limit_usd=config.budget_limit_usd,
- budget_period=config.budget_period,
- )
- if config.cost_tracking_enabled
- else None
- )
-
- self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)
-
- # Prefix cache tracking: freeze already-cached messages to avoid
- # invalidating the provider's prefix cache with our transforms
- from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore
-
- self.session_tracker_store = SessionTrackerStore(
- default_config=PrefixFreezeConfig(
- enabled=config.prefix_freeze_enabled,
- session_ttl_seconds=config.prefix_freeze_session_ttl,
- )
- )
-
- # Compression cache store for token mode (session-scoped)
- self._compression_caches: dict[str, CompressionCache] = {}
-
- self.logger = (
- RequestLogger(
- log_file=config.log_file,
- log_full_messages=config.log_full_messages,
- )
- if config.log_requests
- else None
- )
-
- # Enterprise security plugin (loaded dynamically if available + licensed)
- self.security = None
-
- # HTTP client
- self.http_client: httpx.AsyncClient | None = None
-
- # Shared cold-start warmup registry (populated by startup()).
- # Holds typed slots with loaded / loading / null / error status for
- # each preloaded heavy asset. Exposed as ``proxy.warmup`` and
- # serialized by the /debug/warmup route (Unit 5).
- self.warmup: WarmupRegistry = WarmupRegistry()
- # Unit 3: live registry of Codex WS sessions. Populated by
- # ``handle_openai_responses_ws`` on accept; drained in its
- # outermost ``finally``. Consumed by ``/debug/ws-sessions``.
- self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()
-
- # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP
- # path. Caps how many ``handle_anthropic_messages`` calls may be
- # running deep-copy / first-stage compression / memory-context
- # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,
- # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are
- # intentionally NOT gated by this semaphore.
- #
- # A value of ``0`` or negative disables the semaphore (unbounded
- # mode); this is useful for the Unit 6 counter-factual where we
- # deliberately reproduce the original starvation. The default is
- # ``max(2, min(8, os.cpu_count() or 4))``.
- _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency
- if _pre_upstream_cfg is None:
- _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))
- else:
- _pre_upstream_resolved = _pre_upstream_cfg
- self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved
- self.anthropic_pre_upstream_acquire_timeout_seconds = float(
- config.anthropic_pre_upstream_acquire_timeout_seconds
- )
- self.anthropic_pre_upstream_memory_context_timeout_seconds = float(
- config.anthropic_pre_upstream_memory_context_timeout_seconds
- )
- if _pre_upstream_resolved > 0:
- self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = asyncio.Semaphore(
- _pre_upstream_resolved
- )
- else:
- self.anthropic_pre_upstream_sem = None
-
- # Backend for Anthropic API (direct, LiteLLM, or any-llm)
- # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
- self.anthropic_backend: Backend | None = create_proxy_backend(
- backend=config.backend,
- anyllm_provider=config.anyllm_provider,
- bedrock_region=config.bedrock_region,
- logger=logger,
- anyllm_backend_cls=AnyLLMBackend,
- litellm_backend_cls=LiteLLMBackend,
- )
-
- # Request counter for IDs
- self._request_counter = 0
- self._request_counter_lock = asyncio.Lock()
-
- # CCR tool injectors (one per provider)
- self.anthropic_tool_injector = CCRToolInjector(
- provider="anthropic",
- inject_tool=config.ccr_inject_tool,
- inject_system_instructions=config.ccr_inject_system_instructions,
- )
- self.openai_tool_injector = CCRToolInjector(
- provider="openai",
- inject_tool=config.ccr_inject_tool,
- inject_system_instructions=config.ccr_inject_system_instructions,
- )
-
- # CCR Response Handler (handles CCR tool calls automatically)
- self.ccr_response_handler = (
- CCRResponseHandler(
- ResponseHandlerConfig(
- enabled=True,
- max_retrieval_rounds=config.ccr_max_retrieval_rounds,
- )
- )
- if config.ccr_handle_responses
- else None
- )
-
- # CCR Context Tracker (tracks compressed content across turns)
- self.ccr_context_tracker = (
- ContextTracker(
- ContextTrackerConfig(
- enabled=True,
- proactive_expansion=config.ccr_proactive_expansion,
- max_proactive_expansions=config.ccr_max_proactive_expansions,
- )
- )
- if config.ccr_context_tracking
- else None
- )
-
- # Turn counter for context tracking
- self._turn_counter = 0
-
- # Memory Handler (persistent user memory)
- self.memory_handler: MemoryHandler | None = None
- if config.memory_enabled:
- # Resolve memory DB path: empty → project-scoped default
- _mem_db_path = config.memory_db_path
- if not _mem_db_path:
- _mem_dir = Path.cwd() / ".headroom"
- _mem_dir.mkdir(parents=True, exist_ok=True)
- _mem_db_path = str(_mem_dir / "memory.db")
- logger.info(f"Memory: Project-scoped DB at {_mem_db_path}")
-
- memory_config = MemoryConfig(
- enabled=True,
- backend=config.memory_backend,
- db_path=_mem_db_path,
- inject_tools=config.memory_inject_tools,
- use_native_tool=config.memory_use_native_tool,
- inject_context=config.memory_inject_context,
- top_k=config.memory_top_k,
- min_similarity=config.memory_min_similarity,
- qdrant_host=config.memory_qdrant_host,
- qdrant_port=config.memory_qdrant_port,
- neo4j_uri=config.memory_neo4j_uri,
- neo4j_user=config.memory_neo4j_user,
- neo4j_password=config.memory_neo4j_password,
- bridge_enabled=config.memory_bridge_enabled,
- bridge_md_paths=config.memory_bridge_md_paths,
- bridge_md_format=config.memory_bridge_md_format,
- bridge_auto_import=config.memory_bridge_auto_import,
- bridge_export_path=config.memory_bridge_export_path,
- )
- self.memory_handler = MemoryHandler(
- memory_config,
- agent_type=config.traffic_learning_agent_type,
- )
-
- # Usage Reporter (license validation + phone-home for managed/enterprise)
- self.usage_reporter: UsageReporter | None = None
- if config.license_key:
- from headroom.telemetry.reporter import UsageReporter
-
- self.usage_reporter = UsageReporter(
- license_key=config.license_key,
- cloud_url=config.license_cloud_url,
- report_interval=config.license_report_interval,
- )
-
- # Traffic Learner (live pattern extraction from proxy traffic)
- # Only activates with --learn flag; requires --memory for backend
- self.traffic_learner: TrafficLearner | None = None
- self.traffic_learning_agent_type: str = config.traffic_learning_agent_type
- if config.traffic_learning_enabled:
- from headroom.memory.traffic_learner import TrafficLearner
-
- self.traffic_learner = TrafficLearner(
- user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
- agent_type=config.traffic_learning_agent_type,
- )
-
- # Code graph file watcher (live reindex on file changes)
- self.code_graph_watcher: CodeGraphWatcher | None = None # type: ignore[annotation-unchecked]
- if config.code_graph_watcher:
- from headroom.graph.watcher import CodeGraphWatcher
-
- self.code_graph_watcher = CodeGraphWatcher(project_dir=Path.cwd())
- if self.code_graph_watcher.start():
- logger.info("Code graph: file watcher started")
- else:
- self.code_graph_watcher = None
-
- self.pipeline_extensions.emit(
- PipelineStage.SETUP,
- operation="proxy.setup",
- metadata={
- "mode": self.config.mode,
- "optimize": self.config.optimize,
- "backend": self.config.backend,
- "memory_enabled": self.config.memory_enabled,
- },
- )
-
- def _get_compression_cache(self, session_id: str) -> CompressionCache:
- """Get or create a CompressionCache for a session."""
- if session_id not in self._compression_caches:
- from headroom.cache.compression_cache import CompressionCache
-
- # Evict oldest caches if at capacity
- if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:
- # Remove oldest quarter to amortize cleanup cost
- oldest_keys = list(self._compression_caches.keys())[
- : MAX_COMPRESSION_CACHE_SESSIONS // 4
- ]
- for key in oldest_keys:
- del self._compression_caches[key]
- logger.info(
- "Evicted %d compression caches (exceeded %d max sessions)",
- len(oldest_keys),
- MAX_COMPRESSION_CACHE_SESSIONS,
- )
-
- self._compression_caches[session_id] = CompressionCache()
- return self._compression_caches[session_id]
-
- def _setup_code_aware(self, config: ProxyConfig, transforms: list) -> str:
- """Set up code-aware compression if enabled.
-
- Args:
- config: Proxy configuration
- transforms: Transform list to append to
-
- Returns:
- Status string for logging: 'enabled', 'disabled', 'available', 'unavailable'
- """
- if config.code_aware_enabled:
- if is_tree_sitter_available():
- code_config = CodeCompressorConfig(
- preserve_imports=True,
- preserve_signatures=True,
- preserve_type_annotations=True,
- )
- # Insert before RollingWindow (which should be last)
- transforms.insert(-1, CodeAwareCompressor(code_config))
- return "enabled"
- else:
- logger.warning(
- "Code-aware compression requested but tree-sitter not installed. "
- "Install with: pip install headroom-ai[code]"
- )
- return "unavailable"
- else:
- if is_tree_sitter_available():
- return "available" # Available but not enabled
- return "disabled"
-
- async def startup(self):
- """Initialize async resources."""
- self.pipeline_extensions.emit(
- PipelineStage.PRE_START,
- operation="proxy.startup",
- metadata={"port": self.config.port, "host": self.config.host},
- )
- self.http_client = httpx.AsyncClient(
- timeout=httpx.Timeout(
- connect=self.config.connect_timeout_seconds,
- read=self.config.request_timeout_seconds,
- write=self.config.request_timeout_seconds,
- pool=self.config.connect_timeout_seconds,
- ),
- limits=httpx.Limits(
- max_connections=self.config.max_connections,
- max_keepalive_connections=self.config.max_keepalive_connections,
- ),
- http2=self.config.http2,
- )
- logger.info("Headroom Proxy started")
- logger.info(f"Optimization: {'ENABLED' if self.config.optimize else 'DISABLED'}")
- self.config.mode = normalize_proxy_mode(self.config.mode)
- logger.info(f"Mode: {self.config.mode}")
- if self.config.mode == PROXY_MODE_TOKEN:
- logger.info(" Prefix freeze: re-freeze after compression")
- logger.info(" Read protection window: 30%% of excluded-tool messages")
- logger.info(" CCR TTL: extended for session lifetime")
- logger.info(" Compression cache: active")
- if self.config.mode == PROXY_MODE_CACHE:
- logger.info(" Prefix freeze: strict (all prior turns immutable)")
- logger.info(" Mutations: latest turn only")
- logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}")
- logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}")
- logger.info(
- f"Connection Pool: max_connections={self.config.max_connections}, "
- f"max_keepalive={self.config.max_keepalive_connections}, "
- f"http2={'ENABLED' if self.config.http2 else 'DISABLED'}"
- )
-
- # Unit 4 pre-upstream concurrency announcement. Report the resolved
- # value (auto-detected vs. explicit) so operators can correlate
- # ``pre_upstream_wait_ms`` log lines with the configured cap.
- if self.anthropic_pre_upstream_sem is None:
- logger.info("Anthropic pre-upstream concurrency: unbounded (explicitly disabled)")
- else:
- _explicit = self.config.anthropic_pre_upstream_concurrency
- _origin = "auto-detected" if _explicit is None else "explicit"
- logger.info(
- "Anthropic pre-upstream concurrency: %d (%s)",
- self.anthropic_pre_upstream_concurrency,
- _origin,
- )
- logger.info(
- "Anthropic pre-upstream timeouts: acquire=%.1fs compression=%.1fs memory_context=%.1fs",
- self.anthropic_pre_upstream_acquire_timeout_seconds,
- float(COMPRESSION_TIMEOUT_SECONDS),
- self.anthropic_pre_upstream_memory_context_timeout_seconds,
- )
-
- # Smart routing status
- if self.config.smart_routing:
- logger.info("Smart Routing: ENABLED (intelligent content detection)")
- else:
- logger.info("Smart Routing: DISABLED (legacy sequential mode)")
-
- # Eagerly load ALL compressors, parsers, and detectors at startup
- # This eliminates cold-start latency spikes on first requests.
- # Iterate BOTH pipelines (Anthropic + OpenAI) and dedupe transforms
- # by id() so shared-transform instances never load twice. The
- # resulting status dict is merged into ``self.warmup`` so /debug/warmup
- # (Unit 5) and /readyz have a single source of truth.
- self._kompress_status = "not installed"
- eager_status: dict[str, str] = {}
-
- if self.config.optimize:
- logger.info("Pre-loading compressors and parsers...")
- seen_transform_ids: set[int] = set()
- pipelines = (self.anthropic_pipeline, self.openai_pipeline)
- for pipeline in pipelines:
- for transform in pipeline.transforms:
- if id(transform) in seen_transform_ids:
- continue
- seen_transform_ids.add(id(transform))
- if not hasattr(transform, "eager_load_compressors"):
- continue
- try:
- transform_status = transform.eager_load_compressors()
- except Exception as exc:
- logger.warning(
- "Eager preload failed for %s: %s",
- type(transform).__name__,
- exc,
- )
- continue
- if not isinstance(transform_status, dict):
- continue
- # Merge: later writers win only if the key wasn't set.
- # Preload a transform ONCE — if another pipeline also has
- # ``eager_load_compressors`` it contributes only new keys.
- for key, value in transform_status.items():
- eager_status.setdefault(key, value)
- self.warmup.merge_transform_status(transform_status)
-
- # Update internal status from eager loading results
- if eager_status.get("kompress") == "enabled":
- self._kompress_status = "enabled"
- if eager_status.get("code_aware") == "enabled":
- self._code_aware_status = "enabled"
-
- # Log component status
- if self._kompress_status == "enabled":
- logger.info("Kompress: ENABLED (ModernBERT token compressor)")
- elif self.config.optimize:
- logger.info("Kompress: not installed (pip install headroom-ai[ml] for ML compression)")
-
- if self._code_aware_status == "enabled":
- logger.info("Code-Aware: ENABLED (AST-based compression)")
- if "tree_sitter" in eager_status:
- logger.info(f"Tree-Sitter: {eager_status['tree_sitter']}")
- elif self._code_aware_status == "lazy":
- logger.info("Code-Aware: LAZY (will load when code content detected)")
- elif self._code_aware_status == "available":
- logger.info("Code-Aware: available but disabled (use --code-aware)")
- elif self._code_aware_status == "unavailable":
- logger.info("Code-Aware: not installed (pip install headroom-ai[code])")
- elif self._code_aware_status == "disabled":
- logger.info("Code-Aware: DISABLED")
-
- if eager_status.get("magika") == "enabled":
- logger.info("Magika: ENABLED (ML content detection)")
-
- if self.memory_handler:
- self.warmup.memory_backend.mark_loading()
- try:
- await self.memory_handler.ensure_initialized()
- except Exception as exc: # pragma: no cover - defensive
- self.warmup.memory_backend.mark_error(str(exc))
- logger.warning("Memory: backend initialization failed (startup continues): %s", exc)
- memory_status = self.memory_handler.health_status()
- if memory_status.get("initialized"):
- self.warmup.memory_backend.mark_loaded(
- handle=self.memory_handler,
- backend=memory_status.get("backend"),
- )
- # Force one embed call so the ONNX graph is compiled now,
- # not lazily during the first request. Best-effort — any
- # failure is swallowed inside warmup_embedder.
- self.warmup.memory_embedder.mark_loading()
- warmed = await self.memory_handler.warmup_embedder()
- if warmed:
- self.warmup.memory_embedder.mark_loaded()
- else:
- # Not an error — e.g. qdrant-neo4j has no embedder slot
- # we can reach, or the backend simply exposes no handle.
- self.warmup.memory_embedder.mark_null()
- else:
- if self.warmup.memory_backend.status != "error":
- self.warmup.memory_backend.mark_null()
- self.warmup.memory_embedder.mark_null()
- logger.info(
- "Memory: ENABLED "
- f"(backend={memory_status['backend']}, initialized={memory_status['initialized']})"
- )
- else:
- logger.info("Memory: DISABLED")
-
- # CCR status
- ccr_features = []
- if self.config.ccr_inject_tool:
- ccr_features.append("tool_injection")
- if self.config.ccr_handle_responses:
- ccr_features.append("response_handling")
- if self.config.ccr_context_tracking:
- ccr_features.append("context_tracking")
- if self.config.ccr_proactive_expansion:
- ccr_features.append("proactive_expansion")
- if ccr_features:
- logger.info(f"CCR (Compress-Cache-Retrieve): ENABLED ({', '.join(ccr_features)})")
- else:
- logger.info("CCR: DISABLED")
- logger.info(f"Savings history: {self.metrics.savings_tracker.storage_path}")
-
- # Reset and rebuild the quota tracker registry for this server instance.
- # reset_quota_registry() ensures a clean slate when the proxy is restarted
- # (e.g. in tests that spin up multiple app instances in the same process).
- reset_quota_registry()
- registry = get_quota_registry()
- tracker = configure_subscription_tracker(
- poll_interval_s=self.config.subscription_poll_interval_s,
- active_window_s=self.config.subscription_active_window_s,
- enabled=self.config.subscription_tracking_enabled,
- )
- registry.register(tracker)
- registry.register(get_codex_rate_limit_state())
- registry.register(get_copilot_quota_tracker())
- await registry.start_all()
-
- if self.config.subscription_tracking_enabled:
- logger.info(
- "Subscription tracking: ENABLED "
- f"(poll_interval={self.config.subscription_poll_interval_s}s, "
- f"active_window={self.config.subscription_active_window_s}s)"
- )
- else:
- logger.info("Subscription tracking: DISABLED")
-
- copilot_tracker = get_copilot_quota_tracker()
- if copilot_tracker.is_available():
- logger.info("GitHub Copilot quota tracking: ENABLED")
- else:
- logger.info(
- "GitHub Copilot quota tracking: DISABLED "
- "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)"
- )
-
- # Log anonymous telemetry status so operators can see it in the log stream
- if is_telemetry_enabled():
- logger.info(
- "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). "
- "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry"
- )
- else:
- logger.info("Anonymous telemetry: DISABLED")
-
- self.pipeline_extensions.emit(
- PipelineStage.POST_START,
- operation="proxy.startup",
- metadata={
- "port": self.config.port,
- "host": self.config.host,
- "warmup": self.warmup.to_dict(),
- },
- )
-
- async def shutdown(self):
- """Cleanup async resources."""
- if self.http_client:
- await self.http_client.aclose()
- self.http_client = None
-
- if self.memory_handler and hasattr(self.memory_handler, "close"):
- await self.memory_handler.close()
-
- # Stop all quota trackers via the registry
- await get_quota_registry().stop_all()
-
- # Print final stats
- self._print_summary()
-
- def _print_summary(self):
- """Print session summary."""
- m = self.metrics
- logger.info("=" * 70)
- logger.info("HEADROOM PROXY SESSION SUMMARY")
- logger.info("=" * 70)
- logger.info(f"Total requests: {m.requests_total}")
- logger.info(f"Cached responses: {m.requests_cached}")
- logger.info(f"Rate limited: {m.requests_rate_limited}")
- logger.info(f"Failed: {m.requests_failed}")
- logger.info(f"Input tokens: {m.tokens_input_total:,}")
- logger.info(f"Output tokens: {m.tokens_output_total:,}")
- logger.info(f"Tokens saved: {m.tokens_saved_total:,}")
- if m.tokens_input_total > 0:
- savings_pct = (
- m.tokens_saved_total / (m.tokens_input_total + m.tokens_saved_total)
- ) * 100
- logger.info(f"Token savings: {savings_pct:.1f}%")
- if m.latency_count > 0:
- avg_latency = m.latency_sum_ms / m.latency_count
- logger.info(f"Avg latency: {avg_latency:.0f}ms")
- logger.info("=" * 70)
-
- async def _next_request_id(self) -> str:
- """Generate unique request ID."""
- async with self._request_counter_lock:
- self._request_counter += 1
- return f"hr_{int(time.time())}_{self._request_counter:06d}"
-
- def _extract_tags(self, headers: dict) -> dict[str, str]:
- """Extract Headroom tags from headers."""
- tags = {}
- for key, value in headers.items():
- if key.lower().startswith("x-headroom-"):
- tag_name = key.lower().replace("x-headroom-", "")
- tags[tag_name] = value
- return tags
-
- def _inject_system_context(
- self,
- messages: list[dict[str, Any]],
- context: str,
- body: dict[str, Any] | None = None,
- ) -> list[dict[str, Any]]:
- """Inject context into the system message/parameter.
-
- For Anthropic API: Uses top-level 'system' parameter (not messages array).
- For OpenAI API: Uses system role in messages array.
-
- Args:
- messages: The messages list.
- context: Context to inject.
- body: Optional request body to update system parameter (for Anthropic).
-
- Returns:
- Updated messages list.
- """
- messages = list(messages) # Copy to avoid mutation
-
- # For Anthropic API: use top-level 'system' parameter
- if body is not None:
- existing_system = body.get("system", "")
- if isinstance(existing_system, str):
- body["system"] = (existing_system + "\n\n" + context).strip()
- elif isinstance(existing_system, list):
- # system is a list of content blocks (e.g., with cache_control).
- # Append memory context as a new text block — never overwrite.
- body["system"] = existing_system + [{"type": "text", "text": context}]
- else:
- # No existing system prompt — set as string
- body["system"] = context
- return messages
-
- # For OpenAI API: use system role in messages
- for i, msg in enumerate(messages):
- if msg.get("role") == "system":
- content = msg.get("content", "")
- if isinstance(content, str):
- messages[i] = {**msg, "content": content + "\n\n" + context}
- return messages
-
- # No system message found - prepend one
- messages.insert(0, {"role": "system", "content": context})
- return messages
-
- async def _retry_request(
- self,
- method: str,
- url: str,
- headers: dict,
- body: dict,
- stream: bool = False,
- ) -> httpx.Response:
- """Make request with retry and exponential backoff."""
- last_error = None
-
- for attempt in range(self.config.retry_max_attempts):
- try:
- if stream:
- # For streaming, we return early - retry happens at higher level
- return await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
- else:
- response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
-
- # Don't retry client errors (4xx)
- if 400 <= response.status_code < 500:
- return response
-
- # Retry server errors (5xx)
- if response.status_code >= 500:
- raise httpx.HTTPStatusError(
- f"Server error: {response.status_code}",
- request=response.request,
- response=response,
- )
-
- return response
-
- except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
- last_error = e
-
- if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:
- raise
-
- # Exponential backoff with jitter
- delay_with_jitter = jitter_delay_ms(
- self.config.retry_base_delay_ms,
- self.config.retry_max_delay_ms,
- attempt,
- )
-
- logger.warning(
- f"Request failed (attempt {attempt + 1}), retrying in {delay_with_jitter:.0f}ms: {e}"
- )
- await asyncio.sleep(delay_with_jitter / 1000)
-
- raise last_error # type: ignore[misc]
-
-
-async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:
- """Background task that logs TOIN stats periodically.
-
- Args:
- interval_seconds: How often to log stats (default: 5 minutes).
- """
- while True:
- await asyncio.sleep(interval_seconds)
- try:
- toin = get_toin()
- stats = toin.get_stats()
- total_compressions = stats.get("total_compressions", 0)
- if total_compressions > 0:
- patterns = stats.get("patterns_tracked", 0)
- retrievals = stats.get("total_retrievals", 0)
- retrieval_rate = stats.get("global_retrieval_rate", 0.0)
- logger.info(
- "TOIN: %d patterns, %d compressions, %d retrievals, %.1f%% retrieval rate",
- patterns,
- total_compressions,
- retrievals,
- retrieval_rate * 100,
- )
- except Exception as e:
- logger.debug("Failed to log TOIN stats: %s", e)
-
-
-def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:
- """Register all memory-tracked components with the tracker.
-
- This function is idempotent - it checks if components are already registered.
-
- Args:
- proxy: The HeadroomProxy instance.
- tracker: The MemoryTracker instance.
- """
- # Register compression store (global singleton)
- if "compression_store" not in tracker.registered_components:
- store = get_compression_store()
- tracker.register("compression_store", store.get_memory_stats)
-
- # Register semantic cache (instance on proxy)
- if proxy.cache and "semantic_cache" not in tracker.registered_components:
- tracker.register("semantic_cache", proxy.cache.get_memory_stats)
-
- # Register request logger (instance on proxy)
- if proxy.logger and "request_logger" not in tracker.registered_components:
- tracker.register("request_logger", proxy.logger.get_memory_stats)
-
- # Register batch context store (global singleton)
- if "batch_context_store" not in tracker.registered_components:
- try:
- from ..ccr.batch_store import get_batch_context_store
-
- batch_store = get_batch_context_store()
- if hasattr(batch_store, "get_memory_stats"):
- tracker.register("batch_context_store", batch_store.get_memory_stats)
- except ImportError:
- pass
-
- # Note: graph_store and vector_index are created per-user within the
- # LocalMemoryBackend, not as global singletons. They would need to be
- # registered when the memory system is initialized with specific backends.
-
-
-def create_app(config: ProxyConfig | None = None) -> FastAPI:
- """Create FastAPI application."""
- if not FASTAPI_AVAILABLE:
- raise ImportError("FastAPI required. Install: pip install fastapi uvicorn httpx")
-
- from contextlib import asynccontextmanager
-
- config = config or ProxyConfig()
- proxy = HeadroomProxy(config)
-
- # Telemetry beacon (anonymous aggregate stats).
- # With uvicorn workers > 1, each worker runs the lifespan independently.
- # We must ensure only ONE beacon runs across all workers — otherwise each
- # worker creates its own beacon, spamming the telemetry table with N rows
- # per cycle instead of 1 (all reading the same /stats from the same port).
- #
- # Strategy: use a file lock to ensure only the first worker starts the
- # beacon. Other workers see the lock and skip.
- from headroom.telemetry.beacon import TelemetryBeacon
-
- _beacon = TelemetryBeacon(
- port=config.port if hasattr(config, "port") else 8787,
- sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
- backend=config.backend if hasattr(config, "backend") else "anthropic",
- )
- from headroom import paths as _hr_paths
-
- _beacon_lock_path = _hr_paths.beacon_lock_path(config.port)
- _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor
- _beacon_is_owner: list = [False]
-
- def _try_acquire_beacon_lock() -> bool:
- """Try to acquire the beacon file lock (non-blocking).
-
- Returns True if this process is the beacon owner.
- """
- if not HAS_FCNTL:
- return True
-
- fd = None
- try:
- _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)
- fd = open(_beacon_lock_path, "w") # noqa: SIM115
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- fd.write(str(os.getpid()))
- fd.flush()
- _beacon_lock_fd[0] = fd
- return True
- except OSError:
- if fd is not None:
- fd.close()
- return False
-
- def _release_beacon_lock() -> None:
- """Release the beacon file lock."""
- fd = _beacon_lock_fd[0]
- if fd:
- try:
- if HAS_FCNTL:
- fcntl.flock(fd, fcntl.LOCK_UN)
- fd.close()
- except Exception:
- pass
- _beacon_lock_fd[0] = None
- try:
- _beacon_lock_path.unlink(missing_ok=True)
- except Exception:
- pass
-
- @asynccontextmanager
- async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
- configure_otel_metrics(OTelMetricsConfig.from_env(default_service_name="headroom-proxy"))
- configure_langfuse_tracing(
- LangfuseTracingConfig.from_env(default_service_name="headroom-proxy")
- )
-
- app.state.started_at = time.time()
- app.state.ready = False
- app.state.startup_error = None
-
- try:
- try:
- # Startup
- await proxy.startup()
- asyncio.create_task(_log_toin_stats_periodically())
- if proxy.usage_reporter:
- await proxy.usage_reporter.start(proxy)
- if proxy.traffic_learner:
- await proxy.traffic_learner.start()
-
- # Only start beacon if we acquire the lock (first worker wins)
- _beacon_is_owner[0] = _try_acquire_beacon_lock()
- if _beacon_is_owner[0]:
- await _beacon.start()
- else:
- logger.debug("Beacon: skipping (another worker owns the lock)")
-
- app.state.ready = True
- yield
- except Exception as exc:
- app.state.startup_error = str(exc)
- raise
- finally:
- app.state.ready = False
- # Shutdown
- if _beacon_is_owner[0]:
- await _beacon.stop()
- _release_beacon_lock()
- if proxy.usage_reporter:
- await proxy.usage_reporter.stop()
- if proxy.traffic_learner:
- await proxy.traffic_learner.stop()
- if proxy.code_graph_watcher:
- proxy.code_graph_watcher.stop()
- await proxy.shutdown()
- shutdown_headroom_tracing()
- shutdown_otel_metrics()
-
- app = FastAPI(
- title="Headroom Proxy",
- description="Production-ready LLM optimization proxy",
- version=__version__,
- lifespan=lifespan,
- )
- app.state.proxy = proxy
- app.state.started_at = None
- app.state.ready = False
- app.state.startup_error = None
-
- def _iso_utc_now() -> str:
- return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
-
- def _uptime_seconds() -> float:
- started_at = getattr(app.state, "started_at", None)
- if not isinstance(started_at, int | float):
- return 0.0
- return round(max(0.0, time.time() - float(started_at)), 3)
-
- def _component_health(
- *,
- enabled: bool,
- ready: bool,
- **details: Any,
- ) -> dict[str, Any]:
- status = "disabled" if not enabled else ("healthy" if ready else "unhealthy")
- return {
- "enabled": enabled,
- "ready": (ready if enabled else True),
- "status": status,
- **details,
- }
-
- def _health_checks() -> dict[str, dict[str, Any]]:
- memory_status = (
- proxy.memory_handler.health_status()
- if proxy.memory_handler
- else {
- "enabled": False,
- "backend": None,
- "initialized": False,
- "native_tool": False,
- "bridge_enabled": False,
- }
- )
- memory_enabled = bool(memory_status.get("enabled", False))
- memory_initialized = bool(memory_status.get("initialized", False))
- return {
- "startup": _component_health(
- enabled=True,
- ready=bool(getattr(app.state, "ready", False)),
- error=getattr(app.state, "startup_error", None),
- ),
- "http_client": _component_health(
- enabled=True,
- ready=proxy.http_client is not None,
- ),
- "cache": _component_health(
- enabled=config.cache_enabled,
- ready=(proxy.cache is not None),
- ),
- "rate_limiter": _component_health(
- enabled=config.rate_limit_enabled,
- ready=(proxy.rate_limiter is not None),
- ),
- "memory": _component_health(
- enabled=memory_enabled,
- ready=memory_initialized,
- backend=memory_status["backend"],
- initialized=memory_initialized,
- native_tool=bool(memory_status.get("native_tool", False)),
- bridge_enabled=bool(memory_status.get("bridge_enabled", False)),
- ),
- }
-
- def _runtime_payload() -> dict[str, Any]:
- ws_registry = getattr(proxy, "ws_sessions", None)
- ws_active_sessions = ws_registry.active_count() if ws_registry is not None else 0
- ws_active_relay_tasks = (
- ws_registry.active_relay_task_count() if ws_registry is not None else 0
- )
- return {
- "anthropic_pre_upstream": {
- "enabled": proxy.anthropic_pre_upstream_sem is not None,
- "resolved_concurrency": proxy.anthropic_pre_upstream_concurrency,
- "source": (
- "auto" if config.anthropic_pre_upstream_concurrency is None else "explicit"
- ),
- "acquire_timeout_seconds": proxy.anthropic_pre_upstream_acquire_timeout_seconds,
- "compression_timeout_seconds": float(COMPRESSION_TIMEOUT_SECONDS),
- "memory_context_timeout_seconds": (
- proxy.anthropic_pre_upstream_memory_context_timeout_seconds
- ),
- "codex_ws_gated": False,
- },
- "websocket_sessions": {
- "active_sessions": ws_active_sessions,
- "active_relay_tasks": ws_active_relay_tasks,
- },
- }
-
- def _health_payload(*, include_config: bool) -> dict[str, Any]:
- checks = _health_checks()
- ready = all(check["ready"] for check in checks.values())
- payload: dict[str, Any] = {
- "service": "headroom-proxy",
- "status": "healthy" if ready else "unhealthy",
- "ready": ready,
- "version": __version__,
- "timestamp": _iso_utc_now(),
- "uptime_seconds": _uptime_seconds(),
- "checks": checks,
- "runtime": _runtime_payload(),
- }
- deployment_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE")
- if deployment_profile:
- payload["deployment"] = {
- "profile": deployment_profile,
- "preset": os.environ.get("HEADROOM_DEPLOYMENT_PRESET"),
- "runtime": os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME"),
- "supervisor": os.environ.get("HEADROOM_DEPLOYMENT_SUPERVISOR"),
- "scope": os.environ.get("HEADROOM_DEPLOYMENT_SCOPE"),
- }
- if include_config:
- payload["config"] = {
- "backend": config.backend,
- "optimize": config.optimize,
- "cache": config.cache_enabled,
- "rate_limit": config.rate_limit_enabled,
- "memory": config.memory_enabled,
- "learn": config.traffic_learning_enabled,
- "code_graph": config.code_graph_watcher,
- "pid": os.getpid(),
- }
- return payload
-
- # CORS
- app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"],
- allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
- )
-
- # X-Headroom-Stack: SDK adapters (TS openai/anthropic/etc.) tag their
- # requests so telemetry can segment by integration surface. Registered
- # before extension middleware so any extension-level auth/guards run
- # outermost and we don't count requests they reject.
- @app.middleware("http")
- async def _record_headroom_stack(request, call_next):
- if request.url.path.startswith("/v1/"):
- stack = request.headers.get("x-headroom-stack")
- if stack:
- try:
- proxy.metrics.record_stack(stack)
- except Exception:
- logger.debug("record_stack failed", exc_info=True)
- return await call_next(request)
-
- # Third-party proxy extensions (Enterprise, custom plugins). Discovered via
- # the `headroom.proxy_extension` entry-point group. An extension that raises
- # from its install() is a deliberate fail-closed signal and aborts startup.
- from headroom.proxy.extensions import install_all as _install_extensions
-
- _install_extensions(app, config)
-
- # Health & Metrics
- @app.get("/livez")
- async def livez():
- return JSONResponse(
- status_code=200,
- content={
- "service": "headroom-proxy",
- "status": "healthy",
- "alive": True,
- "version": __version__,
- "timestamp": _iso_utc_now(),
- "uptime_seconds": _uptime_seconds(),
- },
- )
-
- @app.get("/readyz")
- async def readyz():
- payload = _health_payload(include_config=False)
- return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)
-
- @app.get("/health")
- async def health():
- payload = _health_payload(include_config=True)
- return JSONResponse(status_code=200, content=payload)
-
- # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —
- # debug endpoints are invisible to external scanners.
- from headroom.proxy.debug_introspection import (
- collect_tasks as _collect_tasks,
- )
- from headroom.proxy.loopback_guard import require_loopback as _require_loopback
-
- @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])
- async def debug_tasks(stack: bool = False):
- """Enumerate running asyncio tasks.
-
- Default is cheap — ``stack_depth`` is ``null`` in every entry so
- a storm snapshot does not walk 50+ coroutine frames synchronously.
- Pass ``?stack=true`` to compute ``stack_depth`` for each task
- (useful for single-shot human debugging).
- """
- ws_registry = getattr(proxy, "ws_sessions", None)
- return JSONResponse(
- status_code=200,
- content=_collect_tasks(ws_registry, with_stack_depth=stack),
- )
-
- @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])
- async def debug_ws_sessions():
- ws_registry = getattr(proxy, "ws_sessions", None)
- snapshot = ws_registry.snapshot() if ws_registry is not None else []
- return JSONResponse(status_code=200, content=snapshot)
-
- @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])
- async def debug_warmup():
- warmup_registry = getattr(proxy, "warmup", None)
- payload = warmup_registry.to_dict() if warmup_registry is not None else {}
- payload["runtime"] = _runtime_payload()
- return JSONResponse(status_code=200, content=payload)
-
- @app.get("/dashboard", response_class=HTMLResponse)
- async def dashboard():
- """Serve the Headroom dashboard UI."""
- return get_dashboard_html()
-
- @app.get("/stats")
- async def stats():
- """Get comprehensive proxy statistics.
-
- This is the main stats endpoint - it aggregates data from all subsystems:
- - Request metrics (total, cached, failed, by model/provider)
- - Token usage and savings
- - Cost tracking
- - Canonical persisted display_session metrics for downstream dashboards
- - Compression (CCR) statistics
- - Telemetry/TOIN (data flywheel) statistics
- - Cache and rate limiter stats
- """
- m = proxy.metrics
-
- # Calculate average latency
- avg_latency_ms = round(m.latency_sum_ms / m.latency_count, 2) if m.latency_count > 0 else 0
- min_latency_ms = (
- round(m.latency_min_ms, 2)
- if m.latency_count > 0 and m.latency_min_ms != float("inf")
- else 0
- )
- max_latency_ms = round(m.latency_max_ms, 2) if m.latency_count > 0 else 0
-
- # Calculate Headroom overhead (optimization time only, excludes pass-through requests)
- avg_overhead_ms = (
- round(m.overhead_sum_ms / m.overhead_count, 2) if m.overhead_count > 0 else 0
- )
- min_overhead_ms = (
- round(m.overhead_min_ms, 2)
- if m.overhead_count > 0 and m.overhead_min_ms != float("inf")
- else 0
- )
- max_overhead_ms = round(m.overhead_max_ms, 2) if m.overhead_count > 0 else 0
-
- # Calculate TTFB (time to first byte)
- avg_ttfb_ms = round(m.ttfb_sum_ms / m.ttfb_count, 2) if m.ttfb_count > 0 else 0
- min_ttfb_ms = (
- round(m.ttfb_min_ms, 2) if m.ttfb_count > 0 and m.ttfb_min_ms != float("inf") else 0
- )
- max_ttfb_ms = round(m.ttfb_max_ms, 2) if m.ttfb_count > 0 else 0
-
- # Get compression store stats
- store = get_compression_store()
- compression_stats = store.get_stats()
-
- # Get telemetry/TOIN stats
- telemetry = get_telemetry_collector()
- telemetry_stats = telemetry.get_stats()
-
- # Get feedback loop stats
- feedback = get_compression_feedback()
- feedback_stats = feedback.get_stats()
-
- # Build prefix cache stats once (used in both prefix_cache and cost)
- prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)
-
- # Fetch CLI filtering savings (rtk — tokens avoided before reaching context)
- cli_filtering_stats = _get_rtk_stats()
- cli_tokens_avoided = (
- cli_filtering_stats.get("tokens_saved", 0) if cli_filtering_stats else 0
- )
-
- # Calculate total tokens before compression
- total_tokens_before = m.tokens_input_total + m.tokens_saved_total
-
- # Build human-readable summary
- summary = _build_session_summary(
- proxy, m, prefix_cache_stats, cli_tokens_avoided, total_tokens_before
- )
- # DEBUG: log the summary payload for external upsert consumers
- try:
- logger.debug("/stats summary data: %r", summary)
- except Exception:
- logger.warning("Failed to log /stats summary payload")
-
- # Compression cache stats (token mode)
- compression_cache_stats: dict = {}
- if proxy.config.mode == PROXY_MODE_TOKEN and proxy._compression_caches:
- total_entries = 0
- total_hits = 0
- total_misses = 0
- total_tokens_saved = 0
- for cache in proxy._compression_caches.values():
- s = cache.get_stats()
- total_entries += s.get("entries", 0)
- total_hits += s.get("hits", 0)
- total_misses += s.get("misses", 0)
- total_tokens_saved += s.get("total_tokens_saved", 0)
- compression_cache_stats = {
- "mode": PROXY_MODE_TOKEN,
- "active_sessions": len(proxy._compression_caches),
- "total_entries": total_entries,
- "total_hits": total_hits,
- "total_misses": total_misses,
- "hit_rate": round(total_hits / max(1, total_hits + total_misses) * 100, 1),
- "total_tokens_saved": total_tokens_saved,
- }
- else:
- compression_cache_stats = {"mode": proxy.config.mode}
-
- # Build unified savings summary (all layers)
- compression_tokens = m.tokens_saved_total
- cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
- total_tokens_all_layers = compression_tokens + cli_tokens_avoided
- persistent_savings = m.savings_tracker.stats_preview()
- display_session = persistent_savings.get("display_session", {})
-
- return {
- "summary": summary,
- "savings": {
- "total_tokens": total_tokens_all_layers,
- "by_layer": {
- "cli_filtering": {
- "tokens": cli_tokens_avoided,
- "description": "Tokens avoided by CLI output filtering (rtk) before reaching context",
- },
- "compression": {
- "tokens": compression_tokens,
- "description": "Tokens removed by proxy compression (SmartCrusher, ContentRouter, etc.)",
- },
- "prefix_cache": {
- "discount_usd": round(cache_net_usd, 4),
- "description": (
- "Cost discount from provider prefix caching. "
- "Headroom's CacheAligner improves hit rates; "
- "baseline caching is provider-native."
- ),
- },
- },
- },
- "requests": {
- "total": m.requests_total,
- "cached": m.requests_cached,
- "rate_limited": m.requests_rate_limited,
- "failed": m.requests_failed,
- "by_provider": dict(m.requests_by_provider),
- "by_model": dict(m.requests_by_model),
- "by_stack": dict(m.requests_by_stack),
- },
- "tokens": {
- "input": m.tokens_input_total,
- "output": m.tokens_output_total,
- "saved": m.tokens_saved_total,
- "cli_tokens_avoided": cli_tokens_avoided,
- "total_before_compression": total_tokens_before,
- "savings_percent": round(
- (m.tokens_saved_total / total_tokens_before * 100)
- if total_tokens_before > 0
- else 0,
- 2,
- ),
- },
- "latency": {
- "average_ms": avg_latency_ms,
- "min_ms": min_latency_ms,
- "max_ms": max_latency_ms,
- "total_requests": m.latency_count,
- },
- "overhead": {
- "average_ms": avg_overhead_ms,
- "min_ms": min_overhead_ms,
- "max_ms": max_overhead_ms,
- },
- "ttfb": {
- "average_ms": avg_ttfb_ms,
- "min_ms": min_ttfb_ms,
- "max_ms": max_ttfb_ms,
- },
- "pipeline_timing": {
- name: {
- "average_ms": round(
- m.transform_timing_sum[name] / m.transform_timing_count[name], 2
- ),
- "max_ms": round(m.transform_timing_max[name], 2),
- "count": m.transform_timing_count[name],
- }
- for name in sorted(m.transform_timing_sum.keys())
- }
- if m.transform_timing_sum
- else {},
- "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
- "savings_history": m.savings_history[-100:], # Last 100 data points
- "display_session": display_session,
- "persistent_savings": persistent_savings,
- "prefix_cache": prefix_cache_stats,
- "cost": _merge_cost_stats(
- proxy.cost_tracker.stats() if proxy.cost_tracker else None,
- prefix_cache_stats,
- cli_tokens_avoided=cli_tokens_avoided,
- ),
- "compression": {
- "ccr_entries": compression_stats.get("entry_count", 0),
- "ccr_max_entries": compression_stats.get("max_entries", 0),
- "original_tokens_cached": compression_stats.get("total_original_tokens", 0),
- "compressed_tokens_cached": compression_stats.get("total_compressed_tokens", 0),
- "ccr_retrievals": compression_stats.get("total_retrievals", 0),
- },
- "compression_cache": compression_cache_stats,
- "anon_telemetry_shipping": is_telemetry_enabled(),
- "telemetry": {
- "enabled": telemetry_stats.get("enabled", False),
- "total_compressions": telemetry_stats.get("total_compressions", 0),
- "total_retrievals": telemetry_stats.get("total_retrievals", 0),
- "global_retrieval_rate": round(telemetry_stats.get("global_retrieval_rate", 0), 4),
- "tool_signatures_tracked": telemetry_stats.get("tool_signatures_tracked", 0),
- "avg_compression_ratio": round(telemetry_stats.get("avg_compression_ratio", 0), 4),
- "avg_token_reduction": round(telemetry_stats.get("avg_token_reduction", 0), 4),
- },
- "otel": get_otel_metrics_status(),
- "langfuse": get_langfuse_tracing_status(),
- "feedback_loop": {
- "tools_tracked": feedback_stats.get("tools_tracked", 0),
- "total_compressions": feedback_stats.get("total_compressions", 0),
- "total_retrievals": feedback_stats.get("total_retrievals", 0),
- "global_retrieval_rate": round(feedback_stats.get("global_retrieval_rate", 0), 4),
- "tools_with_high_retrieval": sum(
- 1
- for p in feedback_stats.get("tool_patterns", {}).values()
- if p.get("retrieval_rate", 0) > 0.3
- ),
- },
- "toin": get_toin().get_stats(),
- "cli_filtering": cli_filtering_stats,
- "cache": await proxy.cache.stats() if proxy.cache else None,
- "rate_limiter": await proxy.rate_limiter.stats() if proxy.rate_limiter else None,
- "recent_requests": proxy.logger.get_recent(10) if proxy.logger else [],
- "log_full_messages": proxy.config.log_full_messages if proxy else False,
- **get_quota_registry().get_all_stats(),
- }
-
- @app.get("/stats-history")
- async def stats_history(
- format: Literal["json", "csv"] = "json",
- series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
- ):
- """Get durable proxy compression history plus display-session state."""
- if format == "csv":
- filename = f"headroom-stats-history-{series}.csv"
- return Response(
- content=proxy.metrics.savings_tracker.export_csv(series=series),
- media_type="text/csv; charset=utf-8",
- headers={"Content-Disposition": f'attachment; filename="{filename}"'},
- )
-
- return proxy.metrics.savings_tracker.history_response()
-
- @app.get("/transformations/feed")
- async def transformations_feed(limit: int = 20):
- """Get recent message transformations for the live feed.
-
- Returns empty list if log_full_messages is disabled (messages are not stored).
- """
- if limit > 100:
- limit = 100
-
- transformations = []
- log_full_messages = proxy.config.log_full_messages if proxy else False
-
- if proxy and proxy.logger:
- logs = proxy.logger.get_recent_with_messages(limit)
- for log in logs:
- transformations.append(
- {
- "request_id": log.get("request_id"),
- "timestamp": log.get("timestamp"),
- "provider": log.get("provider"),
- "model": log.get("model"),
- "input_tokens_original": log.get("input_tokens_original"),
- "input_tokens_optimized": log.get("input_tokens_optimized"),
- "tokens_saved": log.get("tokens_saved"),
- "savings_percent": log.get("savings_percent"),
- "transforms_applied": log.get("transforms_applied", []),
- "request_messages": log.get("request_messages"),
- "response_content": log.get("response_content"),
- }
- )
-
- return {"transformations": transformations, "log_full_messages": log_full_messages}
-
- @app.get("/subscription-window")
- async def subscription_window():
- """Current Anthropic subscription window utilisation and Headroom contribution."""
- tracker = get_subscription_tracker()
- if tracker is None:
- return JSONResponse(
- status_code=503,
- content={"error": "Subscription tracking is not enabled"},
- )
- return JSONResponse(content=tracker.state)
-
- @app.get("/quota")
- async def quota():
- """Unified quota/rate-limit stats for all registered providers (Anthropic, Codex, Copilot)."""
- return JSONResponse(content=get_quota_registry().get_all_stats())
-
- @app.get("/metrics")
- async def metrics():
- """Prometheus metrics endpoint."""
- return PlainTextResponse(
- await proxy.metrics.export(),
- media_type="text/plain; version=0.0.4",
- )
-
- # Debug endpoints
- @app.get("/debug/memory")
- async def debug_memory():
- """Get detailed memory usage statistics.
-
- Returns memory usage for all tracked components including:
- - Process-level memory (RSS, VMS, percent)
- - Per-component memory usage and budgets
- - Cache hit/miss statistics
- - Total tracked vs target budget
-
- This endpoint is useful for debugging memory issues and
- monitoring memory budgets.
- """
- from ..memory.tracker import MemoryTracker
-
- tracker = MemoryTracker.get()
-
- # Register components if not already registered
- _register_memory_components(proxy, tracker)
-
- report = tracker.get_report()
- return report.to_dict()
-
- @app.post("/cache/clear")
- async def clear_cache():
- """Clear the response cache."""
- if proxy.cache:
- await proxy.cache.clear()
- return {"status": "cleared"}
- return {"status": "cache disabled"}
-
- # CCR (Compress-Cache-Retrieve) endpoints
- @app.post("/v1/retrieve")
- async def ccr_retrieve(request: Request):
- """Retrieve original content from CCR compression cache.
-
- This is the "Retrieve" part of CCR (Compress-Cache-Retrieve).
- When SmartCrusher compresses tool outputs, the original data is cached.
- LLMs can call this endpoint to get more data if needed.
-
- Request body:
- hash (str): Hash key from compression marker (required)
- query (str): Optional search query to filter results
-
- Response:
- Full retrieval: {"hash": "...", "original_content": "...", ...}
- Search: {"hash": "...", "query": "...", "results": [...], "count": N}
- """
- data = await request.json()
- hash_key = data.get("hash")
- query = data.get("query")
-
- if not hash_key:
- raise HTTPException(status_code=400, detail="hash required")
-
- store = get_compression_store()
-
- if query:
- # Search within cached content
- results = store.search(hash_key, query)
- return {
- "hash": hash_key,
- "query": query,
- "results": results,
- "count": len(results),
- }
- else:
- # Return full original content
- entry = store.retrieve(hash_key)
- if entry:
- return {
- "hash": hash_key,
- "original_content": entry.original_content,
- "original_tokens": entry.original_tokens,
- "original_item_count": entry.original_item_count,
- "compressed_item_count": entry.compressed_item_count,
- "tool_name": entry.tool_name,
- "retrieval_count": entry.retrieval_count,
- }
- raise HTTPException(
- status_code=404, detail="Entry not found or expired (TTL: 5 minutes)"
- )
-
- @app.get("/v1/retrieve/stats")
- async def ccr_stats():
- """Get CCR compression store statistics."""
- store = get_compression_store()
- stats = store.get_stats()
- events = store.get_retrieval_events(limit=20)
- return {
- "store": stats,
- "recent_retrievals": [
- {
- "hash": e.hash,
- "query": e.query,
- "items_retrieved": e.items_retrieved,
- "total_items": e.total_items,
- "tool_name": e.tool_name,
- "retrieval_type": e.retrieval_type,
- }
- for e in events
- ],
- }
-
- @app.get("/v1/feedback")
- async def ccr_feedback():
- """Get CCR feedback loop statistics and learned patterns.
-
- This endpoint exposes the feedback loop's learned patterns for monitoring
- and debugging. It shows:
- - Per-tool retrieval rates (high = compress less aggressively)
- - Common search queries per tool
- - Queried fields (suggest what to preserve)
-
- Use this to understand how well compression is working and whether
- the feedback loop is adjusting appropriately.
- """
- feedback = get_compression_feedback()
- stats = feedback.get_stats()
- return {
- "feedback": stats,
- "hints_example": {
- tool_name: {
- "hints": {
- "max_items": hints.max_items
- if (hints := feedback.get_compression_hints(tool_name))
- else 15,
- "suggested_items": hints.suggested_items if hints else None,
- "skip_compression": hints.skip_compression if hints else False,
- "preserve_fields": hints.preserve_fields if hints else [],
- "reason": hints.reason if hints else "",
- }
- }
- for tool_name in list(stats.get("tool_patterns", {}).keys())[:5]
- },
- }
-
- @app.get("/v1/feedback/{tool_name}")
- async def ccr_feedback_for_tool(tool_name: str):
- """Get compression hints for a specific tool.
-
- Returns feedback-based hints that would be used for compressing
- this tool's output.
- """
- feedback = get_compression_feedback()
- hints = feedback.get_compression_hints(tool_name)
- patterns = feedback.get_all_patterns().get(tool_name)
-
- return {
- "tool_name": tool_name,
- "hints": {
- "max_items": hints.max_items,
- "min_items": hints.min_items,
- "suggested_items": hints.suggested_items,
- "aggressiveness": hints.aggressiveness,
- "skip_compression": hints.skip_compression,
- "preserve_fields": hints.preserve_fields,
- "reason": hints.reason,
- },
- "pattern": {
- "total_compressions": patterns.total_compressions if patterns else 0,
- "total_retrievals": patterns.total_retrievals if patterns else 0,
- "retrieval_rate": patterns.retrieval_rate if patterns else 0.0,
- "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0,
- "search_rate": patterns.search_rate if patterns else 0.0,
- "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [],
- "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [],
- }
- if patterns
- else None,
- }
-
- # Telemetry endpoints (Data Flywheel)
- @app.get("/v1/telemetry")
- async def telemetry_stats():
- """Get telemetry statistics for the data flywheel.
-
- This endpoint exposes privacy-preserving telemetry data that powers
- the data flywheel - learning optimal compression strategies across
- tool types based on usage patterns.
-
- What's collected (anonymized):
- - Tool output structure patterns (field types, not values)
- - Compression decisions and ratios
- - Retrieval patterns (rate, type, not content)
- - Strategy effectiveness
-
- What's NOT collected:
- - Actual data values
- - User identifiers
- - Queries or search terms
- - File paths or tool names (hashed by default)
- """
- telemetry = get_telemetry_collector()
- return telemetry.get_stats()
-
- @app.get("/v1/telemetry/export")
- async def telemetry_export():
- """Export full telemetry data for aggregation.
-
- This endpoint exports all telemetry data in a format suitable for
- cross-user aggregation. The data is privacy-preserving - no actual
- values are included, only structural patterns and statistics.
-
- Use this for:
- - Building a central learning service
- - Sharing learned patterns across instances
- - Analysis and debugging
- """
- telemetry = get_telemetry_collector()
- return telemetry.export_stats()
-
- @app.post("/v1/telemetry/import")
- async def telemetry_import(request: Request):
- """Import telemetry data from another source.
-
- This allows merging telemetry from multiple sources for cross-user
- learning. The imported data is merged with existing statistics.
-
- Request body: Telemetry export data from /v1/telemetry/export
- """
- telemetry = get_telemetry_collector()
- data = await request.json()
- telemetry.import_stats(data)
- return {"status": "imported", "current_stats": telemetry.get_stats()}
-
- @app.get("/v1/telemetry/tools")
- async def telemetry_tools():
- """Get telemetry statistics for all tracked tool signatures.
-
- Returns statistics per tool signature (anonymized), including:
- - Compression ratios and strategy usage
- - Retrieval rates (high = compression too aggressive)
- - Learned recommendations
- """
- telemetry = get_telemetry_collector()
- all_stats = telemetry.get_all_tool_stats()
- return {
- "tool_count": len(all_stats),
- "tools": {sig_hash: stats.to_dict() for sig_hash, stats in all_stats.items()},
- }
-
- @app.get("/v1/telemetry/tools/{signature_hash}")
- async def telemetry_tool_detail(signature_hash: str):
- """Get detailed telemetry for a specific tool signature.
-
- Includes learned recommendations if enough data has been collected.
- """
- telemetry = get_telemetry_collector()
- stats = telemetry.get_tool_stats(signature_hash)
- recommendations = telemetry.get_recommendations(signature_hash)
-
- if stats is None:
- raise HTTPException(
- status_code=404, detail=f"No telemetry found for signature: {signature_hash}"
- )
-
- return {
- "signature_hash": signature_hash,
- "stats": stats.to_dict(),
- "recommendations": recommendations,
- }
-
- # TOIN (Tool Output Intelligence Network) endpoints
- @app.get("/v1/toin/stats")
- async def toin_stats():
- """Get overall TOIN statistics.
-
- Returns aggregated statistics from the Tool Output Intelligence Network,
- which learns optimal compression strategies across all tool types.
-
- Response includes:
- - enabled: Whether TOIN is enabled
- - patterns_tracked: Number of unique tool patterns being tracked
- - total_compressions: Total compression events recorded
- - total_retrievals: Total retrieval events recorded
- - global_retrieval_rate: Overall retrieval rate (high = compression too aggressive)
- - patterns_with_recommendations: Patterns with enough data for recommendations
- """
- toin = get_toin()
- return toin.get_stats()
-
- @app.get("/v1/toin/patterns")
- async def toin_patterns(limit: int = 20):
- """List TOIN patterns with most samples.
-
- Returns patterns sorted by sample_size descending. Use this to see
- which tool types have the most data and their learned behaviors.
-
- Query params:
- limit: Maximum number of patterns to return (default 20)
-
- Response includes for each pattern:
- - hash: Truncated tool signature hash (12 chars)
- - compressions: Total compression events
- - retrievals: Total retrieval events
- - retrieval_rate: Percentage of compressions that triggered retrieval
- - confidence: Confidence level in recommendations (0.0-1.0)
- - skip_recommended: Whether TOIN recommends skipping compression
- - optimal_max_items: Learned optimal max_items setting
- """
- toin = get_toin()
- exported = toin.export_patterns()
- patterns_data = exported.get("patterns", {})
-
- # Convert to list and sort by sample_size
- patterns_list = []
- for sig_hash, pattern_dict in patterns_data.items():
- sample_size = pattern_dict.get("sample_size", 0)
- total_compressions = pattern_dict.get("total_compressions", 0)
- total_retrievals = pattern_dict.get("total_retrievals", 0)
- retrieval_rate = (
- total_retrievals / total_compressions if total_compressions > 0 else 0.0
- )
-
- patterns_list.append(
- {
- "hash": sig_hash[:12],
- "compressions": total_compressions,
- "retrievals": total_retrievals,
- "retrieval_rate": f"{retrieval_rate:.1%}",
- "confidence": round(pattern_dict.get("confidence", 0.0), 3),
- "skip_recommended": pattern_dict.get("skip_compression_recommended", False),
- "optimal_max_items": pattern_dict.get("optimal_max_items", 20),
- "sample_size": sample_size,
- }
- )
-
- # Sort by sample_size descending
- patterns_list.sort(key=lambda p: p["sample_size"], reverse=True)
-
- # Remove sample_size from output (used only for sorting)
- for p in patterns_list:
- del p["sample_size"]
-
- return patterns_list[:limit]
-
- @app.get("/v1/toin/pattern/{hash_prefix}")
- async def toin_pattern_detail(hash_prefix: str):
- """Get detailed TOIN pattern info by hash prefix.
-
- Searches for a pattern where the tool signature hash starts with
- the provided prefix. Returns full pattern details if found.
-
- Path params:
- hash_prefix: Beginning of the tool signature hash (min 4 chars recommended)
-
- Response: Full pattern.to_dict() with all learned statistics and recommendations.
- """
- toin = get_toin()
- exported = toin.export_patterns()
- patterns_data = exported.get("patterns", {})
-
- # Search for pattern with matching hash prefix
- for sig_hash, pattern_dict in patterns_data.items():
- if sig_hash.startswith(hash_prefix):
- return pattern_dict
-
- raise HTTPException(
- status_code=404, detail=f"No TOIN pattern found with hash starting with: {hash_prefix}"
- )
-
- @app.get("/v1/retrieve/{hash_key}")
- async def ccr_retrieve_get(hash_key: str, query: str | None = None):
- """GET version of CCR retrieve for easier testing."""
- store = get_compression_store()
-
- if query:
- results = store.search(hash_key, query)
- return {
- "hash": hash_key,
- "query": query,
- "results": results,
- "count": len(results),
- }
- else:
- entry = store.retrieve(hash_key)
- if entry:
- return {
- "hash": hash_key,
- "original_content": entry.original_content,
- "original_tokens": entry.original_tokens,
- "original_item_count": entry.original_item_count,
- "compressed_item_count": entry.compressed_item_count,
- "tool_name": entry.tool_name,
- "retrieval_count": entry.retrieval_count,
- }
- raise HTTPException(status_code=404, detail="Entry not found or expired")
-
- # CCR Tool Call Handler - for agent frameworks to call when LLM uses headroom_retrieve
- @app.post("/v1/retrieve/tool_call")
- async def ccr_handle_tool_call(request: Request):
- """Handle a CCR tool call from an LLM response.
-
- This endpoint accepts tool call formats from various providers and returns
- a properly formatted tool result. Agent frameworks can use this to handle
- CCR tool calls without implementing the retrieval logic themselves.
-
- Request body (Anthropic format):
- {
- "tool_call": {
- "id": "toolu_123",
- "name": "headroom_retrieve",
- "input": {"hash": "abc123", "query": "optional search"}
- },
- "provider": "anthropic"
- }
-
- Request body (OpenAI format):
- {
- "tool_call": {
- "id": "call_123",
- "function": {
- "name": "headroom_retrieve",
- "arguments": "{\"hash\": \"abc123\"}"
- }
- },
- "provider": "openai"
- }
-
- Response:
- {
- "tool_result": {...}, # Formatted for the provider
- "success": true,
- "data": {...} # Raw retrieval data
- }
- """
- data = await request.json()
- tool_call = data.get("tool_call", {})
- provider = data.get("provider", "anthropic")
-
- # Parse the tool call
- hash_key, query = parse_tool_call(tool_call, provider)
-
- if hash_key is None:
- raise HTTPException(
- status_code=400, detail=f"Invalid tool call or not a {CCR_TOOL_NAME} call"
- )
-
- # Perform retrieval
- store = get_compression_store()
-
- if query:
- results = store.search(hash_key, query)
- retrieval_data = {
- "hash": hash_key,
- "query": query,
- "results": results,
- "count": len(results),
- }
- else:
- entry = store.retrieve(hash_key)
- if entry:
- retrieval_data = {
- "hash": hash_key,
- "original_content": entry.original_content,
- "original_item_count": entry.original_item_count,
- "compressed_item_count": entry.compressed_item_count,
- }
- else:
- retrieval_data = {
- "error": "Entry not found or expired (TTL: 5 minutes)",
- "hash": hash_key,
- }
-
- # Format tool result for provider
- tool_call_id = tool_call.get("id", "")
- result_content = json.dumps(retrieval_data, indent=2)
-
- if provider == "anthropic":
- tool_result = {
- "type": "tool_result",
- "tool_use_id": tool_call_id,
- "content": result_content,
- }
- elif provider == "openai":
- tool_result = {
- "role": "tool",
- "tool_call_id": tool_call_id,
- "content": result_content,
- }
- else:
- tool_result = {
- "tool_call_id": tool_call_id,
- "content": result_content,
- }
-
- return {
- "tool_result": tool_result,
- "success": "error" not in retrieval_data,
- "data": retrieval_data,
- }
-
- # Compression-only endpoint (for TypeScript SDK and other HTTP clients)
- @app.post("/v1/compress")
- async def compress_messages(request: Request):
- return await proxy.handle_compress(request)
-
- register_provider_routes(app, proxy)
-
- return app
-
-
-def _get_code_aware_banner_status(config: ProxyConfig) -> str:
- """Get code-aware compression status line for banner."""
- if config.code_aware_enabled:
- if is_tree_sitter_available():
- return "ENABLED (AST-based)"
- else:
- return "NOT INSTALLED (pip install headroom-ai[code])"
- else:
- if is_tree_sitter_available():
- return "DISABLED (remove --no-code-aware to enable)"
- return "DISABLED"
-
-
-def run_server(
- config: ProxyConfig | None = None,
- workers: int = 1,
- limit_concurrency: int = 1000,
-):
- """Run the proxy server.
-
- Args:
- config: Proxy configuration
- workers: Number of worker processes (use N for multi-core scaling)
- limit_concurrency: Max concurrent connections before 503 response
- """
- if not FASTAPI_AVAILABLE:
- print("ERROR: FastAPI required. Install: pip install fastapi uvicorn httpx")
- sys.exit(1)
-
- config = config or ProxyConfig()
- app = create_app(config)
-
- code_aware_status = _get_code_aware_banner_status(config)
-
- # Format connection pool info
- pool_info = f"max={config.max_connections}, keepalive={config.max_keepalive_connections}"
- http2_status = "ENABLED" if config.http2 else "DISABLED"
-
- backend_status = format_backend_status(
- backend=config.backend,
- anyllm_provider=config.anyllm_provider,
- bedrock_region=config.bedrock_region,
- )
-
- print(f"""
-╔══════════════════════════════════════════════════════════════════════╗
-║ HEADROOM PROXY SERVER ║
-╠══════════════════════════════════════════════════════════════════════╣
-║ Version: 1.0.0 ║
-║ Listening: http://{config.host}:{config.port:<5} ║
-║ Workers: {workers:<3} Concurrency Limit: {limit_concurrency:<5} ║
-║ Backend: {backend_status:<59}║
-╠══════════════════════════════════════════════════════════════════════╣
-║ FEATURES: ║
-║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║
-║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║
-║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║
-║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║
-║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║
-║ Code-Aware: {code_aware_status:<52}║
-║ HTTP/2: {http2_status:<52}║
-║ Conn Pool: {pool_info:<52}║
-╠══════════════════════════════════════════════════════════════════════╣
-║ USAGE: ║
-║ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude ║
-║ Cursor: Set base URL in settings ║
-╠══════════════════════════════════════════════════════════════════════╣
-║ ENDPOINTS: ║
-║ /livez Process liveness ║
-║ /readyz Traffic readiness ║
-║ /health Aggregate health ║
-║ /stats Detailed statistics ║
-║ /metrics Prometheus metrics ║
-║ /cache/clear Clear response cache ║
-║ /v1/retrieve CCR: Retrieve compressed content ║
-║ /v1/retrieve/stats CCR: Compression store stats ║
-║ /v1/retrieve/tool_call CCR: Handle LLM tool calls ║
-║ /v1/feedback CCR: Feedback loop stats & patterns ║
-║ /v1/feedback/{{tool}} CCR: Compression hints for a tool ║
-║ /v1/telemetry Data flywheel: Telemetry stats ║
-║ /v1/telemetry/export Data flywheel: Export for aggregation ║
-║ /v1/telemetry/tools Data flywheel: Per-tool stats ║
-║ /v1/toin/stats TOIN: Overall intelligence stats ║
-║ /v1/toin/patterns TOIN: List learned patterns ║
-║ /v1/toin/pattern/{{hash}} TOIN: Pattern details by hash ║
-╚══════════════════════════════════════════════════════════════════════╝
-""")
-
- uvicorn.run(
- app,
- host=config.host,
- port=config.port,
- log_level="warning",
- workers=workers if workers > 1 else None, # None = single process (default)
- limit_concurrency=limit_concurrency,
- # Defense-in-depth: the loopback guard for /debug/* endpoints trusts
- # request.client.host. uvicorn's ProxyHeadersMiddleware rewrites that
- # from X-Forwarded-For when FORWARDED_ALLOW_IPS is broader than the
- # default. Disabling proxy_headers here guarantees the guard sees the
- # real peer address regardless of env.
- proxy_headers=False,
- )
-
-
-def _get_env_bool(name: str, default: bool) -> bool:
- """Get boolean from environment variable."""
- val = os.environ.get(name)
- if val is None:
- return default
- return val.lower() in ("true", "1", "yes", "on")
-
-
-def _get_env_int(name: str, default: int) -> int:
- """Get integer from environment variable."""
- val = os.environ.get(name)
- if val is None:
- return default
- try:
- return int(val)
- except ValueError:
- return default
-
-
-def _get_env_float(name: str, default: float) -> float:
- """Get float from environment variable."""
- val = os.environ.get(name)
- if val is None:
- return default
- try:
- return float(val)
- except ValueError:
- return default
-
-
-def _get_env_str(name: str, default: str) -> str:
- """Get string from environment variable."""
- return os.environ.get(name, default)
-
-
-def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:
- """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.
-
- Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)
- Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")
-
- Returns:
- Dict mapping tool names to CompressionProfile instances.
- """
- from headroom.config import PROFILE_PRESETS, CompressionProfile
-
- profiles: dict[str, CompressionProfile] = {}
- raw_entries: list[str] = list(cli_profiles)
-
- # Also check env var
- env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")
- if env_val:
- raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())
-
- for entry in raw_entries:
- if ":" not in entry:
- logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)
- continue
- tool_name, level = entry.split(":", 1)
- tool_name = tool_name.strip()
- level = level.strip().lower()
-
- if level in PROFILE_PRESETS:
- profiles[tool_name] = PROFILE_PRESETS[level]
- else:
- logger.warning(
- "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",
- level,
- tool_name,
- )
-
- return profiles
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Headroom Proxy Server")
-
- # Server
- parser.add_argument("--host", default="127.0.0.1")
- parser.add_argument("--port", type=int, default=8787)
- parser.add_argument(
- "--openai-api-url", help=f"Custom OpenAI API URL (default: {DEFAULT_OPENAI_API_URL})"
- )
- parser.add_argument(
- "--anthropic-api-url",
- help=f"Custom Anthropic API URL (default: {DEFAULT_ANTHROPIC_API_URL})",
- )
-
- # Backend (anthropic direct, bedrock, openrouter, anyllm, or litellm-<provider>)
- parser.add_argument(
- "--backend",
- default="anthropic",
- help=(
- "Backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', "
- "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-hosted_vllm, litellm-vertex)"
- ),
- )
- parser.add_argument(
- "--bedrock-region",
- default="us-west-2",
- help="AWS region for Bedrock backend (default: us-west-2)",
- )
- parser.add_argument(
- "--bedrock-profile",
- help="AWS profile for Bedrock backend (default: use default credentials)",
- )
- parser.add_argument(
- "--openrouter-api-key",
- help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",
- )
- parser.add_argument(
- "--anyllm-provider",
- default="openai",
- help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",
- )
-
- # Connection pool (scalability)
- parser.add_argument(
- "--max-connections",
- type=int,
- default=500,
- help="Max connections to upstream APIs (default: 500)",
- )
- parser.add_argument(
- "--max-keepalive", type=int, default=100, help="Max keepalive connections (default: 100)"
- )
- parser.add_argument(
- "--no-http2",
- action="store_true",
- help="Disable HTTP/2 (enabled by default for better throughput)",
- )
- parser.add_argument(
- "--workers",
- type=int,
- default=1,
- help="Number of worker processes (default: 1, use N for multi-core)",
- )
- parser.add_argument(
- "--limit-concurrency",
- type=int,
- default=1000,
- help="Max concurrent connections before 503 (default: 1000)",
- )
-
- # Optimization
- parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")
- parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")
- parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")
- parser.add_argument(
- "--tool-profile",
- action="append",
- default=[],
- help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "
- "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",
- )
-
- # Caching
- parser.add_argument("--no-cache", action="store_true", help="Disable caching")
- parser.add_argument("--cache-ttl", type=int, default=3600, help="Cache TTL seconds")
-
- # Rate limiting
- parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")
- parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")
- parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")
-
- # Cost
- parser.add_argument("--budget", type=float, help="Budget limit in USD")
- parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")
-
- # Logging
- parser.add_argument("--log-file", help="Log file path")
- parser.add_argument("--log-messages", action="store_true", help="Log full messages")
-
- # Smart routing (content-aware compression)
- parser.add_argument(
- "--no-smart-routing",
- action="store_true",
- help="Disable smart routing (use legacy sequential pipeline)",
- )
-
- # Code-aware compression
- parser.add_argument(
- "--code-aware",
- action="store_true",
- help="Enable AST-based code compression (requires: pip install headroom-ai[code])",
- )
- parser.add_argument(
- "--no-code-aware",
- action="store_true",
- help="Disable code-aware compression",
- )
-
- args = parser.parse_args()
-
- # Environment variable defaults (HEADROOM_* prefix)
- # CLI args override env vars, env vars override ProxyConfig defaults
- env_smart_routing = _get_env_bool("HEADROOM_SMART_ROUTING", True)
- env_code_aware = _get_env_bool("HEADROOM_CODE_AWARE_ENABLED", True)
- env_optimize = _get_env_bool("HEADROOM_OPTIMIZE", True)
- env_cache = _get_env_bool("HEADROOM_CACHE_ENABLED", True)
- env_rate_limit = _get_env_bool("HEADROOM_RATE_LIMIT_ENABLED", True)
-
- # Determine settings: CLI flags override env vars
- # --no-X explicitly disables, --X explicitly enables, neither uses env var
- smart_routing = env_smart_routing if not args.no_smart_routing else False
- code_aware_enabled = (
- env_code_aware
- if not (args.code_aware or args.no_code_aware)
- else (args.code_aware or not args.no_code_aware)
- )
- optimize = env_optimize if not args.no_optimize else False
- cache_enabled = env_cache if not args.no_cache else False
- rate_limit_enabled = env_rate_limit if not args.no_rate_limit else False
-
- # Set OpenRouter API key from CLI if provided
- if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:
- os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key
-
- # Parse per-tool compression profiles from CLI and env var
- tool_profiles = _parse_tool_profiles(args.tool_profile)
-
- config = ProxyConfig(
- host=_get_env_str("HEADROOM_HOST", args.host),
- port=_get_env_int("HEADROOM_PORT", args.port),
- openai_api_url=_get_env_str("OPENAI_TARGET_API_URL", args.openai_api_url),
- anthropic_api_url=_get_env_str("ANTHROPIC_TARGET_API_URL", args.anthropic_api_url),
- # Backend settings
- backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]
- bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),
- bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),
- anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),
- optimize=optimize,
- min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),
- max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),
- cache_enabled=cache_enabled,
- cache_ttl_seconds=_get_env_int("HEADROOM_CACHE_TTL", args.cache_ttl),
- rate_limit_enabled=rate_limit_enabled,
- rate_limit_requests_per_minute=_get_env_int("HEADROOM_RPM", args.rpm),
- rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),
- budget_limit_usd=args.budget,
- budget_period=args.budget_period,
- log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)
- if args.log_file
- else os.environ.get("HEADROOM_LOG_FILE"),
- log_full_messages=args.log_messages or _get_env_bool("HEADROOM_LOG_MESSAGES", False),
- smart_routing=smart_routing,
- code_aware_enabled=code_aware_enabled,
- # Connection pool settings
- max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),
- max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),
- http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),
- tool_profiles=tool_profiles if tool_profiles else None,
- mode=normalize_proxy_mode(_get_env_str("HEADROOM_MODE", PROXY_MODE_TOKEN)),
- )
-
- # Get worker and concurrency settings
- workers = _get_env_int("HEADROOM_WORKERS", args.workers)
- limit_concurrency = _get_env_int("HEADROOM_LIMIT_CONCURRENCY", args.limit_concurrency)
-
- run_server(config, workers=workers, limit_concurrency=limit_concurrency)
+"""Headroom Proxy Server - Production Ready.
+
+A full-featured LLM proxy with optimization, caching, rate limiting,
+and observability.
+
+Features:
+- Context optimization (SmartCrusher, CacheAligner, RollingWindow)
+- Semantic caching (save costs on repeated queries)
+- Rate limiting (token bucket)
+- Retry with exponential backoff
+- Cost tracking and budgets
+- Request tagging and metadata
+- Provider fallback
+- Prometheus metrics
+- Full request/response logging
+
+Usage:
+ python -m headroom.proxy.server --port 8787
+
+ # With Claude Code:
+ ANTHROPIC_BASE_URL=http://localhost:8787 claude
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+if TYPE_CHECKING:
+ from ..backends.base import Backend
+ from ..cache.compression_cache import CompressionCache
+ from ..memory.tracker import MemoryTracker
+
+
+import httpx
+
+try:
+ import uvicorn
+ from fastapi import Depends, FastAPI, HTTPException, Request, Response
+ from fastapi.middleware.cors import CORSMiddleware
+ from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
+
+ FASTAPI_AVAILABLE = True
+except ImportError:
+ FASTAPI_AVAILABLE = False
+
+# Add parent to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from headroom._version import __version__
+from headroom.cache.compression_feedback import get_compression_feedback
+from headroom.cache.compression_store import get_compression_store
+from headroom.ccr import (
+ CCR_TOOL_NAME,
+ # Batch processing
+ CCRResponseHandler,
+ CCRToolInjector,
+ ContextTracker,
+ ContextTrackerConfig,
+ ResponseHandlerConfig,
+ parse_tool_call,
+)
+from headroom.config import (
+ CacheAlignerConfig,
+ CCRConfig,
+ IntelligentContextConfig,
+ ReadLifecycleConfig,
+ RollingWindowConfig,
+ SmartCrusherConfig,
+)
+from headroom.dashboard import get_dashboard_html
+from headroom.observability import (
+ LangfuseTracingConfig,
+ OTelMetricsConfig,
+ configure_langfuse_tracing,
+ configure_otel_metrics,
+ get_langfuse_tracing_status,
+ get_otel_metrics_status,
+ shutdown_headroom_tracing,
+ shutdown_otel_metrics,
+)
+from headroom.pipeline import PipelineExtensionManager, PipelineStage
+from headroom.providers.proxy_routes import register_provider_routes
+from headroom.providers.registry import (
+ DEFAULT_ANTHROPIC_API_URL,
+ DEFAULT_CLOUDCODE_API_URL,
+ DEFAULT_GEMINI_API_URL,
+ DEFAULT_OPENAI_API_URL,
+ build_proxy_provider_runtime,
+ create_proxy_backend,
+ format_backend_status,
+)
+
+# =============================================================================
+# Extracted modules (re-exported for backward compatibility)
+# =============================================================================
+from headroom.proxy.cost import (
+ _CACHE_ECONOMICS, # noqa: F401
+ CostTracker, # noqa: F401
+ _summarize_transforms, # noqa: F401
+ build_prefix_cache_stats, # noqa: F401
+ build_session_summary, # noqa: F401
+ merge_cost_stats, # noqa: F401
+)
+from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS, # noqa: F401
+ MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401
+ MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401
+ MAX_REQUEST_BODY_SIZE, # noqa: F401
+ MAX_SSE_BUFFER_SIZE, # noqa: F401
+ _get_image_compressor, # noqa: F401
+ _get_rtk_stats, # noqa: F401
+ _read_request_json, # noqa: F401
+ _setup_file_logging, # noqa: F401
+ is_anthropic_auth, # noqa: F401
+ jitter_delay_ms,
+)
+from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
+
+# Data models (extracted to headroom/proxy/models.py for maintainability)
+from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
+from headroom.proxy.modes import (
+ PROXY_MODE_CACHE,
+ PROXY_MODE_TOKEN,
+ is_token_mode,
+ normalize_proxy_mode,
+)
+from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
+from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
+from headroom.proxy.request_logger import RequestLogger # noqa: F401
+from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
+from headroom.proxy.warmup import WarmupRegistry
+from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
+from headroom.subscription.base import get_quota_registry, reset_quota_registry
+from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state
+from headroom.subscription.copilot_quota import get_copilot_quota_tracker
+from headroom.subscription.tracker import (
+ configure_subscription_tracker,
+ get_subscription_tracker,
+)
+from headroom.telemetry import get_telemetry_collector
+from headroom.telemetry.beacon import is_telemetry_enabled
+from headroom.telemetry.toin import get_toin
+from headroom.transforms import (
+ CacheAligner,
+ CodeAwareCompressor,
+ CodeCompressorConfig,
+ ContentRouter,
+ ContentRouterConfig,
+ IntelligentContextManager,
+ RollingWindow,
+ SmartCrusher,
+ Transform,
+ TransformPipeline,
+ is_tree_sitter_available,
+)
+
+AnyLLMBackend: Any = None
+LiteLLMBackend: Any = None
+
+fcntl: Any = None
+try:
+ import fcntl as _fcntl
+
+ fcntl = _fcntl
+ HAS_FCNTL = True
+except ImportError:
+ HAS_FCNTL = False
+
+_build_prefix_cache_stats = build_prefix_cache_stats
+_build_session_summary = build_session_summary
+_merge_cost_stats = merge_cost_stats
+
+
+logging.basicConfig(
+ level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger("headroom.proxy")
+
+# Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis
+_setup_file_logging()
+
+
+# Compression pipeline timeout in seconds
+
+
+from headroom.proxy.handlers import ( # noqa: E402
+ AnthropicHandlerMixin,
+ BatchHandlerMixin,
+ GeminiHandlerMixin,
+ OpenAIHandlerMixin,
+ StreamingMixin,
+)
+
+
+class HeadroomProxy(
+ StreamingMixin,
+ AnthropicHandlerMixin,
+ OpenAIHandlerMixin,
+ GeminiHandlerMixin,
+ BatchHandlerMixin,
+):
+ """Production-ready Headroom optimization proxy."""
+
+ ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL
+ OPENAI_API_URL = DEFAULT_OPENAI_API_URL
+ GEMINI_API_URL = DEFAULT_GEMINI_API_URL
+ CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL
+
+ def __init__(self, config: ProxyConfig):
+ self.config = config
+ self.config.mode = normalize_proxy_mode(self.config.mode)
+ self.pipeline_extensions = PipelineExtensionManager(
+ hooks=config.hooks,
+ extensions=config.pipeline_extensions,
+ discover=config.discover_pipeline_extensions,
+ )
+
+ self.provider_runtime = build_proxy_provider_runtime(config)
+ api_targets = self.provider_runtime.api_targets
+
+ # Preserve the long-standing proxy compatibility surface while keeping
+ # provider_runtime as the source of truth for resolved upstream targets.
+ HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic
+ HeadroomProxy.OPENAI_API_URL = api_targets.openai
+ HeadroomProxy.GEMINI_API_URL = api_targets.gemini
+ HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode
+ self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")
+ self.openai_provider = self.provider_runtime.pipeline_provider("openai")
+
+ # Initialize transforms based on routing mode
+ # Choose context manager: IntelligentContextManager (smart) or RollingWindow (legacy)
+ context_manager: Transform # Can be either IntelligentContextManager or RollingWindow
+ if config.intelligent_context:
+ # Get TOIN instance for learned pattern integration
+ toin = get_toin() if config.intelligent_context_scoring else None
+ context_manager = IntelligentContextManager(
+ config=IntelligentContextConfig(
+ enabled=True,
+ keep_system=True,
+ keep_last_turns=config.keep_last_turns,
+ use_importance_scoring=config.intelligent_context_scoring,
+ toin_integration=config.intelligent_context_scoring,
+ compress_threshold=0.10 if config.intelligent_context_compress_first else 0.0,
+ ),
+ toin=toin,
+ )
+ self._context_manager_status = "intelligent"
+ else:
+ context_manager = RollingWindow(
+ RollingWindowConfig(
+ enabled=True,
+ keep_system=True,
+ keep_last_turns=config.keep_last_turns,
+ )
+ )
+ self._context_manager_status = "rolling_window"
+
+ if config.smart_routing:
+ # Smart routing: ContentRouter handles all content types intelligently
+ # It lazy-loads compressors only when needed
+ router_config = ContentRouterConfig(
+ enable_code_aware=config.code_aware_enabled,
+ tool_profiles=config.tool_profiles,
+ read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),
+ )
+ # Token mode: allow compression of older excluded-tool results
+ if is_token_mode(config.mode):
+ router_config.protect_recent_reads_fraction = 0.3
+ transforms = [
+ CacheAligner(CacheAlignerConfig(enabled=False)),
+ ContentRouter(router_config),
+ context_manager,
+ ]
+ self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"
+ else:
+ # Legacy mode: sequential pipeline
+ transforms = [
+ CacheAligner(CacheAlignerConfig(enabled=False)),
+ SmartCrusher(
+ SmartCrusherConfig( # type: ignore[arg-type]
+ enabled=True,
+ min_tokens_to_crush=config.min_tokens_to_crush,
+ max_items_after_crush=config.max_items_after_crush,
+ ),
+ ccr_config=CCRConfig(
+ enabled=config.ccr_inject_tool,
+ inject_retrieval_marker=config.ccr_inject_tool, # Add CCR markers
+ ),
+ ),
+ context_manager,
+ ]
+ # Add CodeAware if enabled and available
+ self._code_aware_status = self._setup_code_aware(config, transforms)
+
+ self.anthropic_pipeline = TransformPipeline(
+ transforms=transforms,
+ provider=self.anthropic_provider,
+ )
+ self.openai_pipeline = TransformPipeline(
+ transforms=transforms,
+ provider=self.openai_provider,
+ )
+
+ # Initialize components
+ self.cache = (
+ SemanticCache(
+ max_entries=config.cache_max_entries,
+ ttl_seconds=config.cache_ttl_seconds,
+ )
+ if config.cache_enabled
+ else None
+ )
+
+ self.rate_limiter = (
+ TokenBucketRateLimiter(
+ requests_per_minute=config.rate_limit_requests_per_minute,
+ tokens_per_minute=config.rate_limit_tokens_per_minute,
+ )
+ if config.rate_limit_enabled
+ else None
+ )
+
+ self.cost_tracker = (
+ CostTracker(
+ budget_limit_usd=config.budget_limit_usd,
+ budget_period=config.budget_period,
+ )
+ if config.cost_tracking_enabled
+ else None
+ )
+
+ self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)
+
+ # Prefix cache tracking: freeze already-cached messages to avoid
+ # invalidating the provider's prefix cache with our transforms
+ from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore
+
+ self.session_tracker_store = SessionTrackerStore(
+ default_config=PrefixFreezeConfig(
+ enabled=config.prefix_freeze_enabled,
+ session_ttl_seconds=config.prefix_freeze_session_ttl,
+ )
+ )
+
+ # Compression cache store for token mode (session-scoped)
+ self._compression_caches: dict[str, CompressionCache] = {}
+
+ self.logger = (
+ RequestLogger(
+ log_file=config.log_file,
+ log_full_messages=config.log_full_messages,
+ )
+ if config.log_requests
+ else None
+ )
+
+ # Enterprise security plugin (loaded dynamically if available + licensed)
+ self.security = None
+
+ # HTTP client
+ self.http_client: httpx.AsyncClient | None = None
+
+ # Shared cold-start warmup registry (populated by startup()).
+ # Holds typed slots with loaded / loading / null / error status for
+ # each preloaded heavy asset. Exposed as ``proxy.warmup`` and
+ # serialized by the /debug/warmup route (Unit 5).
+ self.warmup: WarmupRegistry = WarmupRegistry()
+ # Unit 3: live registry of Codex WS sessions. Populated by
+ # ``handle_openai_responses_ws`` on accept; drained in its
+ # outermost ``finally``. Consumed by ``/debug/ws-sessions``.
+ self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()
+
+ # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP
+ # path. Caps how many ``handle_anthropic_messages`` calls may be
+ # running deep-copy / first-stage compression / memory-context
+ # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,
+ # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are
+ # intentionally NOT gated by this semaphore.
+ #
+ # A value of ``0`` or negative disables the semaphore (unbounded
+ # mode); this is useful for the Unit 6 counter-factual where we
+ # deliberately reproduce the original starvation. The default is
+ # ``max(2, min(8, os.cpu_count() or 4))``.
+ _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency
+ if _pre_upstream_cfg is None:
+ _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))
+ else:
+ _pre_upstream_resolved = _pre_upstream_cfg
+ self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved
+ self.anthropic_pre_upstream_acquire_timeout_seconds = float(
+ config.anthropic_pre_upstream_acquire_timeout_seconds
+ )
+ self.anthropic_pre_upstream_memory_context_timeout_seconds = float(
+ config.anthropic_pre_upstream_memory_context_timeout_seconds
+ )
+ if _pre_upstream_resolved > 0:
+ self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = asyncio.Semaphore(
+ _pre_upstream_resolved
+ )
+ else:
+ self.anthropic_pre_upstream_sem = None
+
+ # Backend for Anthropic API (direct, LiteLLM, or any-llm)
+ # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
+ self.anthropic_backend: Backend | None = create_proxy_backend(
+ backend=config.backend,
+ anyllm_provider=config.anyllm_provider,
+ bedrock_region=config.bedrock_region,
+ logger=logger,
+ anyllm_backend_cls=AnyLLMBackend,
+ litellm_backend_cls=LiteLLMBackend,
+ )
+
+ # Request counter for IDs
+ self._request_counter = 0
+ self._request_counter_lock = asyncio.Lock()
+
+ # CCR tool injectors (one per provider)
+ self.anthropic_tool_injector = CCRToolInjector(
+ provider="anthropic",
+ inject_tool=config.ccr_inject_tool,
+ inject_system_instructions=config.ccr_inject_system_instructions,
+ )
+ self.openai_tool_injector = CCRToolInjector(
+ provider="openai",
+ inject_tool=config.ccr_inject_tool,
+ inject_system_instructions=config.ccr_inject_system_instructions,
+ )
+
+ # CCR Response Handler (handles CCR tool calls automatically)
+ self.ccr_response_handler = (
+ CCRResponseHandler(
+ ResponseHandlerConfig(
+ enabled=True,
+ max_retrieval_rounds=config.ccr_max_retrieval_rounds,
+ )
+ )
+ if config.ccr_handle_responses
+ else None
+ )
+
+ # CCR Context Tracker (tracks compressed content across turns)
+ self.ccr_context_tracker = (
+ ContextTracker(
+ ContextTrackerConfig(
+ enabled=True,
+ proactive_expansion=config.ccr_proactive_expansion,
+ max_proactive_expansions=config.ccr_max_proactive_expansions,
+ )
+ )
+ if config.ccr_context_tracking
+ else None
+ )
+
+ # Turn counter for context tracking
+ self._turn_counter = 0
+
+ # Memory Handler (persistent user memory)
+ self.memory_handler: MemoryHandler | None = None
+ if config.memory_enabled:
+ # Resolve memory DB path: empty → project-scoped default
+ _mem_db_path = config.memory_db_path
+ if not _mem_db_path:
+ _mem_dir = Path.cwd() / ".headroom"
+ _mem_dir.mkdir(parents=True, exist_ok=True)
+ _mem_db_path = str(_mem_dir / "memory.db")
+ logger.info(f"Memory: Project-scoped DB at {_mem_db_path}")
+
+ memory_config = MemoryConfig(
+ enabled=True,
+ backend=config.memory_backend,
+ db_path=_mem_db_path,
+ inject_tools=config.memory_inject_tools,
+ use_native_tool=config.memory_use_native_tool,
+ inject_context=config.memory_inject_context,
+ top_k=config.memory_top_k,
+ min_similarity=config.memory_min_similarity,
+ qdrant_host=config.memory_qdrant_host,
+ qdrant_port=config.memory_qdrant_port,
+ neo4j_uri=config.memory_neo4j_uri,
+ neo4j_user=config.memory_neo4j_user,
+ neo4j_password=config.memory_neo4j_password,
+ bridge_enabled=config.memory_bridge_enabled,
+ bridge_md_paths=config.memory_bridge_md_paths,
+ bridge_md_format=config.memory_bridge_md_format,
+ bridge_auto_import=config.memory_bridge_auto_import,
+ bridge_export_path=config.memory_bridge_export_path,
+ )
+ self.memory_handler = MemoryHandler(
+ memory_config,
+ agent_type=config.traffic_learning_agent_type,
+ )
+
+ # Usage Reporter (license validation + phone-home for managed/enterprise)
+ self.usage_reporter: UsageReporter | None = None
+ if config.license_key:
+ from headroom.telemetry.reporter import UsageReporter
+
+ self.usage_reporter = UsageReporter(
+ license_key=config.license_key,
+ cloud_url=config.license_cloud_url,
+ report_interval=config.license_report_interval,
+ )
+
+ # Traffic Learner (live pattern extraction from proxy traffic)
+ # Only activates with --learn flag; requires --memory for backend
+ self.traffic_learner: TrafficLearner | None = None
+ self.traffic_learning_agent_type: str = config.traffic_learning_agent_type
+ if config.traffic_learning_enabled:
+ from headroom.memory.traffic_learner import TrafficLearner
+
+ self.traffic_learner = TrafficLearner(
+ user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
+ agent_type=config.traffic_learning_agent_type,
+ )
+
+ # Code graph file watcher (live reindex on file changes)
+ self.code_graph_watcher: CodeGraphWatcher | None = None # type: ignore[annotation-unchecked]
+ if config.code_graph_watcher:
+ from headroom.graph.watcher import CodeGraphWatcher
+
+ self.code_graph_watcher = CodeGraphWatcher(project_dir=Path.cwd())
+ if self.code_graph_watcher.start():
+ logger.info("Code graph: file watcher started")
+ else:
+ self.code_graph_watcher = None
+
+ self.pipeline_extensions.emit(
+ PipelineStage.SETUP,
+ operation="proxy.setup",
+ metadata={
+ "mode": self.config.mode,
+ "optimize": self.config.optimize,
+ "backend": self.config.backend,
+ "memory_enabled": self.config.memory_enabled,
+ },
+ )
+
+ def _get_compression_cache(self, session_id: str) -> CompressionCache:
+ """Get or create a CompressionCache for a session."""
+ if session_id not in self._compression_caches:
+ from headroom.cache.compression_cache import CompressionCache
+
+ # Evict oldest caches if at capacity
+ if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:
+ # Remove oldest quarter to amortize cleanup cost
+ oldest_keys = list(self._compression_caches.keys())[
+ : MAX_COMPRESSION_CACHE_SESSIONS // 4
+ ]
+ for key in oldest_keys:
+ del self._compression_caches[key]
+ logger.info(
+ "Evicted %d compression caches (exceeded %d max sessions)",
+ len(oldest_keys),
+ MAX_COMPRESSION_CACHE_SESSIONS,
+ )
+
+ self._compression_caches[session_id] = CompressionCache()
+ return self._compression_caches[session_id]
+
+ def _setup_code_aware(self, config: ProxyConfig, transforms: list) -> str:
+ """Set up code-aware compression if enabled.
+
+ Args:
+ config: Proxy configuration
+ transforms: Transform list to append to
+
+ Returns:
+ Status string for logging: 'enabled', 'disabled', 'available', 'unavailable'
+ """
+ if config.code_aware_enabled:
+ if is_tree_sitter_available():
+ code_config = CodeCompressorConfig(
+ preserve_imports=True,
+ preserve_signatures=True,
+ preserve_type_annotations=True,
+ )
+ # Insert before RollingWindow (which should be last)
+ transforms.insert(-1, CodeAwareCompressor(code_config))
+ return "enabled"
+ else:
+ logger.warning(
+ "Code-aware compression requested but tree-sitter not installed. "
+ "Install with: pip install headroom-ai[code]"
+ )
+ return "unavailable"
+ else:
+ if is_tree_sitter_available():
+ return "available" # Available but not enabled
+ return "disabled"
+
+ async def startup(self):
+ """Initialize async resources."""
+ self.pipeline_extensions.emit(
+ PipelineStage.PRE_START,
+ operation="proxy.startup",
+ metadata={"port": self.config.port, "host": self.config.host},
+ )
+ self.http_client = httpx.AsyncClient(
+ timeout=httpx.Timeout(
+ connect=self.config.connect_timeout_seconds,
+ read=self.config.request_timeout_seconds,
+ write=self.config.request_timeout_seconds,
+ pool=self.config.connect_timeout_seconds,
+ ),
+ limits=httpx.Limits(
+ max_connections=self.config.max_connections,
+ max_keepalive_connections=self.config.max_keepalive_connections,
+ ),
+ http2=self.config.http2,
+ )
+ logger.info("Headroom Proxy started")
+ logger.info(f"Optimization: {'ENABLED' if self.config.optimize else 'DISABLED'}")
+ self.config.mode = normalize_proxy_mode(self.config.mode)
+ logger.info(f"Mode: {self.config.mode}")
+ if self.config.mode == PROXY_MODE_TOKEN:
+ logger.info(" Prefix freeze: re-freeze after compression")
+ logger.info(" Read protection window: 30%% of excluded-tool messages")
+ logger.info(" CCR TTL: extended for session lifetime")
+ logger.info(" Compression cache: active")
+ if self.config.mode == PROXY_MODE_CACHE:
+ logger.info(" Prefix freeze: strict (all prior turns immutable)")
+ logger.info(" Mutations: latest turn only")
+ logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}")
+ logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}")
+ logger.info(
+ f"Connection Pool: max_connections={self.config.max_connections}, "
+ f"max_keepalive={self.config.max_keepalive_connections}, "
+ f"http2={'ENABLED' if self.config.http2 else 'DISABLED'}"
+ )
+
+ # Unit 4 pre-upstream concurrency announcement. Report the resolved
+ # value (auto-detected vs. explicit) so operators can correlate
+ # ``pre_upstream_wait_ms`` log lines with the configured cap.
+ if self.anthropic_pre_upstream_sem is None:
+ logger.info("Anthropic pre-upstream concurrency: unbounded (explicitly disabled)")
+ else:
+ _explicit = self.config.anthropic_pre_upstream_concurrency
+ _origin = "auto-detected" if _explicit is None else "explicit"
+ logger.info(
+ "Anthropic pre-upstream concurrency: %d (%s)",
+ self.anthropic_pre_upstream_concurrency,
+ _origin,
+ )
+ logger.info(
+ "Anthropic pre-upstream timeouts: acquire=%.1fs compression=%.1fs memory_context=%.1fs",
+ self.anthropic_pre_upstream_acquire_timeout_seconds,
+ float(COMPRESSION_TIMEOUT_SECONDS),
+ self.anthropic_pre_upstream_memory_context_timeout_seconds,
+ )
+
+ # Smart routing status
+ if self.config.smart_routing:
+ logger.info("Smart Routing: ENABLED (intelligent content detection)")
+ else:
+ logger.info("Smart Routing: DISABLED (legacy sequential mode)")
+
+ # Eagerly load ALL compressors, parsers, and detectors at startup
+ # This eliminates cold-start latency spikes on first requests.
+ # Iterate BOTH pipelines (Anthropic + OpenAI) and dedupe transforms
+ # by id() so shared-transform instances never load twice. The
+ # resulting status dict is merged into ``self.warmup`` so /debug/warmup
+ # (Unit 5) and /readyz have a single source of truth.
+ self._kompress_status = "not installed"
+ eager_status: dict[str, str] = {}
+
+ if self.config.optimize:
+ logger.info("Pre-loading compressors and parsers...")
+ seen_transform_ids: set[int] = set()
+ pipelines = (self.anthropic_pipeline, self.openai_pipeline)
+ for pipeline in pipelines:
+ for transform in pipeline.transforms:
+ if id(transform) in seen_transform_ids:
+ continue
+ seen_transform_ids.add(id(transform))
+ if not hasattr(transform, "eager_load_compressors"):
+ continue
+ try:
+ transform_status = transform.eager_load_compressors()
+ except Exception as exc:
+ logger.warning(
+ "Eager preload failed for %s: %s",
+ type(transform).__name__,
+ exc,
+ )
+ continue
+ if not isinstance(transform_status, dict):
+ continue
+ # Merge: later writers win only if the key wasn't set.
+ # Preload a transform ONCE — if another pipeline also has
+ # ``eager_load_compressors`` it contributes only new keys.
+ for key, value in transform_status.items():
+ eager_status.setdefault(key, value)
+ self.warmup.merge_transform_status(transform_status)
+
+ # Update internal status from eager loading results
+ if eager_status.get("kompress") == "enabled":
+ self._kompress_status = "enabled"
+ if eager_status.get("code_aware") == "enabled":
+ self._code_aware_status = "enabled"
+
+ # Log component status
+ if self._kompress_status == "enabled":
+ logger.info("Kompress: ENABLED (ModernBERT token compressor)")
+ elif self.config.optimize:
+ logger.info("Kompress: not installed (pip install headroom-ai[ml] for ML compression)")
+
+ if self._code_aware_status == "enabled":
+ logger.info("Code-Aware: ENABLED (AST-based compression)")
+ if "tree_sitter" in eager_status:
+ logger.info(f"Tree-Sitter: {eager_status['tree_sitter']}")
+ elif self._code_aware_status == "lazy":
+ logger.info("Code-Aware: LAZY (will load when code content detected)")
+ elif self._code_aware_status == "available":
+ logger.info("Code-Aware: available but disabled (use --code-aware)")
+ elif self._code_aware_status == "unavailable":
+ logger.info("Code-Aware: not installed (pip install headroom-ai[code])")
+ elif self._code_aware_status == "disabled":
+ logger.info("Code-Aware: DISABLED")
+
+ if eager_status.get("magika") == "enabled":
+ logger.info("Magika: ENABLED (ML content detection)")
+
+ if self.memory_handler:
+ self.warmup.memory_backend.mark_loading()
+ try:
+ await self.memory_handler.ensure_initialized()
+ except Exception as exc: # pragma: no cover - defensive
+ self.warmup.memory_backend.mark_error(str(exc))
+ logger.warning("Memory: backend initialization failed (startup continues): %s", exc)
+ memory_status = self.memory_handler.health_status()
+ if memory_status.get("initialized"):
+ self.warmup.memory_backend.mark_loaded(
+ handle=self.memory_handler,
+ backend=memory_status.get("backend"),
+ )
+ # Force one embed call so the ONNX graph is compiled now,
+ # not lazily during the first request. Best-effort — any
+ # failure is swallowed inside warmup_embedder.
+ self.warmup.memory_embedder.mark_loading()
+ warmed = await self.memory_handler.warmup_embedder()
+ if warmed:
+ self.warmup.memory_embedder.mark_loaded()
+ else:
+ # Not an error — e.g. qdrant-neo4j has no embedder slot
+ # we can reach, or the backend simply exposes no handle.
+ self.warmup.memory_embedder.mark_null()
+ else:
+ if self.warmup.memory_backend.status != "error":
+ self.warmup.memory_backend.mark_null()
+ self.warmup.memory_embedder.mark_null()
+ logger.info(
+ "Memory: ENABLED "
+ f"(backend={memory_status['backend']}, initialized={memory_status['initialized']})"
+ )
+ else:
+ logger.info("Memory: DISABLED")
+
+ # CCR status
+ ccr_features = []
+ if self.config.ccr_inject_tool:
+ ccr_features.append("tool_injection")
+ if self.config.ccr_handle_responses:
+ ccr_features.append("response_handling")
+ if self.config.ccr_context_tracking:
+ ccr_features.append("context_tracking")
+ if self.config.ccr_proactive_expansion:
+ ccr_features.append("proactive_expansion")
+ if ccr_features:
+ logger.info(f"CCR (Compress-Cache-Retrieve): ENABLED ({', '.join(ccr_features)})")
+ else:
+ logger.info("CCR: DISABLED")
+ logger.info(f"Savings history: {self.metrics.savings_tracker.storage_path}")
+
+ # Reset and rebuild the quota tracker registry for this server instance.
+ # reset_quota_registry() ensures a clean slate when the proxy is restarted
+ # (e.g. in tests that spin up multiple app instances in the same process).
+ reset_quota_registry()
+ registry = get_quota_registry()
+ tracker = configure_subscription_tracker(
+ poll_interval_s=self.config.subscription_poll_interval_s,
+ active_window_s=self.config.subscription_active_window_s,
+ enabled=self.config.subscription_tracking_enabled,
+ )
+ registry.register(tracker)
+ registry.register(get_codex_rate_limit_state())
+ registry.register(get_copilot_quota_tracker())
+ await registry.start_all()
+
+ if self.config.subscription_tracking_enabled:
+ logger.info(
+ "Subscription tracking: ENABLED "
+ f"(poll_interval={self.config.subscription_poll_interval_s}s, "
+ f"active_window={self.config.subscription_active_window_s}s)"
+ )
+ else:
+ logger.info("Subscription tracking: DISABLED")
+
+ copilot_tracker = get_copilot_quota_tracker()
+ if copilot_tracker.is_available():
+ logger.info("GitHub Copilot quota tracking: ENABLED")
+ else:
+ logger.info(
+ "GitHub Copilot quota tracking: DISABLED "
+ "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)"
+ )
+
+ # Log anonymous telemetry status so operators can see it in the log stream
+ if is_telemetry_enabled():
+ logger.info(
+ "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). "
+ "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry"
+ )
+ else:
+ logger.info("Anonymous telemetry: DISABLED")
+
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_START,
+ operation="proxy.startup",
+ metadata={
+ "port": self.config.port,
+ "host": self.config.host,
+ "warmup": self.warmup.to_dict(),
+ },
+ )
+
+ async def shutdown(self):
+ """Cleanup async resources."""
+ if self.http_client:
+ await self.http_client.aclose()
+ self.http_client = None
+
+ if self.memory_handler and hasattr(self.memory_handler, "close"):
+ await self.memory_handler.close()
+
+ # Stop all quota trackers via the registry
+ await get_quota_registry().stop_all()
+
+ # Print final stats
+ self._print_summary()
+
+ def _print_summary(self):
+ """Print session summary."""
+ m = self.metrics
+ logger.info("=" * 70)
+ logger.info("HEADROOM PROXY SESSION SUMMARY")
+ logger.info("=" * 70)
+ logger.info(f"Total requests: {m.requests_total}")
+ logger.info(f"Cached responses: {m.requests_cached}")
+ logger.info(f"Rate limited: {m.requests_rate_limited}")
+ logger.info(f"Failed: {m.requests_failed}")
+ logger.info(f"Input tokens: {m.tokens_input_total:,}")
+ logger.info(f"Output tokens: {m.tokens_output_total:,}")
+ logger.info(f"Tokens saved: {m.tokens_saved_total:,}")
+ if m.tokens_input_total > 0:
+ savings_pct = (
+ m.tokens_saved_total / (m.tokens_input_total + m.tokens_saved_total)
+ ) * 100
+ logger.info(f"Token savings: {savings_pct:.1f}%")
+ if m.latency_count > 0:
+ avg_latency = m.latency_sum_ms / m.latency_count
+ logger.info(f"Avg latency: {avg_latency:.0f}ms")
+ logger.info("=" * 70)
+
+ async def _next_request_id(self) -> str:
+ """Generate unique request ID."""
+ async with self._request_counter_lock:
+ self._request_counter += 1
+ return f"hr_{int(time.time())}_{self._request_counter:06d}"
+
+ def _extract_tags(self, headers: dict) -> dict[str, str]:
+ """Extract Headroom tags from headers."""
+ tags = {}
+ for key, value in headers.items():
+ if key.lower().startswith("x-headroom-"):
+ tag_name = key.lower().replace("x-headroom-", "")
+ tags[tag_name] = value
+ return tags
+
+ def _inject_system_context(
+ self,
+ messages: list[dict[str, Any]],
+ context: str,
+ body: dict[str, Any] | None = None,
+ ) -> list[dict[str, Any]]:
+ """Inject context into the system message/parameter.
+
+ For Anthropic API: Uses top-level 'system' parameter (not messages array).
+ For OpenAI API: Uses system role in messages array.
+
+ Args:
+ messages: The messages list.
+ context: Context to inject.
+ body: Optional request body to update system parameter (for Anthropic).
+
+ Returns:
+ Updated messages list.
+ """
+ messages = list(messages) # Copy to avoid mutation
+
+ # For Anthropic API: use top-level 'system' parameter
+ if body is not None:
+ existing_system = body.get("system", "")
+ if isinstance(existing_system, str):
+ body["system"] = (existing_system + "\n\n" + context).strip()
+ elif isinstance(existing_system, list):
+ # system is a list of content blocks (e.g., with cache_control).
+ # Append memory context as a new text block — never overwrite.
+ body["system"] = existing_system + [{"type": "text", "text": context}]
+ else:
+ # No existing system prompt — set as string
+ body["system"] = context
+ return messages
+
+ # For OpenAI API: use system role in messages
+ for i, msg in enumerate(messages):
+ if msg.get("role") == "system":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ messages[i] = {**msg, "content": content + "\n\n" + context}
+ return messages
+
+ # No system message found - prepend one
+ messages.insert(0, {"role": "system", "content": context})
+ return messages
+
+ async def _retry_request(
+ self,
+ method: str,
+ url: str,
+ headers: dict,
+ body: dict,
+ stream: bool = False,
+ ) -> httpx.Response:
+ """Make request with retry and exponential backoff."""
+ last_error = None
+
+ for attempt in range(self.config.retry_max_attempts):
+ try:
+ if stream:
+ # For streaming, we return early - retry happens at higher level
+ return await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
+ else:
+ response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
+
+ # Don't retry client errors (4xx)
+ if 400 <= response.status_code < 500:
+ return response
+
+ # Retry server errors (5xx)
+ if response.status_code >= 500:
+ raise httpx.HTTPStatusError(
+ f"Server error: {response.status_code}",
+ request=response.request,
+ response=response,
+ )
+
+ return response
+
+ except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+ last_error = e
+
+ if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:
+ raise
+
+ # Exponential backoff with jitter
+ delay_with_jitter = jitter_delay_ms(
+ self.config.retry_base_delay_ms,
+ self.config.retry_max_delay_ms,
+ attempt,
+ )
+
+ logger.warning(
+ f"Request failed (attempt {attempt + 1}), retrying in {delay_with_jitter:.0f}ms: {e}"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ raise last_error # type: ignore[misc]
+
+
+async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:
+ """Background task that logs TOIN stats periodically.
+
+ Args:
+ interval_seconds: How often to log stats (default: 5 minutes).
+ """
+ while True:
+ await asyncio.sleep(interval_seconds)
+ try:
+ toin = get_toin()
+ stats = toin.get_stats()
+ total_compressions = stats.get("total_compressions", 0)
+ if total_compressions > 0:
+ patterns = stats.get("patterns_tracked", 0)
+ retrievals = stats.get("total_retrievals", 0)
+ retrieval_rate = stats.get("global_retrieval_rate", 0.0)
+ logger.info(
+ "TOIN: %d patterns, %d compressions, %d retrievals, %.1f%% retrieval rate",
+ patterns,
+ total_compressions,
+ retrievals,
+ retrieval_rate * 100,
+ )
+ except Exception as e:
+ logger.debug("Failed to log TOIN stats: %s", e)
+
+
+def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:
+ """Register all memory-tracked components with the tracker.
+
+ This function is idempotent - it checks if components are already registered.
+
+ Args:
+ proxy: The HeadroomProxy instance.
+ tracker: The MemoryTracker instance.
+ """
+ # Register compression store (global singleton)
+ if "compression_store" not in tracker.registered_components:
+ store = get_compression_store()
+ tracker.register("compression_store", store.get_memory_stats)
+
+ # Register semantic cache (instance on proxy)
+ if proxy.cache and "semantic_cache" not in tracker.registered_components:
+ tracker.register("semantic_cache", proxy.cache.get_memory_stats)
+
+ # Register request logger (instance on proxy)
+ if proxy.logger and "request_logger" not in tracker.registered_components:
+ tracker.register("request_logger", proxy.logger.get_memory_stats)
+
+ # Register batch context store (global singleton)
+ if "batch_context_store" not in tracker.registered_components:
+ try:
+ from ..ccr.batch_store import get_batch_context_store
+
+ batch_store = get_batch_context_store()
+ if hasattr(batch_store, "get_memory_stats"):
+ tracker.register("batch_context_store", batch_store.get_memory_stats)
+ except ImportError:
+ pass
+
+ # Note: graph_store and vector_index are created per-user within the
+ # LocalMemoryBackend, not as global singletons. They would need to be
+ # registered when the memory system is initialized with specific backends.
+
+
+def create_app(config: ProxyConfig | None = None) -> FastAPI:
+ """Create FastAPI application."""
+ if not FASTAPI_AVAILABLE:
+ raise ImportError("FastAPI required. Install: pip install fastapi uvicorn httpx")
+
+ from contextlib import asynccontextmanager
+
+ config = config or ProxyConfig()
+ proxy = HeadroomProxy(config)
+
+ # Telemetry beacon (anonymous aggregate stats).
+ # With uvicorn workers > 1, each worker runs the lifespan independently.
+ # We must ensure only ONE beacon runs across all workers — otherwise each
+ # worker creates its own beacon, spamming the telemetry table with N rows
+ # per cycle instead of 1 (all reading the same /stats from the same port).
+ #
+ # Strategy: use a file lock to ensure only the first worker starts the
+ # beacon. Other workers see the lock and skip.
+ from headroom.telemetry.beacon import TelemetryBeacon
+
+ _beacon = TelemetryBeacon(
+ port=config.port if hasattr(config, "port") else 8787,
+ sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
+ backend=config.backend if hasattr(config, "backend") else "anthropic",
+ )
+ from headroom import paths as _hr_paths
+
+ _beacon_lock_path = _hr_paths.beacon_lock_path(config.port)
+ _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor
+ _beacon_is_owner: list = [False]
+
+ def _try_acquire_beacon_lock() -> bool:
+ """Try to acquire the beacon file lock (non-blocking).
+
+ Returns True if this process is the beacon owner.
+ """
+ if not HAS_FCNTL:
+ return True
+
+ fd = None
+ try:
+ _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)
+ fd = open(_beacon_lock_path, "w") # noqa: SIM115
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ fd.write(str(os.getpid()))
+ fd.flush()
+ _beacon_lock_fd[0] = fd
+ return True
+ except OSError:
+ if fd is not None:
+ fd.close()
+ return False
+
+ def _release_beacon_lock() -> None:
+ """Release the beacon file lock."""
+ fd = _beacon_lock_fd[0]
+ if fd:
+ try:
+ if HAS_FCNTL:
+ fcntl.flock(fd, fcntl.LOCK_UN)
+ fd.close()
+ except Exception:
+ pass
+ _beacon_lock_fd[0] = None
+ try:
+ _beacon_lock_path.unlink(missing_ok=True)
+ except Exception:
+ pass
+
+ @asynccontextmanager
+ async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
+ configure_otel_metrics(OTelMetricsConfig.from_env(default_service_name="headroom-proxy"))
+ configure_langfuse_tracing(
+ LangfuseTracingConfig.from_env(default_service_name="headroom-proxy")
+ )
+
+ app.state.started_at = time.time()
+ app.state.ready = False
+ app.state.startup_error = None
+
+ try:
+ try:
+ # Startup
+ await proxy.startup()
+ asyncio.create_task(_log_toin_stats_periodically())
+ if proxy.usage_reporter:
+ await proxy.usage_reporter.start(proxy)
+ if proxy.traffic_learner:
+ await proxy.traffic_learner.start()
+
+ # Only start beacon if we acquire the lock (first worker wins)
+ _beacon_is_owner[0] = _try_acquire_beacon_lock()
+ if _beacon_is_owner[0]:
+ await _beacon.start()
+ else:
+ logger.debug("Beacon: skipping (another worker owns the lock)")
+
+ app.state.ready = True
+ yield
+ except Exception as exc:
+ app.state.startup_error = str(exc)
+ raise
+ finally:
+ app.state.ready = False
+ # Shutdown
+ if _beacon_is_owner[0]:
+ await _beacon.stop()
+ _release_beacon_lock()
+ if proxy.usage_reporter:
+ await proxy.usage_reporter.stop()
+ if proxy.traffic_learner:
+ await proxy.traffic_learner.stop()
+ if proxy.code_graph_watcher:
+ proxy.code_graph_watcher.stop()
+ await proxy.shutdown()
+ shutdown_headroom_tracing()
+ shutdown_otel_metrics()
+
+ app = FastAPI(
+ title="Headroom Proxy",
+ description="Production-ready LLM optimization proxy",
+ version=__version__,
+ lifespan=lifespan,
+ )
+ app.state.proxy = proxy
+ app.state.started_at = None
+ app.state.ready = False
+ app.state.startup_error = None
+
+ def _iso_utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+ def _uptime_seconds() -> float:
+ started_at = getattr(app.state, "started_at", None)
+ if not isinstance(started_at, int | float):
+ return 0.0
+ return round(max(0.0, time.time() - float(started_at)), 3)
+
+ def _component_health(
+ *,
+ enabled: bool,
+ ready: bool,
+ **details: Any,
+ ) -> dict[str, Any]:
+ status = "disabled" if not enabled else ("healthy" if ready else "unhealthy")
+ return {
+ "enabled": enabled,
+ "ready": (ready if enabled else True),
+ "status": status,
+ **details,
+ }
+
+ def _health_checks() -> dict[str, dict[str, Any]]:
+ memory_status = (
+ proxy.memory_handler.health_status()
+ if proxy.memory_handler
+ else {
+ "enabled": False,
+ "backend": None,
+ "initialized": False,
+ "native_tool": False,
+ "bridge_enabled": False,
+ }
+ )
+ memory_enabled = bool(memory_status.get("enabled", False))
+ memory_initialized = bool(memory_status.get("initialized", False))
+ return {
+ "startup": _component_health(
+ enabled=True,
+ ready=bool(getattr(app.state, "ready", False)),
+ error=getattr(app.state, "startup_error", None),
+ ),
+ "http_client": _component_health(
+ enabled=True,
+ ready=proxy.http_client is not None,
+ ),
+ "cache": _component_health(
+ enabled=config.cache_enabled,
+ ready=(proxy.cache is not None),
+ ),
+ "rate_limiter": _component_health(
+ enabled=config.rate_limit_enabled,
+ ready=(proxy.rate_limiter is not None),
+ ),
+ "memory": _component_health(
+ enabled=memory_enabled,
+ ready=memory_initialized,
+ backend=memory_status["backend"],
+ initialized=memory_initialized,
+ native_tool=bool(memory_status.get("native_tool", False)),
+ bridge_enabled=bool(memory_status.get("bridge_enabled", False)),
+ ),
+ }
+
+ def _runtime_payload() -> dict[str, Any]:
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ ws_active_sessions = ws_registry.active_count() if ws_registry is not None else 0
+ ws_active_relay_tasks = (
+ ws_registry.active_relay_task_count() if ws_registry is not None else 0
+ )
+ return {
+ "anthropic_pre_upstream": {
+ "enabled": proxy.anthropic_pre_upstream_sem is not None,
+ "resolved_concurrency": proxy.anthropic_pre_upstream_concurrency,
+ "source": (
+ "auto" if config.anthropic_pre_upstream_concurrency is None else "explicit"
+ ),
+ "acquire_timeout_seconds": proxy.anthropic_pre_upstream_acquire_timeout_seconds,
+ "compression_timeout_seconds": float(COMPRESSION_TIMEOUT_SECONDS),
+ "memory_context_timeout_seconds": (
+ proxy.anthropic_pre_upstream_memory_context_timeout_seconds
+ ),
+ "codex_ws_gated": False,
+ },
+ "websocket_sessions": {
+ "active_sessions": ws_active_sessions,
+ "active_relay_tasks": ws_active_relay_tasks,
+ },
+ }
+
+ def _health_payload(*, include_config: bool) -> dict[str, Any]:
+ checks = _health_checks()
+ ready = all(check["ready"] for check in checks.values())
+ payload: dict[str, Any] = {
+ "service": "headroom-proxy",
+ "status": "healthy" if ready else "unhealthy",
+ "ready": ready,
+ "version": __version__,
+ "timestamp": _iso_utc_now(),
+ "uptime_seconds": _uptime_seconds(),
+ "checks": checks,
+ "runtime": _runtime_payload(),
+ }
+ deployment_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE")
+ if deployment_profile:
+ payload["deployment"] = {
+ "profile": deployment_profile,
+ "preset": os.environ.get("HEADROOM_DEPLOYMENT_PRESET"),
+ "runtime": os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME"),
+ "supervisor": os.environ.get("HEADROOM_DEPLOYMENT_SUPERVISOR"),
+ "scope": os.environ.get("HEADROOM_DEPLOYMENT_SCOPE"),
+ }
+ if include_config:
+ payload["config"] = {
+ "backend": config.backend,
+ "optimize": config.optimize,
+ "cache": config.cache_enabled,
+ "rate_limit": config.rate_limit_enabled,
+ "memory": config.memory_enabled,
+ "learn": config.traffic_learning_enabled,
+ "code_graph": config.code_graph_watcher,
+ "pid": os.getpid(),
+ }
+ return payload
+
+ # CORS
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+
+ # X-Headroom-Stack: SDK adapters (TS openai/anthropic/etc.) tag their
+ # requests so telemetry can segment by integration surface. Registered
+ # before extension middleware so any extension-level auth/guards run
+ # outermost and we don't count requests they reject.
+ @app.middleware("http")
+ async def _record_headroom_stack(request, call_next):
+ if request.url.path.startswith("/v1/"):
+ stack = request.headers.get("x-headroom-stack")
+ if stack:
+ try:
+ proxy.metrics.record_stack(stack)
+ except Exception:
+ logger.debug("record_stack failed", exc_info=True)
+ return await call_next(request)
+
+ # Third-party proxy extensions (Enterprise, custom plugins). Discovered via
+ # the `headroom.proxy_extension` entry-point group. An extension that raises
+ # from its install() is a deliberate fail-closed signal and aborts startup.
+ from headroom.proxy.extensions import install_all as _install_extensions
+
+ _install_extensions(app, config)
+
+ # Health & Metrics
+ @app.get("/livez")
+ async def livez():
+ return JSONResponse(
+ status_code=200,
+ content={
+ "service": "headroom-proxy",
+ "status": "healthy",
+ "alive": True,
+ "version": __version__,
+ "timestamp": _iso_utc_now(),
+ "uptime_seconds": _uptime_seconds(),
+ },
+ )
+
+ @app.get("/readyz")
+ async def readyz():
+ payload = _health_payload(include_config=False)
+ return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)
+
+ @app.get("/health")
+ async def health():
+ payload = _health_payload(include_config=True)
+ return JSONResponse(status_code=200, content=payload)
+
+ # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —
+ # debug endpoints are invisible to external scanners.
+ from headroom.proxy.debug_introspection import (
+ collect_tasks as _collect_tasks,
+ )
+ from headroom.proxy.loopback_guard import require_loopback as _require_loopback
+
+ @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])
+ async def debug_tasks(stack: bool = False):
+ """Enumerate running asyncio tasks.
+
+ Default is cheap — ``stack_depth`` is ``null`` in every entry so
+ a storm snapshot does not walk 50+ coroutine frames synchronously.
+ Pass ``?stack=true`` to compute ``stack_depth`` for each task
+ (useful for single-shot human debugging).
+ """
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ return JSONResponse(
+ status_code=200,
+ content=_collect_tasks(ws_registry, with_stack_depth=stack),
+ )
+
+ @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])
+ async def debug_ws_sessions():
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ snapshot = ws_registry.snapshot() if ws_registry is not None else []
+ return JSONResponse(status_code=200, content=snapshot)
+
+ @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])
+ async def debug_warmup():
+ warmup_registry = getattr(proxy, "warmup", None)
+ payload = warmup_registry.to_dict() if warmup_registry is not None else {}
+ payload["runtime"] = _runtime_payload()
+ return JSONResponse(status_code=200, content=payload)
+
+ @app.get("/dashboard", response_class=HTMLResponse)
+ async def dashboard():
+ """Serve the Headroom dashboard UI."""
+ return get_dashboard_html()
+
+ @app.get("/stats")
+ async def stats():
+ """Get comprehensive proxy statistics.
+
+ This is the main stats endpoint - it aggregates data from all subsystems:
+ - Request metrics (total, cached, failed, by model/provider)
+ - Token usage and savings
+ - Cost tracking
+ - Canonical persisted display_session metrics for downstream dashboards
+ - Compression (CCR) statistics
+ - Telemetry/TOIN (data flywheel) statistics
+ - Cache and rate limiter stats
+ """
+ m = proxy.metrics
+
+ # Calculate average latency
+ avg_latency_ms = round(m.latency_sum_ms / m.latency_count, 2) if m.latency_count > 0 else 0
+ min_latency_ms = (
+ round(m.latency_min_ms, 2)
+ if m.latency_count > 0 and m.latency_min_ms != float("inf")
+ else 0
+ )
+ max_latency_ms = round(m.latency_max_ms, 2) if m.latency_count > 0 else 0
+
+ # Calculate Headroom overhead (optimization time only, excludes pass-through requests)
+ avg_overhead_ms = (
+ round(m.overhead_sum_ms / m.overhead_count, 2) if m.overhead_count > 0 else 0
+ )
+ min_overhead_ms = (
+ round(m.overhead_min_ms, 2)
+ if m.overhead_count > 0 and m.overhead_min_ms != float("inf")
+ else 0
+ )
+ max_overhead_ms = round(m.overhead_max_ms, 2) if m.overhead_count > 0 else 0
+
+ # Calculate TTFB (time to first byte)
+ avg_ttfb_ms = round(m.ttfb_sum_ms / m.ttfb_count, 2) if m.ttfb_count > 0 else 0
+ min_ttfb_ms = (
+ round(m.ttfb_min_ms, 2) if m.ttfb_count > 0 and m.ttfb_min_ms != float("inf") else 0
+ )
+ max_ttfb_ms = round(m.ttfb_max_ms, 2) if m.ttfb_count > 0 else 0
+
+ # Get compression store stats
+ store = get_compression_store()
+ compression_stats = store.get_stats()
+
+ # Get telemetry/TOIN stats
+ telemetry = get_telemetry_collector()
+ telemetry_stats = telemetry.get_stats()
+
+ # Get feedback loop stats
+ feedback = get_compression_feedback()
+ feedback_stats = feedback.get_stats()
+
+ # Build prefix cache stats once (used in both prefix_cache and cost)
+ prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)
+
+ # Fetch CLI filtering savings (rtk — tokens avoided before reaching context)
+ cli_filtering_stats = _get_rtk_stats()
+ cli_tokens_avoided = (
+ cli_filtering_stats.get("tokens_saved", 0) if cli_filtering_stats else 0
+ )
+
+ # Calculate total tokens before compression
+ total_tokens_before = m.tokens_input_total + m.tokens_saved_total
+
+ # Build human-readable summary
+ summary = _build_session_summary(
+ proxy, m, prefix_cache_stats, cli_tokens_avoided, total_tokens_before
+ )
+ # DEBUG: log the summary payload for external upsert consumers
+ try:
+ logger.debug("/stats summary data: %r", summary)
+ except Exception:
+ logger.warning("Failed to log /stats summary payload")
+
+ # Compression cache stats (token mode)
+ compression_cache_stats: dict = {}
+ if proxy.config.mode == PROXY_MODE_TOKEN and proxy._compression_caches:
+ total_entries = 0
+ total_hits = 0
+ total_misses = 0
+ total_tokens_saved = 0
+ for cache in proxy._compression_caches.values():
+ s = cache.get_stats()
+ total_entries += s.get("entries", 0)
+ total_hits += s.get("hits", 0)
+ total_misses += s.get("misses", 0)
+ total_tokens_saved += s.get("total_tokens_saved", 0)
+ compression_cache_stats = {
+ "mode": PROXY_MODE_TOKEN,
+ "active_sessions": len(proxy._compression_caches),
+ "total_entries": total_entries,
+ "total_hits": total_hits,
+ "total_misses": total_misses,
+ "hit_rate": round(total_hits / max(1, total_hits + total_misses) * 100, 1),
+ "total_tokens_saved": total_tokens_saved,
+ }
+ else:
+ compression_cache_stats = {"mode": proxy.config.mode}
+
+ # Build unified savings summary (all layers)
+ compression_tokens = m.tokens_saved_total
+ cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
+ total_tokens_all_layers = compression_tokens + cli_tokens_avoided
+ persistent_savings = m.savings_tracker.stats_preview()
+ display_session = persistent_savings.get("display_session", {})
+
+ return {
+ "summary": summary,
+ "savings": {
+ "total_tokens": total_tokens_all_layers,
+ "by_layer": {
+ "cli_filtering": {
+ "tokens": cli_tokens_avoided,
+ "description": "Tokens avoided by CLI output filtering (rtk) before reaching context",
+ },
+ "compression": {
+ "tokens": compression_tokens,
+ "description": "Tokens removed by proxy compression (SmartCrusher, ContentRouter, etc.)",
+ },
+ "prefix_cache": {
+ "discount_usd": round(cache_net_usd, 4),
+ "description": (
+ "Cost discount from provider prefix caching. "
+ "Headroom's CacheAligner improves hit rates; "
+ "baseline caching is provider-native."
+ ),
+ },
+ },
+ },
+ "requests": {
+ "total": m.requests_total,
+ "cached": m.requests_cached,
+ "rate_limited": m.requests_rate_limited,
+ "failed": m.requests_failed,
+ "by_provider": dict(m.requests_by_provider),
+ "by_model": dict(m.requests_by_model),
+ "by_stack": dict(m.requests_by_stack),
+ },
+ "tokens": {
+ "input": m.tokens_input_total,
+ "output": m.tokens_output_total,
+ "saved": m.tokens_saved_total,
+ "cli_tokens_avoided": cli_tokens_avoided,
+ "total_before_compression": total_tokens_before,
+ "savings_percent": round(
+ (m.tokens_saved_total / total_tokens_before * 100)
+ if total_tokens_before > 0
+ else 0,
+ 2,
+ ),
+ },
+ "latency": {
+ "average_ms": avg_latency_ms,
+ "min_ms": min_latency_ms,
+ "max_ms": max_latency_ms,
+ "total_requests": m.latency_count,
+ },
+ "overhead": {
+ "average_ms": avg_overhead_ms,
+ "min_ms": min_overhead_ms,
+ "max_ms": max_overhead_ms,
+ },
+ "ttfb": {
+ "average_ms": avg_ttfb_ms,
+ "min_ms": min_ttfb_ms,
+ "max_ms": max_ttfb_ms,
+ },
+ "pipeline_timing": {
+ name: {
+ "average_ms": round(
+ m.transform_timing_sum[name] / m.transform_timing_count[name], 2
+ ),
+ "max_ms": round(m.transform_timing_max[name], 2),
+ "count": m.transform_timing_count[name],
+ }
+ for name in sorted(m.transform_timing_sum.keys())
+ }
+ if m.transform_timing_sum
+ else {},
+ "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
+ "savings_history": m.savings_history[-100:], # Last 100 data points
+ "display_session": display_session,
+ "persistent_savings": persistent_savings,
+ "prefix_cache": prefix_cache_stats,
+ "cost": _merge_cost_stats(
+ proxy.cost_tracker.stats() if proxy.cost_tracker else None,
+ prefix_cache_stats,
+ cli_tokens_avoided=cli_tokens_avoided,
+ ),
+ "compression": {
+ "ccr_entries": compression_stats.get("entry_count", 0),
+ "ccr_max_entries": compression_stats.get("max_entries", 0),
+ "original_tokens_cached": compression_stats.get("total_original_tokens", 0),
+ "compressed_tokens_cached": compression_stats.get("total_compressed_tokens", 0),
+ "ccr_retrievals": compression_stats.get("total_retrievals", 0),
+ },
+ "compression_cache": compression_cache_stats,
+ "anon_telemetry_shipping": is_telemetry_enabled(),
+ "telemetry": {
+ "enabled": telemetry_stats.get("enabled", False),
+ "total_compressions": telemetry_stats.get("total_compressions", 0),
+ "total_retrievals": telemetry_stats.get("total_retrievals", 0),
+ "global_retrieval_rate": round(telemetry_stats.get("global_retrieval_rate", 0), 4),
+ "tool_signatures_tracked": telemetry_stats.get("tool_signatures_tracked", 0),
+ "avg_compression_ratio": round(telemetry_stats.get("avg_compression_ratio", 0), 4),
+ "avg_token_reduction": round(telemetry_stats.get("avg_token_reduction", 0), 4),
+ },
+ "otel": get_otel_metrics_status(),
+ "langfuse": get_langfuse_tracing_status(),
+ "feedback_loop": {
+ "tools_tracked": feedback_stats.get("tools_tracked", 0),
+ "total_compressions": feedback_stats.get("total_compressions", 0),
+ "total_retrievals": feedback_stats.get("total_retrievals", 0),
+ "global_retrieval_rate": round(feedback_stats.get("global_retrieval_rate", 0), 4),
+ "tools_with_high_retrieval": sum(
+ 1
+ for p in feedback_stats.get("tool_patterns", {}).values()
+ if p.get("retrieval_rate", 0) > 0.3
+ ),
+ },
+ "toin": get_toin().get_stats(),
+ "cli_filtering": cli_filtering_stats,
+ "cache": await proxy.cache.stats() if proxy.cache else None,
+ "rate_limiter": await proxy.rate_limiter.stats() if proxy.rate_limiter else None,
+ "recent_requests": proxy.logger.get_recent(10) if proxy.logger else [],
+ "log_full_messages": proxy.config.log_full_messages if proxy else False,
+ **get_quota_registry().get_all_stats(),
+ }
+
+ @app.get("/stats-history")
+ async def stats_history(
+ format: Literal["json", "csv"] = "json",
+ series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
+ history_mode: Literal["compact", "full", "none"] = "compact",
+ ):
+ """Get durable proxy compression history plus display-session state."""
+ if format == "csv":
+ filename = f"headroom-stats-history-{series}.csv"
+ return Response(
+ content=proxy.metrics.savings_tracker.export_csv(series=series),
+ media_type="text/csv; charset=utf-8",
+ headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+ )
+
+ return proxy.metrics.savings_tracker.history_response(history_mode=history_mode)
+
+ @app.get("/transformations/feed")
+ async def transformations_feed(limit: int = 20):
+ """Get recent message transformations for the live feed.
+
+ Returns empty list if log_full_messages is disabled (messages are not stored).
+ """
+ if limit > 100:
+ limit = 100
+
+ transformations = []
+ log_full_messages = proxy.config.log_full_messages if proxy else False
+
+ if proxy and proxy.logger:
+ logs = proxy.logger.get_recent_with_messages(limit)
+ for log in logs:
+ transformations.append(
+ {
+ "request_id": log.get("request_id"),
+ "timestamp": log.get("timestamp"),
+ "provider": log.get("provider"),
+ "model": log.get("model"),
+ "input_tokens_original": log.get("input_tokens_original"),
+ "input_tokens_optimized": log.get("input_tokens_optimized"),
+ "tokens_saved": log.get("tokens_saved"),
+ "savings_percent": log.get("savings_percent"),
+ "transforms_applied": log.get("transforms_applied", []),
+ "request_messages": log.get("request_messages"),
+ "response_content": log.get("response_content"),
+ }
+ )
+
+ return {"transformations": transformations, "log_full_messages": log_full_messages}
+
+ @app.get("/subscription-window")
+ async def subscription_window():
+ """Current Anthropic subscription window utilisation and Headroom contribution."""
+ tracker = get_subscription_tracker()
+ if tracker is None:
+ return JSONResponse(
+ status_code=503,
+ content={"error": "Subscription tracking is not enabled"},
+ )
+ return JSONResponse(content=tracker.state)
+
+ @app.get("/quota")
+ async def quota():
+ """Unified quota/rate-limit stats for all registered providers (Anthropic, Codex, Copilot)."""
+ return JSONResponse(content=get_quota_registry().get_all_stats())
+
+ @app.get("/metrics")
+ async def metrics():
+ """Prometheus metrics endpoint."""
+ return PlainTextResponse(
+ await proxy.metrics.export(),
+ media_type="text/plain; version=0.0.4",
+ )
+
+ # Debug endpoints
+ @app.get("/debug/memory")
+ async def debug_memory():
+ """Get detailed memory usage statistics.
+
+ Returns memory usage for all tracked components including:
+ - Process-level memory (RSS, VMS, percent)
+ - Per-component memory usage and budgets
+ - Cache hit/miss statistics
+ - Total tracked vs target budget
+
+ This endpoint is useful for debugging memory issues and
+ monitoring memory budgets.
+ """
+ from ..memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+
+ # Register components if not already registered
+ _register_memory_components(proxy, tracker)
+
+ report = tracker.get_report()
+ return report.to_dict()
+
+ @app.post("/cache/clear")
+ async def clear_cache():
+ """Clear the response cache."""
+ if proxy.cache:
+ await proxy.cache.clear()
+ return {"status": "cleared"}
+ return {"status": "cache disabled"}
+
+ # CCR (Compress-Cache-Retrieve) endpoints
+ @app.post("/v1/retrieve")
+ async def ccr_retrieve(request: Request):
+ """Retrieve original content from CCR compression cache.
+
+ This is the "Retrieve" part of CCR (Compress-Cache-Retrieve).
+ When SmartCrusher compresses tool outputs, the original data is cached.
+ LLMs can call this endpoint to get more data if needed.
+
+ Request body:
+ hash (str): Hash key from compression marker (required)
+ query (str): Optional search query to filter results
+
+ Response:
+ Full retrieval: {"hash": "...", "original_content": "...", ...}
+ Search: {"hash": "...", "query": "...", "results": [...], "count": N}
+ """
+ data = await request.json()
+ hash_key = data.get("hash")
+ query = data.get("query")
+
+ if not hash_key:
+ raise HTTPException(status_code=400, detail="hash required")
+
+ store = get_compression_store()
+
+ if query:
+ # Search within cached content
+ results = store.search(hash_key, query)
+ return {
+ "hash": hash_key,
+ "query": query,
+ "results": results,
+ "count": len(results),
+ }
+ else:
+ # Return full original content
+ entry = store.retrieve(hash_key)
+ if entry:
+ return {
+ "hash": hash_key,
+ "original_content": entry.original_content,
+ "original_tokens": entry.original_tokens,
+ "original_item_count": entry.original_item_count,
+ "compressed_item_count": entry.compressed_item_count,
+ "tool_name": entry.tool_name,
+ "retrieval_count": entry.retrieval_count,
+ }
+ raise HTTPException(
+ status_code=404, detail="Entry not found or expired (TTL: 5 minutes)"
+ )
+
+ @app.get("/v1/retrieve/stats")
+ async def ccr_stats():
+ """Get CCR compression store statistics."""
+ store = get_compression_store()
+ stats = store.get_stats()
+ events = store.get_retrieval_events(limit=20)
+ return {
+ "store": stats,
+ "recent_retrievals": [
+ {
+ "hash": e.hash,
+ "query": e.query,
+ "items_retrieved": e.items_retrieved,
+ "total_items": e.total_items,
+ "tool_name": e.tool_name,
+ "retrieval_type": e.retrieval_type,
+ }
+ for e in events
+ ],
+ }
+
+ @app.get("/v1/feedback")
+ async def ccr_feedback():
+ """Get CCR feedback loop statistics and learned patterns.
+
+ This endpoint exposes the feedback loop's learned patterns for monitoring
+ and debugging. It shows:
+ - Per-tool retrieval rates (high = compress less aggressively)
+ - Common search queries per tool
+ - Queried fields (suggest what to preserve)
+
+ Use this to understand how well compression is working and whether
+ the feedback loop is adjusting appropriately.
+ """
+ feedback = get_compression_feedback()
+ stats = feedback.get_stats()
+ return {
+ "feedback": stats,
+ "hints_example": {
+ tool_name: {
+ "hints": {
+ "max_items": hints.max_items
+ if (hints := feedback.get_compression_hints(tool_name))
+ else 15,
+ "suggested_items": hints.suggested_items if hints else None,
+ "skip_compression": hints.skip_compression if hints else False,
+ "preserve_fields": hints.preserve_fields if hints else [],
+ "reason": hints.reason if hints else "",
+ }
+ }
+ for tool_name in list(stats.get("tool_patterns", {}).keys())[:5]
+ },
+ }
+
+ @app.get("/v1/feedback/{tool_name}")
+ async def ccr_feedback_for_tool(tool_name: str):
+ """Get compression hints for a specific tool.
+
+ Returns feedback-based hints that would be used for compressing
+ this tool's output.
+ """
+ feedback = get_compression_feedback()
+ hints = feedback.get_compression_hints(tool_name)
+ patterns = feedback.get_all_patterns().get(tool_name)
+
+ return {
+ "tool_name": tool_name,
+ "hints": {
+ "max_items": hints.max_items,
+ "min_items": hints.min_items,
+ "suggested_items": hints.suggested_items,
+ "aggressiveness": hints.aggressiveness,
+ "skip_compression": hints.skip_compression,
+ "preserve_fields": hints.preserve_fields,
+ "reason": hints.reason,
+ },
+ "pattern": {
+ "total_compressions": patterns.total_compressions if patterns else 0,
+ "total_retrievals": patterns.total_retrievals if patterns else 0,
+ "retrieval_rate": patterns.retrieval_rate if patterns else 0.0,
+ "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0,
+ "search_rate": patterns.search_rate if patterns else 0.0,
+ "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [],
+ "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [],
+ }
+ if patterns
+ else None,
+ }
+
+ # Telemetry endpoints (Data Flywheel)
+ @app.get("/v1/telemetry")
+ async def telemetry_stats():
+ """Get telemetry statistics for the data flywheel.
+
+ This endpoint exposes privacy-preserving telemetry data that powers
+ the data flywheel - learning optimal compression strategies across
+ tool types based on usage patterns.
+
+ What's collected (anonymized):
+ - Tool output structure patterns (field types, not values)
+ - Compression decisions and ratios
+ - Retrieval patterns (rate, type, not content)
+ - Strategy effectiveness
+
+ What's NOT collected:
+ - Actual data values
+ - User identifiers
+ - Queries or search terms
+ - File paths or tool names (hashed by default)
+ """
+ telemetry = get_telemetry_collector()
+ return telemetry.get_stats()
+
+ @app.get("/v1/telemetry/export")
+ async def telemetry_export():
+ """Export full telemetry data for aggregation.
+
+ This endpoint exports all telemetry data in a format suitable for
+ cross-user aggregation. The data is privacy-preserving - no actual
+ values are included, only structural patterns and statistics.
+
+ Use this for:
+ - Building a central learning service
+ - Sharing learned patterns across instances
+ - Analysis and debugging
+ """
+ telemetry = get_telemetry_collector()
+ return telemetry.export_stats()
+
+ @app.post("/v1/telemetry/import")
+ async def telemetry_import(request: Request):
+ """Import telemetry data from another source.
+
+ This allows merging telemetry from multiple sources for cross-user
+ learning. The imported data is merged with existing statistics.
+
+ Request body: Telemetry export data from /v1/telemetry/export
+ """
+ telemetry = get_telemetry_collector()
+ data = await request.json()
+ telemetry.import_stats(data)
+ return {"status": "imported", "current_stats": telemetry.get_stats()}
+
+ @app.get("/v1/telemetry/tools")
+ async def telemetry_tools():
+ """Get telemetry statistics for all tracked tool signatures.
+
+ Returns statistics per tool signature (anonymized), including:
+ - Compression ratios and strategy usage
+ - Retrieval rates (high = compression too aggressive)
+ - Learned recommendations
+ """
+ telemetry = get_telemetry_collector()
+ all_stats = telemetry.get_all_tool_stats()
+ return {
+ "tool_count": len(all_stats),
+ "tools": {sig_hash: stats.to_dict() for sig_hash, stats in all_stats.items()},
+ }
+
+ @app.get("/v1/telemetry/tools/{signature_hash}")
+ async def telemetry_tool_detail(signature_hash: str):
+ """Get detailed telemetry for a specific tool signature.
+
+ Includes learned recommendations if enough data has been collected.
+ """
+ telemetry = get_telemetry_collector()
+ stats = telemetry.get_tool_stats(signature_hash)
+ recommendations = telemetry.get_recommendations(signature_hash)
+
+ if stats is None:
+ raise HTTPException(
+ status_code=404, detail=f"No telemetry found for signature: {signature_hash}"
+ )
+
+ return {
+ "signature_hash": signature_hash,
+ "stats": stats.to_dict(),
+ "recommendations": recommendations,
+ }
+
+ # TOIN (Tool Output Intelligence Network) endpoints
+ @app.get("/v1/toin/stats")
+ async def toin_stats():
+ """Get overall TOIN statistics.
+
+ Returns aggregated statistics from the Tool Output Intelligence Network,
+ which learns optimal compression strategies across all tool types.
+
+ Response includes:
+ - enabled: Whether TOIN is enabled
+ - patterns_tracked: Number of unique tool patterns being tracked
+ - total_compressions: Total compression events recorded
+ - total_retrievals: Total retrieval events recorded
+ - global_retrieval_rate: Overall retrieval rate (high = compression too aggressive)
+ - patterns_with_recommendations: Patterns with enough data for recommendations
+ """
+ toin = get_toin()
+ return toin.get_stats()
+
+ @app.get("/v1/toin/patterns")
+ async def toin_patterns(limit: int = 20):
+ """List TOIN patterns with most samples.
+
+ Returns patterns sorted by sample_size descending. Use this to see
+ which tool types have the most data and their learned behaviors.
+
+ Query params:
+ limit: Maximum number of patterns to return (default 20)
+
+ Response includes for each pattern:
+ - hash: Truncated tool signature hash (12 chars)
+ - compressions: Total compression events
+ - retrievals: Total retrieval events
+ - retrieval_rate: Percentage of compressions that triggered retrieval
+ - confidence: Confidence level in recommendations (0.0-1.0)
+ - skip_recommended: Whether TOIN recommends skipping compression
+ - optimal_max_items: Learned optimal max_items setting
+ """
+ toin = get_toin()
+ exported = toin.export_patterns()
+ patterns_data = exported.get("patterns", {})
+
+ # Convert to list and sort by sample_size
+ patterns_list = []
+ for sig_hash, pattern_dict in patterns_data.items():
+ sample_size = pattern_dict.get("sample_size", 0)
+ total_compressions = pattern_dict.get("total_compressions", 0)
+ total_retrievals = pattern_dict.get("total_retrievals", 0)
+ retrieval_rate = (
+ total_retrievals / total_compressions if total_compressions > 0 else 0.0
+ )
+
+ patterns_list.append(
+ {
+ "hash": sig_hash[:12],
+ "compressions": total_compressions,
+ "retrievals": total_retrievals,
+ "retrieval_rate": f"{retrieval_rate:.1%}",
+ "confidence": round(pattern_dict.get("confidence", 0.0), 3),
+ "skip_recommended": pattern_dict.get("skip_compression_recommended", False),
+ "optimal_max_items": pattern_dict.get("optimal_max_items", 20),
+ "sample_size": sample_size,
+ }
+ )
+
+ # Sort by sample_size descending
+ patterns_list.sort(key=lambda p: p["sample_size"], reverse=True)
+
+ # Remove sample_size from output (used only for sorting)
+ for p in patterns_list:
+ del p["sample_size"]
+
+ return patterns_list[:limit]
+
+ @app.get("/v1/toin/pattern/{hash_prefix}")
+ async def toin_pattern_detail(hash_prefix: str):
+ """Get detailed TOIN pattern info by hash prefix.
+
+ Searches for a pattern where the tool signature hash starts with
+ the provided prefix. Returns full pattern details if found.
+
+ Path params:
+ hash_prefix: Beginning of the tool signature hash (min 4 chars recommended)
+
+ Response: Full pattern.to_dict() with all learned statistics and recommendations.
+ """
+ toin = get_toin()
+ exported = toin.export_patterns()
+ patterns_data = exported.get("patterns", {})
+
+ # Search for pattern with matching hash prefix
+ for sig_hash, pattern_dict in patterns_data.items():
+ if sig_hash.startswith(hash_prefix):
+ return pattern_dict
+
+ raise HTTPException(
+ status_code=404, detail=f"No TOIN pattern found with hash starting with: {hash_prefix}"
+ )
+
+ @app.get("/v1/retrieve/{hash_key}")
+ async def ccr_retrieve_get(hash_key: str, query: str | None = None):
+ """GET version of CCR retrieve for easier testing."""
+ store = get_compression_store()
+
+ if query:
+ results = store.search(hash_key, query)
+ return {
+ "hash": hash_key,
+ "query": query,
+ "results": results,
+ "count": len(results),
+ }
+ else:
+ entry = store.retrieve(hash_key)
+ if entry:
+ return {
+ "hash": hash_key,
+ "original_content": entry.original_content,
+ "original_tokens": entry.original_tokens,
+ "original_item_count": entry.original_item_count,
+ "compressed_item_count": entry.compressed_item_count,
+ "tool_name": entry.tool_name,
+ "retrieval_count": entry.retrieval_count,
+ }
+ raise HTTPException(status_code=404, detail="Entry not found or expired")
+
+ # CCR Tool Call Handler - for agent frameworks to call when LLM uses headroom_retrieve
+ @app.post("/v1/retrieve/tool_call")
+ async def ccr_handle_tool_call(request: Request):
+ """Handle a CCR tool call from an LLM response.
+
+ This endpoint accepts tool call formats from various providers and returns
+ a properly formatted tool result. Agent frameworks can use this to handle
+ CCR tool calls without implementing the retrieval logic themselves.
+
+ Request body (Anthropic format):
+ {
+ "tool_call": {
+ "id": "toolu_123",
+ "name": "headroom_retrieve",
+ "input": {"hash": "abc123", "query": "optional search"}
+ },
+ "provider": "anthropic"
+ }
+
+ Request body (OpenAI format):
+ {
+ "tool_call": {
+ "id": "call_123",
+ "function": {
+ "name": "headroom_retrieve",
+ "arguments": "{\"hash\": \"abc123\"}"
+ }
+ },
+ "provider": "openai"
+ }
+
+ Response:
+ {
+ "tool_result": {...}, # Formatted for the provider
+ "success": true,
+ "data": {...} # Raw retrieval data
+ }
+ """
+ data = await request.json()
+ tool_call = data.get("tool_call", {})
+ provider = data.get("provider", "anthropic")
+
+ # Parse the tool call
+ hash_key, query = parse_tool_call(tool_call, provider)
+
+ if hash_key is None:
+ raise HTTPException(
+ status_code=400, detail=f"Invalid tool call or not a {CCR_TOOL_NAME} call"
+ )
+
+ # Perform retrieval
+ store = get_compression_store()
+
+ if query:
+ results = store.search(hash_key, query)
+ retrieval_data = {
+ "hash": hash_key,
+ "query": query,
+ "results": results,
+ "count": len(results),
+ }
+ else:
+ entry = store.retrieve(hash_key)
+ if entry:
+ retrieval_data = {
+ "hash": hash_key,
+ "original_content": entry.original_content,
+ "original_item_count": entry.original_item_count,
+ "compressed_item_count": entry.compressed_item_count,
+ }
+ else:
+ retrieval_data = {
+ "error": "Entry not found or expired (TTL: 5 minutes)",
+ "hash": hash_key,
+ }
+
+ # Format tool result for provider
+ tool_call_id = tool_call.get("id", "")
+ result_content = json.dumps(retrieval_data, indent=2)
+
+ if provider == "anthropic":
+ tool_result = {
+ "type": "tool_result",
+ "tool_use_id": tool_call_id,
+ "content": result_content,
+ }
+ elif provider == "openai":
+ tool_result = {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": result_content,
+ }
+ else:
+ tool_result = {
+ "tool_call_id": tool_call_id,
+ "content": result_content,
+ }
+
+ return {
+ "tool_result": tool_result,
+ "success": "error" not in retrieval_data,
+ "data": retrieval_data,
+ }
+
+ # Compression-only endpoint (for TypeScript SDK and other HTTP clients)
+ @app.post("/v1/compress")
+ async def compress_messages(request: Request):
+ return await proxy.handle_compress(request)
+
+ register_provider_routes(app, proxy)
+
+ return app
+
+
+def _get_code_aware_banner_status(config: ProxyConfig) -> str:
+ """Get code-aware compression status line for banner."""
+ if config.code_aware_enabled:
+ if is_tree_sitter_available():
+ return "ENABLED (AST-based)"
+ else:
+ return "NOT INSTALLED (pip install headroom-ai[code])"
+ else:
+ if is_tree_sitter_available():
+ return "DISABLED (remove --no-code-aware to enable)"
+ return "DISABLED"
+
+
+def run_server(
+ config: ProxyConfig | None = None,
+ workers: int = 1,
+ limit_concurrency: int = 1000,
+):
+ """Run the proxy server.
+
+ Args:
+ config: Proxy configuration
+ workers: Number of worker processes (use N for multi-core scaling)
+ limit_concurrency: Max concurrent connections before 503 response
+ """
+ if not FASTAPI_AVAILABLE:
+ print("ERROR: FastAPI required. Install: pip install fastapi uvicorn httpx")
+ sys.exit(1)
+
+ config = config or ProxyConfig()
+ app = create_app(config)
+
+ code_aware_status = _get_code_aware_banner_status(config)
+
+ # Format connection pool info
+ pool_info = f"max={config.max_connections}, keepalive={config.max_keepalive_connections}"
+ http2_status = "ENABLED" if config.http2 else "DISABLED"
+
+ backend_status = format_backend_status(
+ backend=config.backend,
+ anyllm_provider=config.anyllm_provider,
+ bedrock_region=config.bedrock_region,
+ )
+
+ print(f"""
+╔══════════════════════════════════════════════════════════════════════╗
+║ HEADROOM PROXY SERVER ║
+╠══════════════════════════════════════════════════════════════════════╣
+║ Version: 1.0.0 ║
+║ Listening: http://{config.host}:{config.port:<5} ║
+║ Workers: {workers:<3} Concurrency Limit: {limit_concurrency:<5} ║
+║ Backend: {backend_status:<59}║
+╠══════════════════════════════════════════════════════════════════════╣
+║ FEATURES: ║
+║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║
+║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║
+║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║
+║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║
+║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║
+║ Code-Aware: {code_aware_status:<52}║
+║ HTTP/2: {http2_status:<52}║
+║ Conn Pool: {pool_info:<52}║
+╠══════════════════════════════════════════════════════════════════════╣
+║ USAGE: ║
+║ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude ║
+║ Cursor: Set base URL in settings ║
+╠══════════════════════════════════════════════════════════════════════╣
+║ ENDPOINTS: ║
+║ /livez Process liveness ║
+║ /readyz Traffic readiness ║
+║ /health Aggregate health ║
+║ /stats Detailed statistics ║
+║ /metrics Prometheus metrics ║
+║ /cache/clear Clear response cache ║
+║ /v1/retrieve CCR: Retrieve compressed content ║
+║ /v1/retrieve/stats CCR: Compression store stats ║
+║ /v1/retrieve/tool_call CCR: Handle LLM tool calls ║
+║ /v1/feedback CCR: Feedback loop stats & patterns ║
+║ /v1/feedback/{{tool}} CCR: Compression hints for a tool ║
+║ /v1/telemetry Data flywheel: Telemetry stats ║
+║ /v1/telemetry/export Data flywheel: Export for aggregation ║
+║ /v1/telemetry/tools Data flywheel: Per-tool stats ║
+║ /v1/toin/stats TOIN: Overall intelligence stats ║
+║ /v1/toin/patterns TOIN: List learned patterns ║
+║ /v1/toin/pattern/{{hash}} TOIN: Pattern details by hash ║
+╚══════════════════════════════════════════════════════════════════════╝
+""")
+
+ uvicorn.run(
+ app,
+ host=config.host,
+ port=config.port,
+ log_level="warning",
+ workers=workers if workers > 1 else None, # None = single process (default)
+ limit_concurrency=limit_concurrency,
+ # Defense-in-depth: the loopback guard for /debug/* endpoints trusts
+ # request.client.host. uvicorn's ProxyHeadersMiddleware rewrites that
+ # from X-Forwarded-For when FORWARDED_ALLOW_IPS is broader than the
+ # default. Disabling proxy_headers here guarantees the guard sees the
+ # real peer address regardless of env.
+ proxy_headers=False,
+ )
+
+
+def _get_env_bool(name: str, default: bool) -> bool:
+ """Get boolean from environment variable."""
+ val = os.environ.get(name)
+ if val is None:
+ return default
+ return val.lower() in ("true", "1", "yes", "on")
+
+
+def _get_env_int(name: str, default: int) -> int:
+ """Get integer from environment variable."""
+ val = os.environ.get(name)
+ if val is None:
+ return default
+ try:
+ return int(val)
+ except ValueError:
+ return default
+
+
+def _get_env_float(name: str, default: float) -> float:
+ """Get float from environment variable."""
+ val = os.environ.get(name)
+ if val is None:
+ return default
+ try:
+ return float(val)
+ except ValueError:
+ return default
+
+
+def _get_env_str(name: str, default: str) -> str:
+ """Get string from environment variable."""
+ return os.environ.get(name, default)
+
+
+def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:
+ """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.
+
+ Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)
+ Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")
+
+ Returns:
+ Dict mapping tool names to CompressionProfile instances.
+ """
+ from headroom.config import PROFILE_PRESETS, CompressionProfile
+
+ profiles: dict[str, CompressionProfile] = {}
+ raw_entries: list[str] = list(cli_profiles)
+
+ # Also check env var
+ env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")
+ if env_val:
+ raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())
+
+ for entry in raw_entries:
+ if ":" not in entry:
+ logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)
+ continue
+ tool_name, level = entry.split(":", 1)
+ tool_name = tool_name.strip()
+ level = level.strip().lower()
+
+ if level in PROFILE_PRESETS:
+ profiles[tool_name] = PROFILE_PRESETS[level]
+ else:
+ logger.warning(
+ "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",
+ level,
+ tool_name,
+ )
+
+ return profiles
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Headroom Proxy Server")
+
+ # Server
+ parser.add_argument("--host", default="127.0.0.1")
+ parser.add_argument("--port", type=int, default=8787)
+ parser.add_argument(
+ "--openai-api-url", help=f"Custom OpenAI API URL (default: {DEFAULT_OPENAI_API_URL})"
+ )
+ parser.add_argument(
+ "--anthropic-api-url",
+ help=f"Custom Anthropic API URL (default: {DEFAULT_ANTHROPIC_API_URL})",
+ )
+
+ # Backend (anthropic direct, bedrock, openrouter, anyllm, or litellm-<provider>)
+ parser.add_argument(
+ "--backend",
+ default="anthropic",
+ help=(
+ "Backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', "
+ "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-hosted_vllm, litellm-vertex)"
+ ),
+ )
+ parser.add_argument(
+ "--bedrock-region",
+ default="us-west-2",
+ help="AWS region for Bedrock backend (default: us-west-2)",
+ )
+ parser.add_argument(
+ "--bedrock-profile",
+ help="AWS profile for Bedrock backend (default: use default credentials)",
+ )
+ parser.add_argument(
+ "--openrouter-api-key",
+ help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",
+ )
+ parser.add_argument(
+ "--anyllm-provider",
+ default="openai",
+ help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",
+ )
+
+ # Connection pool (scalability)
+ parser.add_argument(
+ "--max-connections",
+ type=int,
+ default=500,
+ help="Max connections to upstream APIs (default: 500)",
+ )
+ parser.add_argument(
+ "--max-keepalive", type=int, default=100, help="Max keepalive connections (default: 100)"
+ )
+ parser.add_argument(
+ "--no-http2",
+ action="store_true",
+ help="Disable HTTP/2 (enabled by default for better throughput)",
+ )
+ parser.add_argument(
+ "--workers",
+ type=int,
+ default=1,
+ help="Number of worker processes (default: 1, use N for multi-core)",
+ )
+ parser.add_argument(
+ "--limit-concurrency",
+ type=int,
+ default=1000,
+ help="Max concurrent connections before 503 (default: 1000)",
+ )
+
+ # Optimization
+ parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")
+ parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")
+ parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")
+ parser.add_argument(
+ "--tool-profile",
+ action="append",
+ default=[],
+ help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "
+ "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",
+ )
+
+ # Caching
+ parser.add_argument("--no-cache", action="store_true", help="Disable caching")
+ parser.add_argument("--cache-ttl", type=int, default=3600, help="Cache TTL seconds")
+
+ # Rate limiting
+ parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")
+ parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")
+ parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")
+
+ # Cost
+ parser.add_argument("--budget", type=float, help="Budget limit in USD")
+ parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")
+
+ # Logging
+ parser.add_argument("--log-file", help="Log file path")
+ parser.add_argument("--log-messages", action="store_true", help="Log full messages")
+
+ # Smart routing (content-aware compression)
+ parser.add_argument(
+ "--no-smart-routing",
+ action="store_true",
+ help="Disable smart routing (use legacy sequential pipeline)",
+ )
+
+ # Code-aware compression
+ parser.add_argument(
+ "--code-aware",
+ action="store_true",
+ help="Enable AST-based code compression (requires: pip install headroom-ai[code])",
+ )
+ parser.add_argument(
+ "--no-code-aware",
+ action="store_true",
+ help="Disable code-aware compression",
+ )
+
+ args = parser.parse_args()
+
+ # Environment variable defaults (HEADROOM_* prefix)
+ # CLI args override env vars, env vars override ProxyConfig defaults
+ env_smart_routing = _get_env_bool("HEADROOM_SMART_ROUTING", True)
+ env_code_aware = _get_env_bool("HEADROOM_CODE_AWARE_ENABLED", True)
+ env_optimize = _get_env_bool("HEADROOM_OPTIMIZE", True)
+ env_cache = _get_env_bool("HEADROOM_CACHE_ENABLED", True)
+ env_rate_limit = _get_env_bool("HEADROOM_RATE_LIMIT_ENABLED", True)
+
+ # Determine settings: CLI flags override env vars
+ # --no-X explicitly disables, --X explicitly enables, neither uses env var
+ smart_routing = env_smart_routing if not args.no_smart_routing else False
+ code_aware_enabled = (
+ env_code_aware
+ if not (args.code_aware or args.no_code_aware)
+ else (args.code_aware or not args.no_code_aware)
+ )
+ optimize = env_optimize if not args.no_optimize else False
+ cache_enabled = env_cache if not args.no_cache else False
+ rate_limit_enabled = env_rate_limit if not args.no_rate_limit else False
+
+ # Set OpenRouter API key from CLI if provided
+ if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:
+ os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key
+
+ # Parse per-tool compression profiles from CLI and env var
+ tool_profiles = _parse_tool_profiles(args.tool_profile)
+
+ config = ProxyConfig(
+ host=_get_env_str("HEADROOM_HOST", args.host),
+ port=_get_env_int("HEADROOM_PORT", args.port),
+ openai_api_url=_get_env_str("OPENAI_TARGET_API_URL", args.openai_api_url),
+ anthropic_api_url=_get_env_str("ANTHROPIC_TARGET_API_URL", args.anthropic_api_url),
+ # Backend settings
+ backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]
+ bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),
+ bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),
+ anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),
+ optimize=optimize,
+ min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),
+ max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),
+ cache_enabled=cache_enabled,
+ cache_ttl_seconds=_get_env_int("HEADROOM_CACHE_TTL", args.cache_ttl),
+ rate_limit_enabled=rate_limit_enabled,
+ rate_limit_requests_per_minute=_get_env_int("HEADROOM_RPM", args.rpm),
+ rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),
+ budget_limit_usd=args.budget,
+ budget_period=args.budget_period,
+ log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)
+ if args.log_file
+ else os.environ.get("HEADROOM_LOG_FILE"),
+ log_full_messages=args.log_messages or _get_env_bool("HEADROOM_LOG_MESSAGES", False),
+ smart_routing=smart_routing,
+ code_aware_enabled=code_aware_enabled,
+ # Connection pool settings
+ max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),
+ max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),
+ http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),
+ tool_profiles=tool_profiles if tool_profiles else None,
+ mode=normalize_proxy_mode(_get_env_str("HEADROOM_MODE", PROXY_MODE_TOKEN)),
+ )
+
+ # Get worker and concurrency settings
+ workers = _get_env_int("HEADROOM_WORKERS", args.workers)
+ limit_concurrency = _get_env_int("HEADROOM_LIMIT_CONCURRENCY", args.limit_concurrency)
+
+ run_server(config, workers=workers, limit_concurrency=limit_concurrency)
diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
index 0a6e74107..4b2fd52e0 100644
--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
@@ -1,17 +1,17 @@
-{
- "name": "headroom",
- "version": "0.9.2",
- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "author": {
- "name": "Headroom Contributors",
- "url": "https://github.com/chopratejas/headroom"
- },
- "homepage": "https://github.com/chopratejas/headroom",
- "repository": "https://github.com/chopratejas/headroom",
- "keywords": [
- "headroom",
- "hooks",
- "claude-code",
- "copilot-cli"
- ]
-}
+{
+ "name": "headroom",
+ "version": "0.10.0",
+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
+ "author": {
+ "name": "Headroom Contributors",
+ "url": "https://github.com/chopratejas/headroom"
+ },
+ "homepage": "https://github.com/chopratejas/headroom",
+ "repository": "https://github.com/chopratejas/headroom",
+ "keywords": [
+ "headroom",
+ "hooks",
+ "claude-code",
+ "copilot-cli"
+ ]
+}
diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.json
index bdb07cc0e..f8fe831d1 100644
--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json
+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json
@@ -1,18 +1,18 @@
-{
- "name": "headroom",
- "version": "0.9.2",
- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "author": {
- "name": "Headroom Contributors",
- "url": "https://github.com/chopratejas/headroom"
- },
- "homepage": "https://github.com/chopratejas/headroom",
- "repository": "https://github.com/chopratejas/headroom",
- "keywords": [
- "headroom",
- "hooks",
- "claude-code",
- "copilot-cli"
- ],
- "hooks": "./hooks"
-}
+{
+ "name": "headroom",
+ "version": "0.10.0",
+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
+ "author": {
+ "name": "Headroom Contributors",
+ "url": "https://github.com/chopratejas/headroom"
+ },
+ "homepage": "https://github.com/chopratejas/headroom",
+ "repository": "https://github.com/chopratejas/headroom",
+ "keywords": [
+ "headroom",
+ "hooks",
+ "claude-code",
+ "copilot-cli"
+ ],
+ "hooks": "./hooks"
+}
diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py
index c7387136e..f261c95dc 100644
--- a/tests/test_proxy_savings_history.py
+++ b/tests/test_proxy_savings_history.py
@@ -1,710 +1,711 @@
-"""Tests for durable proxy savings history."""
-
-from __future__ import annotations
-
-import asyncio
-import json
-from datetime import datetime, timedelta, timezone
-from types import SimpleNamespace
-
-import pytest
-
-pytest.importorskip("fastapi")
-
-from fastapi.testclient import TestClient
-
-import headroom.proxy.savings_tracker as savings_tracker_module
-from headroom.proxy.savings_tracker import HEADROOM_SAVINGS_PATH_ENV_VAR, SavingsTracker
-from headroom.proxy.server import ProxyConfig, create_app
-
-
-def _record_request(
- client: TestClient,
- *,
- model: str,
- tokens_saved: int,
- input_tokens: int = 120,
-) -> None:
- proxy = client.app.state.proxy
- if proxy.cost_tracker:
- proxy.cost_tracker.record_tokens(model, tokens_saved, input_tokens)
- asyncio.run(
- proxy.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=input_tokens,
- output_tokens=24,
- tokens_saved=tokens_saved,
- latency_ms=15.0,
- )
- )
-
-
-def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatch):
- override_path = tmp_path / "custom-savings.json"
- monkeypatch.setenv(HEADROOM_SAVINGS_PATH_ENV_VAR, str(override_path))
- assert savings_tracker_module.get_default_savings_storage_path() == str(override_path)
-
- monkeypatch.delenv(HEADROOM_SAVINGS_PATH_ENV_VAR, raising=False)
- default_path = savings_tracker_module.get_default_savings_storage_path()
- assert default_path.endswith(".headroom/proxy_savings.json")
-
- assert savings_tracker_module._parse_timestamp("") is None
- assert savings_tracker_module._parse_timestamp("not-a-timestamp") is None
- assert savings_tracker_module._parse_timestamp("2026-03-27T09:00:00") == datetime(
- 2026, 3, 27, 9, 0, tzinfo=timezone.utc
- )
-
- assert savings_tracker_module._coerce_int("7") == 7
- assert savings_tracker_module._coerce_int(-5) == 0
- assert savings_tracker_module._coerce_float("0.25") == pytest.approx(0.25)
- assert savings_tracker_module._coerce_float(-0.25) == 0.0
-
- assert savings_tracker_module._normalize_history_entry(
- ["2026-03-27T09:00:00Z", "12", "0.5"]
- ) == {
- "timestamp": "2026-03-27T09:00:00Z",
- "total_tokens_saved": 12,
- "compression_savings_usd": 0.5,
- "total_input_tokens": 0,
- "total_input_cost_usd": 0.0,
- }
- assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None
- assert savings_tracker_module._normalize_history_entry(object()) is None
-
-
-def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
- path = tmp_path / "proxy_savings.json"
- path.write_text(
- json.dumps(
- {
- "schema_version": 0,
- "lifetime": {
- "tokens_saved": 1,
- "compression_savings_usd": 0.001,
- },
- "history": [
- ["2026-03-24T08:00:00Z", 10, 0.01],
- {
- "timestamp": "2026-03-26T12:00:00Z",
- "total_tokens_saved": 20,
- "compression_savings_usd": 0.02,
- },
- {
- "timestamp": "2026-03-27T09:00:00Z",
- "total_tokens_saved": 30,
- "compression_savings_usd": 0.03,
- },
- {"timestamp": "bad", "total_tokens_saved": 999},
- ],
- }
- ),
- encoding="utf-8",
- )
-
- tracker = SavingsTracker(
- path=str(path),
- max_history_points=1,
- max_history_age_days=2,
- )
- snapshot = tracker.snapshot()
-
- assert snapshot["schema_version"] == 2
- assert snapshot["lifetime"] == {
- "requests": 0,
- "tokens_saved": 30,
- "compression_savings_usd": pytest.approx(0.03),
- "total_input_tokens": 0,
- "total_input_cost_usd": 0.0,
- }
- assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
- assert snapshot["history"] == [
- {
- "timestamp": "2026-03-27T09:00:00Z",
- "total_tokens_saved": 30,
- "compression_savings_usd": 0.03,
- "total_input_tokens": 0,
- "total_input_cost_usd": 0.0,
- }
- ]
- assert snapshot["retention"] == {
- "max_history_points": 1,
- "max_history_age_days": 2,
- "max_response_history_points": 500,
- }
-
-
-def test_non_dict_savings_state_resets_to_default(tmp_path):
- path = tmp_path / "proxy_savings.json"
- path.write_text("[]", encoding="utf-8")
-
- tracker = SavingsTracker(path=str(path))
- snapshot = tracker.snapshot()
-
- assert snapshot["lifetime"] == {
- "requests": 0,
- "tokens_saved": 0,
- "compression_savings_usd": 0.0,
- "total_input_tokens": 0,
- "total_input_cost_usd": 0.0,
- }
- assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
- assert snapshot["history"] == []
-
-
-def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamps(
- tmp_path, monkeypatch
-):
- path = tmp_path / "proxy_savings.json"
- tracker = SavingsTracker(path=str(path))
- monkeypatch.setattr(
- savings_tracker_module,
- "_estimate_compression_savings_usd",
- lambda model, tokens_saved: tokens_saved / 1000.0,
- )
-
- assert tracker.record_compression_savings(model="gpt-4o", tokens_saved=0) is False
- assert not path.exists()
-
- local_time = datetime(2026, 3, 27, 10, 0, tzinfo=timezone(timedelta(hours=2)))
- assert tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=10,
- total_input_tokens=120,
- total_input_cost_usd=0.24,
- timestamp=local_time,
- )
-
- fallback_time = datetime(2026, 3, 27, 12, 34, tzinfo=timezone.utc)
- monkeypatch.setattr(savings_tracker_module, "_utc_now", lambda: fallback_time)
- assert tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=5,
- total_input_tokens=180,
- total_input_cost_usd=0.36,
- timestamp="not-a-timestamp",
- )
-
- snapshot = tracker.snapshot()
- assert snapshot["history"] == [
- {
- "timestamp": "2026-03-27T08:00:00Z",
- "total_tokens_saved": 10,
- "compression_savings_usd": 0.01,
- "total_input_tokens": 120,
- "total_input_cost_usd": 0.24,
- },
- {
- "timestamp": "2026-03-27T12:34:00Z",
- "total_tokens_saved": 15,
- "compression_savings_usd": 0.015,
- "total_input_tokens": 180,
- "total_input_cost_usd": 0.36,
- },
- ]
-
- persisted = json.loads(path.read_text(encoding="utf-8"))
- assert persisted["lifetime"]["tokens_saved"] == 15
- assert persisted["lifetime"]["total_input_tokens"] == 180
- assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.36)
- assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"
-
-
-def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
- def fake_cost_per_token(*, model, prompt_tokens, completion_tokens):
- if model in {"gpt-4o", "anthropic/claude-sonnet-4-6"}:
- return {
- "model": model,
- "prompt_tokens": prompt_tokens,
- "completion_tokens": completion_tokens,
- }
- raise RuntimeError("unknown model")
-
- fake_litellm = SimpleNamespace(
- cost_per_token=fake_cost_per_token,
- model_cost={
- "anthropic/claude-sonnet-4-6": {"input_cost_per_token": 0.002},
- "gpt-4o": {"input_cost_per_token": 0.001},
- },
- )
- monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True)
- monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm)
-
- assert savings_tracker_module._resolve_litellm_model("gpt-4o") == "gpt-4o"
- assert (
- savings_tracker_module._resolve_litellm_model("claude-sonnet-4-6")
- == "anthropic/claude-sonnet-4-6"
- )
- assert savings_tracker_module._estimate_compression_savings_usd(
- "claude-sonnet-4-6", 100
- ) == pytest.approx(0.2)
- assert savings_tracker_module._estimate_input_cost_usd(
- "claude-sonnet-4-6",
- 100,
- cache_read_tokens=10,
- cache_write_tokens=5,
- uncached_input_tokens=85,
- ) == pytest.approx(0.2)
-
- fake_litellm.model_cost = {}
- assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
- assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
-
- monkeypatch.setattr(
- fake_litellm,
- "cost_per_token",
- lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom")),
- )
- assert savings_tracker_module._resolve_litellm_model("mystery-model") == "mystery-model"
- assert savings_tracker_module._estimate_compression_savings_usd("mystery-model", 100) == 0.0
-
- monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
- assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
- assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
-
-
-def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests(
- tmp_path, monkeypatch
-):
- path = tmp_path / "proxy_savings.json"
- tracker = SavingsTracker(path=str(path), display_session_inactivity_minutes=30)
- monkeypatch.setattr(
- savings_tracker_module,
- "_estimate_compression_savings_usd",
- lambda model, tokens_saved: tokens_saved / 1000.0,
- )
- monkeypatch.setattr(
- savings_tracker_module,
- "_estimate_input_cost_usd",
- lambda model, input_tokens, **kwargs: input_tokens / 1000.0,
- )
-
- tracker.record_request(
- model="gpt-4o",
- input_tokens=120,
- tokens_saved=0,
- timestamp="2026-03-27T09:00:00Z",
- )
- tracker.record_request(
- model="gpt-4o",
- input_tokens=80,
- tokens_saved=20,
- timestamp="2026-03-27T09:10:00Z",
- )
-
- monkeypatch.setattr(
- savings_tracker_module,
- "_utc_now",
- lambda: datetime(2026, 3, 27, 9, 15, tzinfo=timezone.utc),
- )
- active_session = tracker.snapshot()["display_session"]
- assert active_session == {
- "requests": 2,
- "tokens_saved": 20,
- "compression_savings_usd": pytest.approx(0.02),
- "total_input_tokens": 200,
- "total_input_cost_usd": pytest.approx(0.2),
- "savings_percent": pytest.approx(9.09),
- "started_at": "2026-03-27T09:00:00Z",
- "last_activity_at": "2026-03-27T09:10:00Z",
- }
-
- monkeypatch.setattr(
- savings_tracker_module,
- "_utc_now",
- lambda: datetime(2026, 3, 27, 9, 45, tzinfo=timezone.utc),
- )
- assert tracker.snapshot()["display_session"] == savings_tracker_module._empty_display_session()
-
- tracker.record_request(
- model="gpt-4o",
- input_tokens=50,
- tokens_saved=5,
- timestamp="2026-03-27T10:05:00Z",
- )
-
- monkeypatch.setattr(
- savings_tracker_module,
- "_utc_now",
- lambda: datetime(2026, 3, 27, 10, 10, tzinfo=timezone.utc),
- )
- rolled = tracker.snapshot()
- assert rolled["lifetime"]["requests"] == 3
- assert rolled["display_session"] == {
- "requests": 1,
- "tokens_saved": 5,
- "compression_savings_usd": pytest.approx(0.005),
- "total_input_tokens": 50,
- "total_input_cost_usd": pytest.approx(0.05),
- "savings_percent": pytest.approx(9.09),
- "started_at": "2026-03-27T10:05:00Z",
- "last_activity_at": "2026-03-27T10:05:00Z",
- }
-
-
-def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
- path = tmp_path / "proxy_savings.json"
- tracker = SavingsTracker(
- path=str(path),
- max_history_points=100,
- max_history_age_days=30,
- )
- monkeypatch.setattr(
- "headroom.proxy.savings_tracker._estimate_compression_savings_usd",
- lambda model, tokens_saved: tokens_saved / 1000.0,
- )
-
- tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=100,
- total_input_tokens=120,
- total_input_cost_usd=0.24,
- timestamp="2026-03-27T09:10:00Z",
- )
- tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=50,
- total_input_tokens=210,
- total_input_cost_usd=0.42,
- timestamp="2026-03-27T09:40:00Z",
- )
- tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=25,
- total_input_tokens=300,
- total_input_cost_usd=0.63,
- timestamp="2026-03-27T10:05:00Z",
- )
- tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=10,
- total_input_tokens=360,
- total_input_cost_usd=0.75,
- timestamp="2026-03-28T08:00:00Z",
- )
- tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=20,
- total_input_tokens=450,
- total_input_cost_usd=0.93,
- timestamp="2026-04-02T14:00:00Z",
- )
-
- response = tracker.history_response()
-
- assert response["lifetime"]["tokens_saved"] == 205
- assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)
- assert response["lifetime"]["total_input_tokens"] == 450
- assert response["lifetime"]["total_input_cost_usd"] == pytest.approx(0.93)
- assert len(response["history"]) == 5
-
- hourly = response["series"]["hourly"]
- assert [point["timestamp"] for point in hourly] == [
- "2026-03-27T09:00:00Z",
- "2026-03-27T10:00:00Z",
- "2026-03-28T08:00:00Z",
- "2026-04-02T14:00:00Z",
- ]
- assert hourly[0]["tokens_saved"] == 150
- assert hourly[0]["total_tokens_saved"] == 150
- assert hourly[0]["total_input_tokens_delta"] == 210
- assert hourly[0]["total_input_tokens"] == 210
- assert hourly[0]["total_input_cost_usd_delta"] == pytest.approx(0.42)
- assert hourly[0]["total_input_cost_usd"] == pytest.approx(0.42)
- assert hourly[1]["tokens_saved"] == 25
- assert hourly[1]["total_tokens_saved"] == 175
- assert hourly[1]["total_input_tokens_delta"] == 90
- assert hourly[1]["total_input_tokens"] == 300
- assert hourly[1]["total_input_cost_usd_delta"] == pytest.approx(0.21)
- assert hourly[1]["total_input_cost_usd"] == pytest.approx(0.63)
- assert hourly[2]["tokens_saved"] == 10
- assert hourly[2]["total_tokens_saved"] == 185
- assert hourly[2]["total_input_tokens_delta"] == 60
- assert hourly[2]["total_input_tokens"] == 360
- assert hourly[2]["total_input_cost_usd_delta"] == pytest.approx(0.12)
- assert hourly[2]["total_input_cost_usd"] == pytest.approx(0.75)
- assert hourly[3]["tokens_saved"] == 20
- assert hourly[3]["total_tokens_saved"] == 205
- assert hourly[3]["total_input_tokens_delta"] == 90
- assert hourly[3]["total_input_tokens"] == 450
- assert hourly[3]["total_input_cost_usd_delta"] == pytest.approx(0.18)
- assert hourly[3]["total_input_cost_usd"] == pytest.approx(0.93)
-
- daily = response["series"]["daily"]
- assert [point["timestamp"] for point in daily] == [
- "2026-03-27T00:00:00Z",
- "2026-03-28T00:00:00Z",
- "2026-04-02T00:00:00Z",
- ]
- assert daily[0]["tokens_saved"] == 175
- assert daily[0]["total_tokens_saved"] == 175
- assert daily[0]["total_input_tokens_delta"] == 300
- assert daily[0]["total_input_tokens"] == 300
- assert daily[0]["total_input_cost_usd_delta"] == pytest.approx(0.63)
- assert daily[0]["total_input_cost_usd"] == pytest.approx(0.63)
- assert daily[1]["tokens_saved"] == 10
- assert daily[1]["total_tokens_saved"] == 185
- assert daily[1]["total_input_tokens_delta"] == 60
- assert daily[1]["total_input_tokens"] == 360
- assert daily[1]["total_input_cost_usd_delta"] == pytest.approx(0.12)
- assert daily[1]["total_input_cost_usd"] == pytest.approx(0.75)
- assert daily[2]["tokens_saved"] == 20
- assert daily[2]["total_tokens_saved"] == 205
- assert daily[2]["total_input_tokens_delta"] == 90
- assert daily[2]["total_input_tokens"] == 450
- assert daily[2]["total_input_cost_usd_delta"] == pytest.approx(0.18)
- assert daily[2]["total_input_cost_usd"] == pytest.approx(0.93)
-
- weekly = response["series"]["weekly"]
- assert [point["timestamp"] for point in weekly] == [
- "2026-03-23T00:00:00Z",
- "2026-03-30T00:00:00Z",
- ]
- assert weekly[0]["tokens_saved"] == 185
- assert weekly[0]["total_tokens_saved"] == 185
- assert weekly[1]["tokens_saved"] == 20
- assert weekly[1]["total_tokens_saved"] == 205
-
- monthly = response["series"]["monthly"]
- assert [point["timestamp"] for point in monthly] == [
- "2026-03-01T00:00:00Z",
- "2026-04-01T00:00:00Z",
- ]
- assert monthly[0]["tokens_saved"] == 185
- assert monthly[0]["total_tokens_saved"] == 185
- assert monthly[1]["tokens_saved"] == 20
- assert monthly[1]["total_tokens_saved"] == 205
-
- assert response["exports"]["available_formats"] == ["json", "csv"]
- assert response["exports"]["available_series"] == [
- "history",
- "hourly",
- "daily",
- "weekly",
- "monthly",
- ]
-
-
-def test_stats_history_defaults_to_compact_history_but_can_return_full_history(
- tmp_path, monkeypatch
-):
- path = tmp_path / "proxy_savings.json"
- tracker = SavingsTracker(
- path=str(path),
- max_history_points=100,
- max_history_age_days=30,
- max_response_history_points=5,
- )
- monkeypatch.setattr(
- "headroom.proxy.savings_tracker._estimate_compression_savings_usd",
- lambda model, tokens_saved: tokens_saved / 1000.0,
- )
-
- for i in range(8):
- tracker.record_compression_savings(
- model="gpt-4o",
- tokens_saved=10,
- total_input_tokens=(i + 1) * 100,
- total_input_cost_usd=(i + 1) * 0.1,
- timestamp=f"2026-03-27T09:{i:02d}:00Z",
- )
-
- compact = tracker.history_response()
- assert compact["history_summary"] == {
- "mode": "compact",
- "stored_points": 8,
- "returned_points": 5,
- "compacted": True,
- }
- assert len(compact["history"]) == 5
- assert compact["history"][0]["timestamp"] == "2026-03-27T09:00:00Z"
- assert compact["history"][-1]["timestamp"] == "2026-03-27T09:07:00Z"
-
- full = tracker.history_response(history_mode="full")
- assert full["history_summary"] == {
- "mode": "full",
- "stored_points": 8,
- "returned_points": 8,
- "compacted": False,
- }
- assert len(full["history"]) == 8
-
- none = tracker.history_response(history_mode="none")
- assert none["history"] == []
- assert none["history_summary"] == {
- "mode": "none",
- "stored_points": 8,
- "returned_points": 0,
- "compacted": True,
- }
-
-
-def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):
- savings_path = tmp_path / "proxy_savings.json"
- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
- monkeypatch.setattr(
- "headroom.proxy.server.CostTracker._get_cache_prices",
- lambda self, model: (0.001, 0.0015, 0.002),
- )
-
- config = ProxyConfig(
- cache_enabled=False,
- rate_limit_enabled=False,
- log_requests=False,
- )
-
- with TestClient(create_app(config)) as client:
- _record_request(client, model="gpt-4o", tokens_saved=40)
-
- stats = client.get("/stats")
- assert stats.status_code == 200
- stats_data = stats.json()
- assert "savings_history" in stats_data
- assert "persistent_savings" in stats_data
- assert all(len(point) == 2 for point in stats_data["savings_history"])
- assert stats_data["persistent_savings"]["lifetime"]["tokens_saved"] == 40
- assert stats_data["persistent_savings"]["storage_path"] == str(savings_path)
-
- history = client.get("/stats-history")
- assert history.status_code == 200
- history_data = history.json()
- assert history_data["schema_version"] == 2
- assert history_data["storage_path"] == str(savings_path)
- assert history_data["lifetime"]["tokens_saved"] == 40
- assert history_data["lifetime"]["total_input_tokens"] == 120
- assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
- assert history_data["display_session"]["requests"] == 1
- assert history_data["display_session"]["tokens_saved"] == 40
- assert history_data["display_session"]["total_input_tokens"] == 120
- assert history_data["display_session"]["savings_percent"] == pytest.approx(25.0)
- assert list(history_data["series"].keys()) == [
- "hourly",
- "daily",
- "weekly",
- "monthly",
- ]
- assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
- assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
- assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(
- 0.24
- )
- assert history_data["history_summary"] == {
- "mode": "compact",
- "stored_points": 1,
- "returned_points": 1,
- "compacted": False,
- }
-
- assert stats_data["display_session"] == history_data["display_session"]
- assert (
- stats_data["persistent_savings"]["display_session"] == history_data["display_session"]
- )
-
- with TestClient(create_app(config)) as client:
- history = client.get("/stats-history")
- assert history.status_code == 200
- assert history.json()["lifetime"]["tokens_saved"] == 40
- assert history.json()["display_session"]["requests"] == 1
-
- _record_request(client, model="gpt-4o", tokens_saved=15)
-
- updated = client.get("/stats-history").json()
- assert updated["lifetime"]["tokens_saved"] == 55
- assert updated["lifetime"]["total_input_tokens"] == 240
- assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
- assert updated["lifetime"]["requests"] == 2
- assert len(updated["history"]) == 2
- assert updated["display_session"]["requests"] == 2
- assert updated["display_session"]["tokens_saved"] == 55
- assert updated["display_session"]["total_input_tokens"] == 240
- assert updated["display_session"]["savings_percent"] == pytest.approx(18.64)
- assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
- assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
-
- full = client.get("/stats-history?history_mode=full").json()
- assert full["history_summary"]["mode"] == "full"
- assert full["history_summary"]["stored_points"] == 2
- assert full["history_summary"]["returned_points"] == 2
-
- persisted = json.loads(savings_path.read_text())
- assert persisted["lifetime"]["tokens_saved"] == 55
- assert persisted["lifetime"]["total_input_tokens"] == 240
- assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
- assert persisted["display_session"]["requests"] == 2
-
-
-def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
- savings_path = tmp_path / "proxy_savings.json"
- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
- monkeypatch.setattr(
- "headroom.proxy.server.CostTracker._get_cache_prices",
- lambda self, model: (0.001, 0.0015, 0.002),
- )
-
- config = ProxyConfig(
- cache_enabled=False,
- rate_limit_enabled=False,
- log_requests=False,
- )
-
- with TestClient(create_app(config)) as client:
- _record_request(client, model="gpt-4o", tokens_saved=40)
- _record_request(client, model="gpt-4o", tokens_saved=10)
-
- response = client.get("/stats-history?format=csv&series=daily")
- assert response.status_code == 200
- assert response.headers["content-type"].startswith("text/csv")
- assert (
- 'attachment; filename="headroom-stats-history-daily.csv"'
- == response.headers["content-disposition"]
- )
- lines = response.text.strip().splitlines()
- assert lines[0] == (
- "timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"
- "compression_savings_usd,total_input_tokens_delta,total_input_tokens,"
- "total_input_cost_usd_delta,total_input_cost_usd"
- )
- assert len(lines) >= 2
- assert "total_tokens_saved" in lines[0]
- assert "total_input_cost_usd" in lines[0]
-
-
-def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):
- savings_path = tmp_path / "proxy_savings.json"
- savings_path.write_text("{not valid json", encoding="utf-8")
- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
-
- config = ProxyConfig(
- cache_enabled=False,
- rate_limit_enabled=False,
- log_requests=False,
- )
-
- with TestClient(create_app(config)) as client:
- response = client.get("/stats-history")
- assert response.status_code == 200
- data = response.json()
- assert data["lifetime"]["tokens_saved"] == 0
- assert data["history"] == []
-
-
-def test_dashboard_includes_history_toggle_and_endpoint(tmp_path, monkeypatch):
- savings_path = tmp_path / "proxy_savings.json"
- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
-
- config = ProxyConfig(
- cache_enabled=False,
- rate_limit_enabled=False,
- log_requests=False,
- )
-
- with TestClient(create_app(config)) as client:
- response = client.get("/dashboard")
- assert response.status_code == 200
- html = response.text
- assert "Session" in html
- assert "Historical" in html
- assert "fetch('/stats-history')" in html
- assert "Export CSV" in html
- assert "Weekly Savings" in html
- assert "Monthly Savings" in html
+"""Tests for durable proxy savings history."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+import headroom.proxy.savings_tracker as savings_tracker_module
+from headroom.proxy.savings_tracker import HEADROOM_SAVINGS_PATH_ENV_VAR, SavingsTracker
+from headroom.proxy.server import ProxyConfig, create_app
+
+
+def _record_request(
+ client: TestClient,
+ *,
+ model: str,
+ tokens_saved: int,
+ input_tokens: int = 120,
+) -> None:
+ proxy = client.app.state.proxy
+ if proxy.cost_tracker:
+ proxy.cost_tracker.record_tokens(model, tokens_saved, input_tokens)
+ asyncio.run(
+ proxy.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=input_tokens,
+ output_tokens=24,
+ tokens_saved=tokens_saved,
+ latency_ms=15.0,
+ )
+ )
+
+
+def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatch):
+ override_path = tmp_path / "custom-savings.json"
+ monkeypatch.setenv(HEADROOM_SAVINGS_PATH_ENV_VAR, str(override_path))
+ assert savings_tracker_module.get_default_savings_storage_path() == str(override_path)
+
+ monkeypatch.delenv(HEADROOM_SAVINGS_PATH_ENV_VAR, raising=False)
+ default_path = savings_tracker_module.get_default_savings_storage_path()
+ assert Path(default_path).as_posix().endswith(".headroom/proxy_savings.json")
+
+ assert savings_tracker_module._parse_timestamp("") is None
+ assert savings_tracker_module._parse_timestamp("not-a-timestamp") is None
+ assert savings_tracker_module._parse_timestamp("2026-03-27T09:00:00") == datetime(
+ 2026, 3, 27, 9, 0, tzinfo=timezone.utc
+ )
+
+ assert savings_tracker_module._coerce_int("7") == 7
+ assert savings_tracker_module._coerce_int(-5) == 0
+ assert savings_tracker_module._coerce_float("0.25") == pytest.approx(0.25)
+ assert savings_tracker_module._coerce_float(-0.25) == 0.0
+
+ assert savings_tracker_module._normalize_history_entry(
+ ["2026-03-27T09:00:00Z", "12", "0.5"]
+ ) == {
+ "timestamp": "2026-03-27T09:00:00Z",
+ "total_tokens_saved": 12,
+ "compression_savings_usd": 0.5,
+ "total_input_tokens": 0,
+ "total_input_cost_usd": 0.0,
+ }
+ assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None
+ assert savings_tracker_module._normalize_history_entry(object()) is None
+
+
+def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
+ path = tmp_path / "proxy_savings.json"
+ path.write_text(
+ json.dumps(
+ {
+ "schema_version": 0,
+ "lifetime": {
+ "tokens_saved": 1,
+ "compression_savings_usd": 0.001,
+ },
+ "history": [
+ ["2026-03-24T08:00:00Z", 10, 0.01],
+ {
+ "timestamp": "2026-03-26T12:00:00Z",
+ "total_tokens_saved": 20,
+ "compression_savings_usd": 0.02,
+ },
+ {
+ "timestamp": "2026-03-27T09:00:00Z",
+ "total_tokens_saved": 30,
+ "compression_savings_usd": 0.03,
+ },
+ {"timestamp": "bad", "total_tokens_saved": 999},
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ tracker = SavingsTracker(
+ path=str(path),
+ max_history_points=1,
+ max_history_age_days=2,
+ )
+ snapshot = tracker.snapshot()
+
+ assert snapshot["schema_version"] == 2
+ assert snapshot["lifetime"] == {
+ "requests": 0,
+ "tokens_saved": 30,
+ "compression_savings_usd": pytest.approx(0.03),
+ "total_input_tokens": 0,
+ "total_input_cost_usd": 0.0,
+ }
+ assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
+ assert snapshot["history"] == [
+ {
+ "timestamp": "2026-03-27T09:00:00Z",
+ "total_tokens_saved": 30,
+ "compression_savings_usd": 0.03,
+ "total_input_tokens": 0,
+ "total_input_cost_usd": 0.0,
+ }
+ ]
+ assert snapshot["retention"] == {
+ "max_history_points": 1,
+ "max_history_age_days": 2,
+ "max_response_history_points": 500,
+ }
+
+
+def test_non_dict_savings_state_resets_to_default(tmp_path):
+ path = tmp_path / "proxy_savings.json"
+ path.write_text("[]", encoding="utf-8")
+
+ tracker = SavingsTracker(path=str(path))
+ snapshot = tracker.snapshot()
+
+ assert snapshot["lifetime"] == {
+ "requests": 0,
+ "tokens_saved": 0,
+ "compression_savings_usd": 0.0,
+ "total_input_tokens": 0,
+ "total_input_cost_usd": 0.0,
+ }
+ assert snapshot["display_session"] == savings_tracker_module._empty_display_session()
+ assert snapshot["history"] == []
+
+
+def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamps(
+ tmp_path, monkeypatch
+):
+ path = tmp_path / "proxy_savings.json"
+ tracker = SavingsTracker(path=str(path))
+ monkeypatch.setattr(
+ savings_tracker_module,
+ "_estimate_compression_savings_usd",
+ lambda model, tokens_saved: tokens_saved / 1000.0,
+ )
+
+ assert tracker.record_compression_savings(model="gpt-4o", tokens_saved=0) is False
+ assert not path.exists()
+
+ local_time = datetime(2026, 3, 27, 10, 0, tzinfo=timezone(timedelta(hours=2)))
+ assert tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=10,
+ total_input_tokens=120,
+ total_input_cost_usd=0.24,
+ timestamp=local_time,
+ )
+
+ fallback_time = datetime(2026, 3, 27, 12, 34, tzinfo=timezone.utc)
+ monkeypatch.setattr(savings_tracker_module, "_utc_now", lambda: fallback_time)
+ assert tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=5,
+ total_input_tokens=180,
+ total_input_cost_usd=0.36,
+ timestamp="not-a-timestamp",
+ )
+
+ snapshot = tracker.snapshot()
+ assert snapshot["history"] == [
+ {
+ "timestamp": "2026-03-27T08:00:00Z",
+ "total_tokens_saved": 10,
+ "compression_savings_usd": 0.01,
+ "total_input_tokens": 120,
+ "total_input_cost_usd": 0.24,
+ },
+ {
+ "timestamp": "2026-03-27T12:34:00Z",
+ "total_tokens_saved": 15,
+ "compression_savings_usd": 0.015,
+ "total_input_tokens": 180,
+ "total_input_cost_usd": 0.36,
+ },
+ ]
+
+ persisted = json.loads(path.read_text(encoding="utf-8"))
+ assert persisted["lifetime"]["tokens_saved"] == 15
+ assert persisted["lifetime"]["total_input_tokens"] == 180
+ assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.36)
+ assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"
+
+
+def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):
+ def fake_cost_per_token(*, model, prompt_tokens, completion_tokens):
+ if model in {"gpt-4o", "anthropic/claude-sonnet-4-6"}:
+ return {
+ "model": model,
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ }
+ raise RuntimeError("unknown model")
+
+ fake_litellm = SimpleNamespace(
+ cost_per_token=fake_cost_per_token,
+ model_cost={
+ "anthropic/claude-sonnet-4-6": {"input_cost_per_token": 0.002},
+ "gpt-4o": {"input_cost_per_token": 0.001},
+ },
+ )
+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True)
+ monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm)
+
+ assert savings_tracker_module._resolve_litellm_model("gpt-4o") == "gpt-4o"
+ assert (
+ savings_tracker_module._resolve_litellm_model("claude-sonnet-4-6")
+ == "anthropic/claude-sonnet-4-6"
+ )
+ assert savings_tracker_module._estimate_compression_savings_usd(
+ "claude-sonnet-4-6", 100
+ ) == pytest.approx(0.2)
+ assert savings_tracker_module._estimate_input_cost_usd(
+ "claude-sonnet-4-6",
+ 100,
+ cache_read_tokens=10,
+ cache_write_tokens=5,
+ uncached_input_tokens=85,
+ ) == pytest.approx(0.2)
+
+ fake_litellm.model_cost = {}
+ assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
+ assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
+
+ monkeypatch.setattr(
+ fake_litellm,
+ "cost_per_token",
+ lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom")),
+ )
+ assert savings_tracker_module._resolve_litellm_model("mystery-model") == "mystery-model"
+ assert savings_tracker_module._estimate_compression_savings_usd("mystery-model", 100) == 0.0
+
+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)
+ assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0
+ assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0
+
+
+def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests(
+ tmp_path, monkeypatch
+):
+ path = tmp_path / "proxy_savings.json"
+ tracker = SavingsTracker(path=str(path), display_session_inactivity_minutes=30)
+ monkeypatch.setattr(
+ savings_tracker_module,
+ "_estimate_compression_savings_usd",
+ lambda model, tokens_saved: tokens_saved / 1000.0,
+ )
+ monkeypatch.setattr(
+ savings_tracker_module,
+ "_estimate_input_cost_usd",
+ lambda model, input_tokens, **kwargs: input_tokens / 1000.0,
+ )
+
+ tracker.record_request(
+ model="gpt-4o",
+ input_tokens=120,
+ tokens_saved=0,
+ timestamp="2026-03-27T09:00:00Z",
+ )
+ tracker.record_request(
+ model="gpt-4o",
+ input_tokens=80,
+ tokens_saved=20,
+ timestamp="2026-03-27T09:10:00Z",
+ )
+
+ monkeypatch.setattr(
+ savings_tracker_module,
+ "_utc_now",
+ lambda: datetime(2026, 3, 27, 9, 15, tzinfo=timezone.utc),
+ )
+ active_session = tracker.snapshot()["display_session"]
+ assert active_session == {
+ "requests": 2,
+ "tokens_saved": 20,
+ "compression_savings_usd": pytest.approx(0.02),
+ "total_input_tokens": 200,
+ "total_input_cost_usd": pytest.approx(0.2),
+ "savings_percent": pytest.approx(9.09),
+ "started_at": "2026-03-27T09:00:00Z",
+ "last_activity_at": "2026-03-27T09:10:00Z",
+ }
+
+ monkeypatch.setattr(
+ savings_tracker_module,
+ "_utc_now",
+ lambda: datetime(2026, 3, 27, 9, 45, tzinfo=timezone.utc),
+ )
+ assert tracker.snapshot()["display_session"] == savings_tracker_module._empty_display_session()
+
+ tracker.record_request(
+ model="gpt-4o",
+ input_tokens=50,
+ tokens_saved=5,
+ timestamp="2026-03-27T10:05:00Z",
+ )
+
+ monkeypatch.setattr(
+ savings_tracker_module,
+ "_utc_now",
+ lambda: datetime(2026, 3, 27, 10, 10, tzinfo=timezone.utc),
+ )
+ rolled = tracker.snapshot()
+ assert rolled["lifetime"]["requests"] == 3
+ assert rolled["display_session"] == {
+ "requests": 1,
+ "tokens_saved": 5,
+ "compression_savings_usd": pytest.approx(0.005),
+ "total_input_tokens": 50,
+ "total_input_cost_usd": pytest.approx(0.05),
+ "savings_percent": pytest.approx(9.09),
+ "started_at": "2026-03-27T10:05:00Z",
+ "last_activity_at": "2026-03-27T10:05:00Z",
+ }
+
+
+def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):
+ path = tmp_path / "proxy_savings.json"
+ tracker = SavingsTracker(
+ path=str(path),
+ max_history_points=100,
+ max_history_age_days=30,
+ )
+ monkeypatch.setattr(
+ "headroom.proxy.savings_tracker._estimate_compression_savings_usd",
+ lambda model, tokens_saved: tokens_saved / 1000.0,
+ )
+
+ tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=100,
+ total_input_tokens=120,
+ total_input_cost_usd=0.24,
+ timestamp="2026-03-27T09:10:00Z",
+ )
+ tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=50,
+ total_input_tokens=210,
+ total_input_cost_usd=0.42,
+ timestamp="2026-03-27T09:40:00Z",
+ )
+ tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=25,
+ total_input_tokens=300,
+ total_input_cost_usd=0.63,
+ timestamp="2026-03-27T10:05:00Z",
+ )
+ tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=10,
+ total_input_tokens=360,
+ total_input_cost_usd=0.75,
+ timestamp="2026-03-28T08:00:00Z",
+ )
+ tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=20,
+ total_input_tokens=450,
+ total_input_cost_usd=0.93,
+ timestamp="2026-04-02T14:00:00Z",
+ )
+
+ response = tracker.history_response()
+
+ assert response["lifetime"]["tokens_saved"] == 205
+ assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)
+ assert response["lifetime"]["total_input_tokens"] == 450
+ assert response["lifetime"]["total_input_cost_usd"] == pytest.approx(0.93)
+ assert len(response["history"]) == 5
+
+ hourly = response["series"]["hourly"]
+ assert [point["timestamp"] for point in hourly] == [
+ "2026-03-27T09:00:00Z",
+ "2026-03-27T10:00:00Z",
+ "2026-03-28T08:00:00Z",
+ "2026-04-02T14:00:00Z",
+ ]
+ assert hourly[0]["tokens_saved"] == 150
+ assert hourly[0]["total_tokens_saved"] == 150
+ assert hourly[0]["total_input_tokens_delta"] == 210
+ assert hourly[0]["total_input_tokens"] == 210
+ assert hourly[0]["total_input_cost_usd_delta"] == pytest.approx(0.42)
+ assert hourly[0]["total_input_cost_usd"] == pytest.approx(0.42)
+ assert hourly[1]["tokens_saved"] == 25
+ assert hourly[1]["total_tokens_saved"] == 175
+ assert hourly[1]["total_input_tokens_delta"] == 90
+ assert hourly[1]["total_input_tokens"] == 300
+ assert hourly[1]["total_input_cost_usd_delta"] == pytest.approx(0.21)
+ assert hourly[1]["total_input_cost_usd"] == pytest.approx(0.63)
+ assert hourly[2]["tokens_saved"] == 10
+ assert hourly[2]["total_tokens_saved"] == 185
+ assert hourly[2]["total_input_tokens_delta"] == 60
+ assert hourly[2]["total_input_tokens"] == 360
+ assert hourly[2]["total_input_cost_usd_delta"] == pytest.approx(0.12)
+ assert hourly[2]["total_input_cost_usd"] == pytest.approx(0.75)
+ assert hourly[3]["tokens_saved"] == 20
+ assert hourly[3]["total_tokens_saved"] == 205
+ assert hourly[3]["total_input_tokens_delta"] == 90
+ assert hourly[3]["total_input_tokens"] == 450
+ assert hourly[3]["total_input_cost_usd_delta"] == pytest.approx(0.18)
+ assert hourly[3]["total_input_cost_usd"] == pytest.approx(0.93)
+
+ daily = response["series"]["daily"]
+ assert [point["timestamp"] for point in daily] == [
+ "2026-03-27T00:00:00Z",
+ "2026-03-28T00:00:00Z",
+ "2026-04-02T00:00:00Z",
+ ]
+ assert daily[0]["tokens_saved"] == 175
+ assert daily[0]["total_tokens_saved"] == 175
+ assert daily[0]["total_input_tokens_delta"] == 300
+ assert daily[0]["total_input_tokens"] == 300
+ assert daily[0]["total_input_cost_usd_delta"] == pytest.approx(0.63)
+ assert daily[0]["total_input_cost_usd"] == pytest.approx(0.63)
+ assert daily[1]["tokens_saved"] == 10
+ assert daily[1]["total_tokens_saved"] == 185
+ assert daily[1]["total_input_tokens_delta"] == 60
+ assert daily[1]["total_input_tokens"] == 360
+ assert daily[1]["total_input_cost_usd_delta"] == pytest.approx(0.12)
+ assert daily[1]["total_input_cost_usd"] == pytest.approx(0.75)
+ assert daily[2]["tokens_saved"] == 20
+ assert daily[2]["total_tokens_saved"] == 205
+ assert daily[2]["total_input_tokens_delta"] == 90
+ assert daily[2]["total_input_tokens"] == 450
+ assert daily[2]["total_input_cost_usd_delta"] == pytest.approx(0.18)
+ assert daily[2]["total_input_cost_usd"] == pytest.approx(0.93)
+
+ weekly = response["series"]["weekly"]
+ assert [point["timestamp"] for point in weekly] == [
+ "2026-03-23T00:00:00Z",
+ "2026-03-30T00:00:00Z",
+ ]
+ assert weekly[0]["tokens_saved"] == 185
+ assert weekly[0]["total_tokens_saved"] == 185
+ assert weekly[1]["tokens_saved"] == 20
+ assert weekly[1]["total_tokens_saved"] == 205
+
+ monthly = response["series"]["monthly"]
+ assert [point["timestamp"] for point in monthly] == [
+ "2026-03-01T00:00:00Z",
+ "2026-04-01T00:00:00Z",
+ ]
+ assert monthly[0]["tokens_saved"] == 185
+ assert monthly[0]["total_tokens_saved"] == 185
+ assert monthly[1]["tokens_saved"] == 20
+ assert monthly[1]["total_tokens_saved"] == 205
+
+ assert response["exports"]["available_formats"] == ["json", "csv"]
+ assert response["exports"]["available_series"] == [
+ "history",
+ "hourly",
+ "daily",
+ "weekly",
+ "monthly",
+ ]
+
+
+def test_stats_history_defaults_to_compact_history_but_can_return_full_history(
+ tmp_path, monkeypatch
+):
+ path = tmp_path / "proxy_savings.json"
+ tracker = SavingsTracker(
+ path=str(path),
+ max_history_points=100,
+ max_history_age_days=30,
+ max_response_history_points=5,
+ )
+ monkeypatch.setattr(
+ "headroom.proxy.savings_tracker._estimate_compression_savings_usd",
+ lambda model, tokens_saved: tokens_saved / 1000.0,
+ )
+
+ for i in range(8):
+ tracker.record_compression_savings(
+ model="gpt-4o",
+ tokens_saved=10,
+ total_input_tokens=(i + 1) * 100,
+ total_input_cost_usd=(i + 1) * 0.1,
+ timestamp=f"2026-03-27T09:{i:02d}:00Z",
+ )
+
+ compact = tracker.history_response()
+ assert compact["history_summary"] == {
+ "mode": "compact",
+ "stored_points": 8,
+ "returned_points": 5,
+ "compacted": True,
+ }
+ assert len(compact["history"]) == 5
+ assert compact["history"][0]["timestamp"] == "2026-03-27T09:00:00Z"
+ assert compact["history"][-1]["timestamp"] == "2026-03-27T09:07:00Z"
+
+ full = tracker.history_response(history_mode="full")
+ assert full["history_summary"] == {
+ "mode": "full",
+ "stored_points": 8,
+ "returned_points": 8,
+ "compacted": False,
+ }
+ assert len(full["history"]) == 8
+
+ none = tracker.history_response(history_mode="none")
+ assert none["history"] == []
+ assert none["history_summary"] == {
+ "mode": "none",
+ "stored_points": 8,
+ "returned_points": 0,
+ "compacted": True,
+ }
+
+
+def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):
+ savings_path = tmp_path / "proxy_savings.json"
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
+ monkeypatch.setattr(
+ "headroom.proxy.server.CostTracker._get_cache_prices",
+ lambda self, model: (0.001, 0.0015, 0.002),
+ )
+
+ config = ProxyConfig(
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ log_requests=False,
+ )
+
+ with TestClient(create_app(config)) as client:
+ _record_request(client, model="gpt-4o", tokens_saved=40)
+
+ stats = client.get("/stats")
+ assert stats.status_code == 200
+ stats_data = stats.json()
+ assert "savings_history" in stats_data
+ assert "persistent_savings" in stats_data
+ assert all(len(point) == 2 for point in stats_data["savings_history"])
+ assert stats_data["persistent_savings"]["lifetime"]["tokens_saved"] == 40
+ assert stats_data["persistent_savings"]["storage_path"] == str(savings_path)
+
+ history = client.get("/stats-history")
+ assert history.status_code == 200
+ history_data = history.json()
+ assert history_data["schema_version"] == 2
+ assert history_data["storage_path"] == str(savings_path)
+ assert history_data["lifetime"]["tokens_saved"] == 40
+ assert history_data["lifetime"]["total_input_tokens"] == 120
+ assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)
+ assert history_data["display_session"]["requests"] == 1
+ assert history_data["display_session"]["tokens_saved"] == 40
+ assert history_data["display_session"]["total_input_tokens"] == 120
+ assert history_data["display_session"]["savings_percent"] == pytest.approx(25.0)
+ assert list(history_data["series"].keys()) == [
+ "hourly",
+ "daily",
+ "weekly",
+ "monthly",
+ ]
+ assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]
+ assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120
+ assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(
+ 0.24
+ )
+ assert history_data["history_summary"] == {
+ "mode": "compact",
+ "stored_points": 1,
+ "returned_points": 1,
+ "compacted": False,
+ }
+
+ assert stats_data["display_session"] == history_data["display_session"]
+ assert (
+ stats_data["persistent_savings"]["display_session"] == history_data["display_session"]
+ )
+
+ with TestClient(create_app(config)) as client:
+ history = client.get("/stats-history")
+ assert history.status_code == 200
+ assert history.json()["lifetime"]["tokens_saved"] == 40
+ assert history.json()["display_session"]["requests"] == 1
+
+ _record_request(client, model="gpt-4o", tokens_saved=15)
+
+ updated = client.get("/stats-history").json()
+ assert updated["lifetime"]["tokens_saved"] == 55
+ assert updated["lifetime"]["total_input_tokens"] == 240
+ assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
+ assert updated["lifetime"]["requests"] == 2
+ assert len(updated["history"]) == 2
+ assert updated["display_session"]["requests"] == 2
+ assert updated["display_session"]["tokens_saved"] == 55
+ assert updated["display_session"]["total_input_tokens"] == 240
+ assert updated["display_session"]["savings_percent"] == pytest.approx(18.64)
+ assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
+ assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
+
+ full = client.get("/stats-history?history_mode=full").json()
+ assert full["history_summary"]["mode"] == "full"
+ assert full["history_summary"]["stored_points"] == 2
+ assert full["history_summary"]["returned_points"] == 2
+
+ persisted = json.loads(savings_path.read_text())
+ assert persisted["lifetime"]["tokens_saved"] == 55
+ assert persisted["lifetime"]["total_input_tokens"] == 240
+ assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)
+ assert persisted["display_session"]["requests"] == 2
+
+
+def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):
+ savings_path = tmp_path / "proxy_savings.json"
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
+ monkeypatch.setattr(
+ "headroom.proxy.server.CostTracker._get_cache_prices",
+ lambda self, model: (0.001, 0.0015, 0.002),
+ )
+
+ config = ProxyConfig(
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ log_requests=False,
+ )
+
+ with TestClient(create_app(config)) as client:
+ _record_request(client, model="gpt-4o", tokens_saved=40)
+ _record_request(client, model="gpt-4o", tokens_saved=10)
+
+ response = client.get("/stats-history?format=csv&series=daily")
+ assert response.status_code == 200
+ assert response.headers["content-type"].startswith("text/csv")
+ assert (
+ 'attachment; filename="headroom-stats-history-daily.csv"'
+ == response.headers["content-disposition"]
+ )
+ lines = response.text.strip().splitlines()
+ assert lines[0] == (
+ "timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"
+ "compression_savings_usd,total_input_tokens_delta,total_input_tokens,"
+ "total_input_cost_usd_delta,total_input_cost_usd"
+ )
+ assert len(lines) >= 2
+ assert "total_tokens_saved" in lines[0]
+ assert "total_input_cost_usd" in lines[0]
+
+
+def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):
+ savings_path = tmp_path / "proxy_savings.json"
+ savings_path.write_text("{not valid json", encoding="utf-8")
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
+
+ config = ProxyConfig(
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ log_requests=False,
+ )
+
+ with TestClient(create_app(config)) as client:
+ response = client.get("/stats-history")
+ assert response.status_code == 200
+ data = response.json()
+ assert data["lifetime"]["tokens_saved"] == 0
+ assert data["history"] == []
+
+
+def test_dashboard_includes_history_toggle_and_endpoint(tmp_path, monkeypatch):
+ savings_path = tmp_path / "proxy_savings.json"
+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
+
+ config = ProxyConfig(
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ log_requests=False,
+ )
+
+ with TestClient(create_app(config)) as client:
+ response = client.get("/dashboard")
+ assert response.status_code == 200
+ html = response.text
+ assert "Session" in html
+ assert "Historical" in html
+ assert "fetch('/stats-history')" in html
+ assert "Export CSV" in html
+ assert "Weekly Savings" in html
+ assert "Monthly Savings" in html
diff --git a/tests/test_release_version.py b/tests/test_release_version.py
index 52bb0fcac..5485eeb20 100644
--- a/tests/test_release_version.py
+++ b/tests/test_release_version.py
@@ -14,6 +14,7 @@ from headroom.release_version import (
compute_release_version,
determine_bump_level,
find_latest_release_tag,
+ get_canonical_version,
list_release_commits,
normalize_release_tag,
parse_release_tag,
@@ -174,10 +175,11 @@ def test_release_version_script_runs_directly_without_importing_headroom_package
)
assert result.returncode == 0, result.stderr
+ canonical_version = get_canonical_version(ROOT)
assert output_path.read_text(encoding="utf-8").splitlines() == [
"version=0.6.0",
"npm_version=0.6.0",
- "canonical=0.5.25",
+ f"canonical={canonical_version}",
"height=0",
"bump=manual",
"previous_tag=",