Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: resolve rebased ci regressions
JerrettDavis470bb6cfb9e2
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.jsonindex 734d6b89e..afc9d8b26 100644--- a/.claude-plugin/marketplace.json+++ b/.claude-plugin/marketplace.json@@ -1,30 +1,30 @@-{- "name": "headroom-marketplace",- "owner": {- "name": "Headroom Contributors"- },- "metadata": {- "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.9.2"- },- "plugins": [- {- "name": "headroom",- "source": "./plugins/headroom-agent-hooks",- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.9.2",- "author": {- "name": "Headroom Contributors",- "url": "https://github.com/chopratejas/headroom"- },- "homepage": "https://github.com/chopratejas/headroom",- "repository": "https://github.com/chopratejas/headroom",- "keywords": [- "headroom",- "hooks",- "claude-code",- "copilot-cli"- ]- }- ]-}+{+ "name": "headroom-marketplace",+ "owner": {+ "name": "Headroom Contributors"+ },+ "metadata": {+ "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",+ "version": "0.10.0"+ },+ "plugins": [+ {+ "name": "headroom",+ "source": "./plugins/headroom-agent-hooks",+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",+ "version": "0.10.0",+ "author": {+ "name": "Headroom Contributors",+ "url": "https://github.com/chopratejas/headroom"+ },+ "homepage": "https://github.com/chopratejas/headroom",+ "repository": "https://github.com/chopratejas/headroom",+ "keywords": [+ "headroom",+ "hooks",+ "claude-code",+ "copilot-cli"+ ]+ }+ ]+}diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.jsonindex 734d6b89e..afc9d8b26 100644--- a/.github/plugin/marketplace.json+++ b/.github/plugin/marketplace.json@@ -1,30 +1,30 @@-{- "name": "headroom-marketplace",- "owner": {- "name": "Headroom Contributors"- },- "metadata": {- "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.9.2"- },- "plugins": [- {- "name": "headroom",- "source": "./plugins/headroom-agent-hooks",- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.9.2",- "author": {- "name": "Headroom Contributors",- "url": "https://github.com/chopratejas/headroom"- },- "homepage": "https://github.com/chopratejas/headroom",- "repository": "https://github.com/chopratejas/headroom",- "keywords": [- "headroom",- "hooks",- "claude-code",- "copilot-cli"- ]- }- ]-}+{+ "name": "headroom-marketplace",+ "owner": {+ "name": "Headroom Contributors"+ },+ "metadata": {+ "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",+ "version": "0.10.0"+ },+ "plugins": [+ {+ "name": "headroom",+ "source": "./plugins/headroom-agent-hooks",+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",+ "version": "0.10.0",+ "author": {+ "name": "Headroom Contributors",+ "url": "https://github.com/chopratejas/headroom"+ },+ "homepage": "https://github.com/chopratejas/headroom",+ "repository": "https://github.com/chopratejas/headroom",+ "keywords": [+ "headroom",+ "hooks",+ "claude-code",+ "copilot-cli"+ ]+ }+ ]+}diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 49938c1ae..41a422842 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -1,2746 +1,2746 @@-"""OpenAI handler mixin for HeadroomProxy.--Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.-"""--from __future__ import annotations--import asyncio-import base64-import contextlib-import copy-import json-import logging-import os-import time-import uuid-from datetime import datetime-from typing import TYPE_CHECKING, Any--from headroom.proxy.helpers import jitter_delay_ms-from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log-from headroom.proxy.ws_session_registry import (- TerminationCause,- WebSocketSessionRegistry,- WSSessionHandle,-)--if TYPE_CHECKING:- from fastapi import Request, WebSocket- from fastapi.responses import JSONResponse, Response, StreamingResponse--import httpx--from headroom.copilot_auth import apply_copilot_api_auth-from headroom.pipeline import PipelineStage, summarize_routing_markers--logger = logging.getLogger("headroom.proxy")---# Interactive Responses turns are latency-sensitive. Fail open quickly rather-# than holding the session hostage on memory lookup.-RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0--# Cap the wait for the first client frame after the WS handshake completes.-# A zombie or malicious client that accepts the upgrade but never sends the-# first response.create frame would otherwise hold a slot indefinitely and-# starve the session registry. 60 s is generous for real clients (Codex-# typically sends the first frame within a few hundred milliseconds of the-# accept) but short enough to bound the damage from a hung peer.-WS_FIRST_FRAME_TIMEOUT_SECONDS = 60.0---def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:- """Best-effort decode of an OpenAI OAuth bearer token payload.-- OpenClaw's Codex OAuth flow may forward only the bearer token after the- provider base URL is overridden. In that case the explicit- ``ChatGPT-Account-ID`` header can be missing even though the JWT still- carries the account id we need to route to the ChatGPT Codex backend.- """- auth = headers.get("authorization") or headers.get("Authorization")- if not auth:- return None-- scheme, _, token = auth.partition(" ")- if scheme.lower() != "bearer" or token.count(".") < 2:- return None-- payload = token.split(".", 2)[1]- payload += "=" * (-len(payload) % 4)- # Intentionally no signature verification here: this is only a best-effort- # routing hint extractor. Upstream still performs the actual auth/authz checks.- try:- decoded = base64.urlsafe_b64decode(payload.encode("ascii"))- data = json.loads(decoded.decode("utf-8"))- except (ValueError, UnicodeDecodeError):- return None-- return data if isinstance(data, dict) else None---def _resolve_codex_routing_headers(headers: dict[str, str]) -> tuple[dict[str, str], bool]:- """Resolve ChatGPT Codex routing hints from explicit headers or OAuth JWT."""- resolved = dict(headers)- lower_lookup = {k.lower(): k for k in resolved}-- if "chatgpt-account-id" in lower_lookup:- return resolved, True-- payload = _decode_openai_bearer_payload(resolved)- auth_claims = payload.get("https://api.openai.com/auth") if isinstance(payload, dict) else None- account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None- if isinstance(account_id, str) and account_id.strip():- resolved["ChatGPT-Account-ID"] = account_id.strip()- return resolved, True-- return resolved, False---class OpenAIHandlerMixin:- """Mixin providing OpenAI API handler methods for HeadroomProxy."""-- @staticmethod- def _strict_previous_turn_frozen_count(- messages: list[dict[str, Any]],- base_frozen_count: int,- ) -> int:- """Freeze all prior turns in cache mode; only final user turn is mutable."""- if not messages:- return base_frozen_count- final_idx = len(messages) - 1- if messages[final_idx].get("role") == "user":- return max(base_frozen_count, final_idx)- return len(messages)-- @staticmethod- def _restore_frozen_prefix(- original_messages: list[dict[str, Any]],- candidate_messages: list[dict[str, Any]],- *,- frozen_message_count: int,- ) -> tuple[list[dict[str, Any]], int]:- """Force frozen prefix bytes to match original request exactly."""- if frozen_message_count <= 0 or not original_messages:- return candidate_messages, 0-- frozen = min(frozen_message_count, len(original_messages))- restored = list(candidate_messages)-- if len(restored) < frozen:- return list(original_messages[:frozen]) + restored, frozen-- changed = 0- for idx in range(frozen):- if restored[idx] != original_messages[idx]:- restored[idx] = original_messages[idx]- changed += 1- return restored, changed-- async def handle_openai_chat(- self,- request: Request,- ) -> Response | StreamingResponse:- """Handle OpenAI /v1/chat/completions endpoint."""- if not hasattr(self, "pipeline_extensions"):- from headroom.pipeline import PipelineExtensionManager-- self.pipeline_extensions = PipelineExtensionManager(discover=False)-- from fastapi import HTTPException- from fastapi.responses import JSONResponse, Response-- from headroom.ccr import CCRToolInjector- from headroom.proxy.helpers import (- COMPRESSION_TIMEOUT_SECONDS,- MAX_MESSAGE_ARRAY_LENGTH,- MAX_REQUEST_BODY_SIZE,- _read_request_json,- )- from headroom.proxy.modes import is_cache_mode, is_token_mode- from headroom.tokenizers import get_tokenizer- from headroom.utils import extract_user_query-- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "error": {- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "type": "invalid_request_error",- "code": "request_too_large",- }- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "type": "invalid_request_error",- "code": "invalid_json",- }- },- )- model = body.get("model", "unknown")- messages = body.get("messages", [])- original_client_messages = copy.deepcopy(messages)- input_event = self.pipeline_extensions.emit(- PipelineStage.INPUT_RECEIVED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=messages,- tools=body.get("tools"),- metadata={"path": "/v1/chat/completions", "stream": body.get("stream", False)},- )- if input_event.messages is not None:- messages = input_event.messages- original_client_messages = copy.deepcopy(messages)- if input_event.tools is not None:- body["tools"] = input_event.tools-- # Validate message array size- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Message array too large ({len(messages)} messages). "- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",- "type": "invalid_request_error",- "code": "invalid_request",- }- },- )-- stream = body.get("stream", False)-- # Bypass: skip ALL compression for explicit opt-out- _bypass = (- request.headers.get("x-headroom-bypass", "").lower() == "true"- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"- )- if _bypass:- logger.info(f"[{request_id}] Bypass: skipping compression (header)")-- # Image compression: tile alignment + ML-based technique routing- if self.config.image_optimize and messages and not _bypass:- from headroom.proxy.helpers import _get_image_compressor-- compressor = _get_image_compressor()- if compressor and compressor.has_images(messages):- messages = compressor.compress(messages, provider="openai")- if compressor.last_result:- logger.info(- f"[{request_id}] Image: {compressor.last_result.technique.value} "- f"({compressor.last_result.savings_percent:.0f}% saved, "- f"{compressor.last_result.original_tokens} → "- f"{compressor.last_result.compressed_tokens} tokens)"- )-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- # Strip accept-encoding so httpx negotiates its own encoding.- # Cloudflare Workers forward "br, zstd" which OpenAI may honor;- # if httpx lacks brotli support the response body is undecipherable → 502.- headers.pop("accept-encoding", None)- tags = self._extract_tags(headers)-- # Memory: Get user ID when memory is enabled- memory_user_id: str | None = None- if self.memory_handler:- memory_user_id = headers.get(- "x-headroom-user-id",- os.environ.get("USER", os.environ.get("USERNAME", "default")),- )-- # Rate limiting- if self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited(provider="openai")- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",- )-- # Check cache- if self.cache and not stream:- cached = await self.cache.get(messages, model)- if cached:- self.pipeline_extensions.emit(- PipelineStage.INPUT_CACHED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=messages,- metadata={"cache_hit": True, "path": "/v1/chat/completions"},- )- await self.metrics.record_request(- provider="openai",- model=model,- input_tokens=0,- output_tokens=0,- tokens_saved=0, # Savings already counted when response was cached- latency_ms=(time.time() - start_time) * 1000,- cached=True,- )-- # Remove compression headers from cached response- response_headers = dict(cached.response_headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(content=cached.response_body, headers=response_headers)-- # Token counting- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Hook: pre_compress- _hook_biases = None- if self.config.hooks:- from headroom.hooks import CompressContext-- _hook_ctx = CompressContext(model=model, provider="openai")- try:- messages = self.config.hooks.pre_compress(messages, _hook_ctx)- _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)- except Exception as e:- logger.debug(f"[{request_id}] Hook error: {e}")-- # Optimization- transforms_applied = []- pipeline_timing: dict[str, float] = {}- waste_signals_dict: dict[str, int] | None = None- optimized_messages = messages- optimized_tokens = original_tokens-- # Get prefix cache tracker for this session- openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)- openai_prefix_tracker = self.session_tracker_store.get_or_create(- openai_session_id, "openai"- )- openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()- if is_cache_mode(self.config.mode):- openai_frozen_count = self._strict_previous_turn_frozen_count(- original_client_messages,- openai_frozen_count,- )-- _compression_failed = False- original_messages = messages # Preserve for 400-retry fallback- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True- if self.config.optimize and messages and not _bypass and _license_ok:- try:- context_limit = self.openai_provider.get_context_limit(model)-- if is_token_mode(self.config.mode):- comp_cache = self._get_compression_cache(openai_session_id)-- # Zone 1: Swap cached compressed versions- working_messages = comp_cache.apply_cached(messages)-- # Re-freeze boundary- openai_frozen_count = comp_cache.compute_frozen_count(messages)-- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=working_messages,- model=model,- model_limit=context_limit,- context=extract_user_query(working_messages),- frozen_message_count=openai_frozen_count,- biases=_hook_biases,- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != working_messages:- comp_cache.update_from_result(messages, result.messages)-- # Always use pipeline result in token mode- optimized_messages = result.messages- transforms_applied = result.transforms_applied- pipeline_timing = result.timing- # Keep original_tokens as the REAL original (pre-Zone-1-swap)- # so tokens_saved captures both Zone 1 + Zone 2 savings.- optimized_tokens = result.tokens_after- else:- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- frozen_message_count=openai_frozen_count,- biases=_hook_biases,- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- pipeline_timing = result.timing- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after-- if result.waste_signals:- waste_signals_dict = result.waste_signals.to_dict()- except Exception as e:- logger.warning(f"Optimization failed: {e}")- # Flag compression failure for observability- _compression_failed = True-- # Guard: if "optimization" inflated tokens, revert to originals- if optimized_tokens > original_tokens:- logger.warning(- f"[{request_id}] Optimization inflated tokens "- f"({original_tokens} -> {optimized_tokens}), reverting to original messages"- )- optimized_messages = original_messages- optimized_tokens = original_tokens- transforms_applied = []-- tokens_saved = original_tokens - optimized_tokens- optimization_latency = (time.time() - start_time) * 1000-- routing_markers = summarize_routing_markers(transforms_applied)- if routing_markers:- routed_event = self.pipeline_extensions.emit(- PipelineStage.INPUT_ROUTED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=optimized_messages,- metadata={- "routing_markers": routing_markers,- "transforms_applied": transforms_applied,- },- )- if routed_event.messages is not None:- optimized_messages = routed_event.messages- optimized_tokens = tokenizer.count_messages(optimized_messages)- tokens_saved = original_tokens - optimized_tokens-- compressed_event = self.pipeline_extensions.emit(- PipelineStage.INPUT_COMPRESSED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=optimized_messages,- metadata={- "tokens_before": original_tokens,- "tokens_after": optimized_tokens,- "transforms_applied": transforms_applied,- },- )- if compressed_event.messages is not None:- optimized_messages = compressed_event.messages- optimized_tokens = tokenizer.count_messages(optimized_messages)- tokens_saved = original_tokens - optimized_tokens-- # Hook: post_compress- if self.config.hooks and tokens_saved > 0:- from headroom.hooks import CompressEvent-- try:- self.config.hooks.post_compress(- CompressEvent(- tokens_before=original_tokens,- tokens_after=optimized_tokens,- tokens_saved=tokens_saved,- compression_ratio=tokens_saved / original_tokens- if original_tokens > 0- else 0,- transforms_applied=transforms_applied,- model=model,- provider="openai",- )- )- except Exception as e:- logger.debug(f"[{request_id}] post_compress hook error: {e}")-- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = body.get("tools")- _original_tools = tools # Preserve for diagnostic / future retry- if (- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions- ) and not _bypass:- injector = CCRToolInjector(- provider="openai",- inject_tool=self.config.ccr_inject_tool,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- optimized_messages, tools, was_injected = injector.process_request(- optimized_messages, tools- )-- if injector.has_compressed_content:- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"- )- else:- logger.debug(- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"- )-- # Query Echo: disabled — hurts prefix caching in long conversations.- if is_cache_mode(self.config.mode):- optimized_messages, restored_count = self._restore_frozen_prefix(- original_client_messages,- optimized_messages,- frozen_message_count=openai_frozen_count,- )- if restored_count > 0:- logger.warning(- f"[{request_id}] Restored {restored_count} frozen prefix message(s) "- "to preserve cache stability (openai)"- )-- # Memory: inject context and tools for OpenAI requests- memory_context_injected = False- memory_tools_injected = False- if self.memory_handler and memory_user_id:- try:- # Inject memory context (search similar memories, add as system message)- if self.memory_handler.config.inject_context:- memory_context = await self.memory_handler.search_and_format_context(- memory_user_id, optimized_messages- )- if memory_context:- # Prepend as system message for OpenAI format- optimized_messages = [- {"role": "system", "content": memory_context},- *optimized_messages,- ]- memory_context_injected = True- logger.info(- f"[{request_id}] Memory: Injected {len(memory_context)} chars "- f"of context for user {memory_user_id}"- )-- # Inject memory tools- if self.memory_handler.config.inject_tools:- tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "openai")- if mem_tools_injected:- memory_tools_injected = True- logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")- except Exception as e:- logger.warning(f"[{request_id}] Memory injection failed: {e}")-- if memory_context_injected or memory_tools_injected:- remembered_event = self.pipeline_extensions.emit(- PipelineStage.INPUT_REMEMBERED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=optimized_messages,- tools=tools,- metadata={- "memory_context_injected": memory_context_injected,- "memory_tools_injected": memory_tools_injected,- },- )- if remembered_event.messages is not None:- optimized_messages = remembered_event.messages- if remembered_event.tools is not None:- tools = remembered_event.tools-- body["messages"] = optimized_messages- if tools is not None:- body["tools"] = tools-- presend_event = self.pipeline_extensions.emit(- PipelineStage.PRE_SEND,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=optimized_messages,- tools=tools,- headers=headers,- metadata={"path": "/v1/chat/completions", "stream": stream},- )- if presend_event.messages is not None:- optimized_messages = presend_event.messages- body["messages"] = optimized_messages- if presend_event.tools is not None:- tools = presend_event.tools- body["tools"] = tools- if presend_event.headers is not None:- headers = presend_event.headers- optimized_tokens = tokenizer.count_messages(body["messages"])- tokens_saved = original_tokens - optimized_tokens-- # Route through LiteLLM/any-llm backend if configured- if self.anthropic_backend is not None:- try:- if stream:- self.pipeline_extensions.emit(- PipelineStage.POST_SEND,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=body["messages"],- tools=tools,- metadata={"path": "/v1/chat/completions", "stream": True},- )- # Streaming: use stream_openai_message() → SSE events- return await self._stream_openai_via_backend(- body,- headers,- model,- request_id,- start_time,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- pipeline_timing=pipeline_timing,- )- else:- # Non-streaming: use send_openai_message() → JSON- backend_response = await self.anthropic_backend.send_openai_message(- body, headers- )- self.pipeline_extensions.emit(- PipelineStage.POST_SEND,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=body["messages"],- tools=tools,- response=backend_response.body,- metadata={- "path": "/v1/chat/completions",- "stream": False,- "status_code": backend_response.status_code,- },- )- self.pipeline_extensions.emit(- PipelineStage.RESPONSE_RECEIVED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- response=backend_response.body,- metadata={- "path": "/v1/chat/completions",- "stream": False,- "status_code": backend_response.status_code,- },- )-- if backend_response.error:- return JSONResponse(- status_code=backend_response.status_code,- content=backend_response.body,- )-- # Track metrics- total_latency = (time.time() - start_time) * 1000- usage = backend_response.body.get("usage", {})- output_tokens = usage.get("completion_tokens", 0)- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)-- await self.metrics.record_request(- provider=self.anthropic_backend.name,- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- cached=False,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- )-- if tokens_saved > 0:- logger.info(- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"- )-- return JSONResponse(- status_code=backend_response.status_code,- content=backend_response.body,- )- except Exception as e:- logger.error(f"[{request_id}] Backend error: {e}")- return JSONResponse(- status_code=500,- content={- "error": {- "message": str(e),- "type": "api_error",- "code": "backend_error",- }- },- )-- # Direct OpenAI API (no backend configured)- url = f"{self.OPENAI_API_URL}/v1/chat/completions"-- try:- if stream:- # Inject stream_options to get usage stats in streaming response- # This allows accurate token counting instead of byte-based estimation- if "stream_options" not in body:- body["stream_options"] = {"include_usage": True}- elif isinstance(body.get("stream_options"), dict):- body["stream_options"]["include_usage"] = True-- self.pipeline_extensions.emit(- PipelineStage.POST_SEND,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=body["messages"],- tools=tools,- metadata={"path": "/v1/chat/completions", "stream": True},- )- return await self._stream_response(- url,- headers,- body,- "openai",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- pipeline_timing=pipeline_timing,- prefix_tracker=openai_prefix_tracker,- )- else:- headers = await apply_copilot_api_auth(headers, url=url)- response = await self._retry_request("POST", url, headers, body)- self.pipeline_extensions.emit(- PipelineStage.POST_SEND,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- messages=body["messages"],- tools=tools,- response=response,- metadata={- "path": "/v1/chat/completions",- "stream": False,- "status_code": response.status_code,- },- )- self.pipeline_extensions.emit(- PipelineStage.RESPONSE_RECEIVED,- operation="proxy.request",- request_id=request_id,- provider="openai",- model=model,- response=response,- metadata={- "path": "/v1/chat/completions",- "stream": False,- "status_code": response.status_code,- },- )-- # Full diagnostic dump on upstream errors (OpenAI handler)- if response.status_code >= 400:- try:- err_body = response.json()- err_msg = err_body.get("error", {}).get("message", "")- err_type = err_body.get("error", {}).get("type", "")- except Exception:- err_body = {"raw": response.text[:2000]}- err_msg = str(response.text[:500])- err_type = "parse_error"-- logger.warning(- f"[{request_id}] UPSTREAM_ERROR "- f"status={response.status_code} "- f"error_type={err_type} "- f"error_msg={err_msg!r} "- f"model={model} "- f"compressed={'yes' if transforms_applied else 'no'} "- f"transforms={transforms_applied} "- f"original_tokens={original_tokens} "- f"optimized_tokens={optimized_tokens} "- f"message_count={len(body.get('messages', []))} "- f"stream={stream}"- )-- try:- from headroom import paths as _hr_paths-- debug_dir = _hr_paths.debug_400_dir()- debug_dir.mkdir(parents=True, exist_ok=True)- ts = datetime.now().strftime("%Y%m%d_%H%M%S")- debug_file = debug_dir / f"{ts}_{request_id}.json"-- safe_headers = {}- for k, v in headers.items():- if k.lower() in ("x-api-key", "authorization"):- safe_headers[k] = v[:12] + "..." if v else ""- else:- safe_headers[k] = v-- debug_payload = {- "request_id": request_id,- "timestamp": datetime.now().isoformat(),- "status_code": response.status_code,- "error_response": err_body,- "model": model,- "stream": stream,- "headers": safe_headers,- "compression": {- "was_compressed": bool(transforms_applied),- "transforms": transforms_applied,- "original_tokens": original_tokens,- "optimized_tokens": optimized_tokens,- "tokens_saved": tokens_saved,- "compression_failed": _compression_failed,- },- "tools_sent": body.get("tools"),- "tool_count": len(body.get("tools") or []),- "original_tool_count": len(_original_tools or []),- "messages_sent": body.get("messages"),- "message_count": len(body.get("messages", [])),- "original_messages": (- original_messages- if original_messages is not body.get("messages")- else "__same_as_sent__"- ),- "original_message_count": len(original_messages),- "system_prompt": body.get("system"),- }-- with open(debug_file, "w") as f:- json.dump(debug_payload, f, indent=2, default=str)-- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")- except Exception as dump_err:- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")-- total_latency = (time.time() - start_time) * 1000-- total_input_tokens = optimized_tokens # fallback- output_tokens = 0- cache_read_tokens = 0- resp_json = None- try:- resp_json = response.json()- usage = resp_json.get("usage", {})- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)- output_tokens = usage.get("completion_tokens", 0)- # OpenAI returns cached_tokens in prompt_tokens_details- # These are charged at 50% of the input price- prompt_details = usage.get("prompt_tokens_details") or {}- cache_read_tokens = prompt_details.get("cached_tokens", 0)- except (KeyError, TypeError, AttributeError) as e:- logger.debug(- f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"- )-- # Update prefix cache tracker for next turn- openai_prefix_tracker.update_from_response(- cache_read_tokens=cache_read_tokens,- cache_write_tokens=0, # OpenAI doesn't report write tokens- messages=optimized_messages,- )-- # OpenAI has no write penalty — uncached = total - cached- uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)-- if self.cost_tracker:- self.cost_tracker.record_tokens(- model,- tokens_saved,- optimized_tokens,- cache_read_tokens=cache_read_tokens,- uncached_tokens=uncached_input_tokens,- )-- # Memory: handle memory tool calls in OpenAI Chat Completions response.- # After executing tools, send a continuation request so the model- # can produce a final user-facing response (not just tool_calls).- if (- self.memory_handler- and memory_user_id- and resp_json- and response.status_code == 200- and self.memory_handler.has_memory_tool_calls(resp_json, "openai")- ):- try:- tool_results = await self.memory_handler.handle_memory_tool_calls(- resp_json, memory_user_id, "openai"- )- if tool_results:- # Build continuation: original messages + assistant tool_calls + tool results- assistant_msg = resp_json.get("choices", [{}])[0].get("message", {})- continuation_messages = list(optimized_messages)- continuation_messages.append(assistant_msg)- continuation_messages.extend(tool_results)-- continuation_body = {- **body,- "messages": continuation_messages,- }-- cont_response = await self._retry_request(- "POST", url, headers, continuation_body- )- if cont_response.status_code == 200:- resp_json = cont_response.json()- response = cont_response-- logger.info(- f"[{request_id}] Memory: Handled {len(tool_results)} "- f"tool call(s) with continuation for user {memory_user_id}"- )- except Exception as e:- logger.warning(f"[{request_id}] Memory tool handling failed: {e}")-- # Cache- if self.cache and response.status_code == 200:- await self.cache.set(- messages,- model,- response.content,- dict(response.headers),- tokens_saved,- )-- # Capture Codex rate-limit window data from response headers- from headroom.subscription.codex_rate_limits import (- get_codex_rate_limit_state,- )-- get_codex_rate_limit_state().update_from_headers(dict(response.headers))-- await self.metrics.record_request(- provider="openai",- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- waste_signals=waste_signals_dict,- cache_read_tokens=cache_read_tokens,- uncached_input_tokens=uncached_input_tokens,- )-- if tokens_saved > 0:- logger.info(- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "- f"(saved {tokens_saved:,} tokens)"- )-- # Remove compression headers since httpx already decompressed the response- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression-- # Inject Headroom compression metrics (for SaaS metering)- response_headers["x-headroom-tokens-before"] = str(original_tokens)- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)- response_headers["x-headroom-model"] = model- if transforms_applied:- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)- if cache_read_tokens > 0:- response_headers["x-headroom-cached"] = "true"- if _compression_failed:- response_headers["x-headroom-compression-failed"] = "true"-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )- except Exception as e:- await self.metrics.record_failed(provider="openai")- # Log full error details internally for debugging- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")- # Return sanitized error message to client (don't expose internal details)- return JSONResponse(- status_code=502,- content={- "error": {- "message": "An error occurred while processing your request. Please try again.",- "type": "server_error",- "code": "proxy_error",- }- },- )-- async def handle_openai_responses(- self,- request: Request,- ) -> Response | StreamingResponse:- """Handle OpenAI /v1/responses endpoint (new Responses API).-- The Responses API differs from /v1/chat/completions:- - Input: `input` (string or array) instead of `messages`- - System: `instructions` instead of system message- - Output: `output[]` array instead of `choices[].message`- - State: `previous_response_id` for multi-turn- - Built-in tools: web_search, file_search, code_interpreter- """- from fastapi import HTTPException- from fastapi.responses import JSONResponse, Response-- from headroom.proxy.helpers import (- COMPRESSION_TIMEOUT_SECONDS,- MAX_REQUEST_BODY_SIZE,- _read_request_json,- )- from headroom.proxy.responses_converter import (- messages_to_responses_items,- responses_items_to_messages,- )- from headroom.tokenizers import get_tokenizer- from headroom.utils import extract_user_query-- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "error": {- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "type": "invalid_request_error",- "code": "request_too_large",- }- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "type": "invalid_request_error",- "code": "invalid_json",- }- },- )-- model = body.get("model", "unknown")- stream = body.get("stream", False)-- # Convert Responses API input to messages format for optimization.- # The Responses API uses a different item model (function_call,- # function_call_output, reasoning as top-level items) — we convert to- # Chat Completions messages for the pipeline, then convert back.-- input_data = body.get("input", "")- instructions = body.get("instructions")- previous_response_id = body.get("previous_response_id")-- messages: list[dict[str, Any]] = []- original_items: list[dict[str, Any]] | None = None- preserved_indices: list[int] = []-- if instructions:- messages.append({"role": "system", "content": instructions})-- if isinstance(input_data, str):- messages.append({"role": "user", "content": input_data})- elif isinstance(input_data, list):- original_items = input_data- converted, preserved_indices = responses_items_to_messages(input_data)- messages.extend(converted)-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- # Strip accept-encoding so httpx negotiates its own encoding.- # Cloudflare Workers forward "br, zstd" which OpenAI may honor;- # if httpx lacks brotli support the response body is undecipherable → 502.- headers.pop("accept-encoding", None)- tags = self._extract_tags(headers)-- # Memory: Get user ID when memory is enabled- memory_user_id: str | None = None- if self.memory_handler:- memory_user_id = headers.get(- "x-headroom-user-id",- os.environ.get("USER", os.environ.get("USERNAME", "default")),- )-- # Rate limiting- if self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited(provider="openai")- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",- )-- # Token counting on converted messages- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Optimize: convert items → compress → convert back- tokens_saved = 0- transforms_applied: list[str] = []- optimized_messages = messages- optimized_tokens = original_tokens-- _bypass = (- request.headers.get("x-headroom-bypass", "").lower() == "true"- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"- )- _should_compress = (- self.config.optimize- and original_items is not None- and not previous_response_id- and not _bypass- and len(messages) > 1- )- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True-- if _should_compress and _license_ok:- try:- context_limit = self.openai_provider.get_context_limit(model)- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after- except Exception as e:- logger.warning(f"[{request_id}] Responses API optimization failed: {e}")-- # Guard: if "optimization" inflated tokens, revert to originals- if optimized_tokens > original_tokens:- logger.warning(- f"[{request_id}] Optimization inflated tokens "- f"({original_tokens} -> {optimized_tokens}), reverting to original messages"- )- optimized_messages = messages- optimized_tokens = original_tokens- transforms_applied = []-- tokens_saved = original_tokens - optimized_tokens- optimization_latency = (time.time() - start_time) * 1000-- # Convert compressed messages back to Responses API items- if optimized_messages is not messages and original_items is not None:- opt_msgs = optimized_messages- # Strip system message (instructions) — it's separate in Responses API- if instructions and opt_msgs and opt_msgs[0].get("role") == "system":- body["instructions"] = opt_msgs[0]["content"]- opt_msgs = opt_msgs[1:]-- body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)-- # Memory: inject context and tools for Responses API requests- if self.memory_handler and memory_user_id:- try:- # Inject memory context into instructions (Responses API system prompt)- if self.memory_handler.config.inject_context:- try:- memory_context = await asyncio.wait_for(- self.memory_handler.search_and_format_context(- memory_user_id, optimized_messages- ),- timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,- )- except asyncio.TimeoutError:- memory_context = None- logger.info(- f"[{request_id}] Memory context lookup exceeded "- f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"- )- if memory_context:- existing_instructions = body.get("instructions") or ""- if existing_instructions:- body["instructions"] = f"{existing_instructions}\n\n{memory_context}"- else:- body["instructions"] = memory_context- logger.info(- f"[{request_id}] Memory: Injected {len(memory_context)} chars "- f"of context into instructions for user {memory_user_id}"- )-- # Inject memory tools (Responses API format)- if self.memory_handler.config.inject_tools:- resp_tools = body.get("tools") or []- resp_tools, mem_tools_injected = self.memory_handler.inject_tools(- resp_tools, "openai"- )- if mem_tools_injected:- # Convert Chat Completions format to Responses API format- converted_tools = []- for t in resp_tools:- if t.get("type") == "function" and "function" in t:- fn = t["function"]- converted_tools.append(- {- "type": "function",- "name": fn.get("name"),- "description": fn.get("description", ""),- "parameters": fn.get("parameters", {}),- }- )- else:- converted_tools.append(t)- body["tools"] = converted_tools- logger.info(- f"[{request_id}] Memory: Injected memory tools (openai/responses)"- )- except Exception as e:- logger.warning(f"[{request_id}] Memory injection failed (responses): {e}")-- # /v1/responses is OpenAI-specific (Codex) — always routes direct.- # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.- if self.anthropic_backend is not None:- logger.debug(- f"[{request_id}] /v1/responses always routes to OpenAI direct "- f"(backend '{self.anthropic_backend.name}' not used for Responses API)"- )-- headers, is_chatgpt_auth = _resolve_codex_routing_headers(headers)-- # Route to correct endpoint based on auth mode.- # ChatGPT session auth (codex login) uses chatgpt.com, not api.openai.com.- if is_chatgpt_auth:- url = "https://chatgpt.com/backend-api/codex/responses"- else:- url = f"{self.OPENAI_API_URL}/v1/responses"-- try:- if stream:- # Streaming for Responses API uses semantic events- return await self._stream_response(- url,- headers,- body,- "openai",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- memory_user_id=memory_user_id,- )- else:- headers = await apply_copilot_api_auth(headers, url=url)- response = await self._retry_request("POST", url, headers, body)- total_latency = (time.time() - start_time) * 1000-- total_input_tokens = original_tokens # fallback- output_tokens = 0- try:- resp_json = response.json()- usage = resp_json.get("usage", {})- total_input_tokens = usage.get("input_tokens", original_tokens)- output_tokens = usage.get("output_tokens", 0)- except (KeyError, TypeError, AttributeError) as e:- logger.debug(- f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"- )-- # Memory: handle memory tool calls in Responses API response- if (- self.memory_handler- and memory_user_id- and resp_json- and response.status_code == 200- and self.memory_handler.has_memory_tool_calls(resp_json, "openai")- ):- try:- # Extract function_call items from output- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES-- output_items = resp_json.get("output", [])- memory_fc_items = [- item- for item in output_items- if isinstance(item, dict)- and item.get("type") == "function_call"- and item.get("name") in MEMORY_TOOL_NAMES- ]-- # Execute memory tool calls- tool_outputs: list[dict[str, Any]] = []- for fc in memory_fc_items:- call_id = fc.get("call_id", fc.get("id", ""))- name = fc.get("name", "")- args_str = fc.get("arguments", "{}")- try:- args = json.loads(args_str)- except json.JSONDecodeError:- args = {}-- await self.memory_handler._ensure_initialized()- if self.memory_handler._backend:- result = await self.memory_handler._execute_memory_tool(- name, args, memory_user_id, "openai"- )- else:- result = json.dumps({"error": "Memory backend not initialized"})-- tool_outputs.append(- {- "type": "function_call_output",- "call_id": call_id,- "output": result,- }- )-- if tool_outputs:- # Make continuation request with tool results- response_id = resp_json.get("id")- continuation_body = {- "model": model,- "input": tool_outputs,- }- if response_id:- continuation_body["previous_response_id"] = response_id- existing_tools = body.get("tools")- if existing_tools:- continuation_body["tools"] = existing_tools-- cont_response = await self._retry_request(- "POST", url, headers, continuation_body- )- resp_json = cont_response.json()- response = cont_response- logger.info(- f"[{request_id}] Memory: Handled {len(tool_outputs)} "- f"tool call(s) with continuation for user {memory_user_id} (responses)"- )- except Exception as e:- logger.warning(- f"[{request_id}] Memory tool handling failed (responses): {e}"- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)-- await self.metrics.record_request(- provider="openai",- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- )-- logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")-- # Capture Codex rate-limit window data from response headers- from headroom.subscription.codex_rate_limits import (- get_codex_rate_limit_state,- )-- get_codex_rate_limit_state().update_from_headers(dict(response.headers))-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )- except Exception as e:- await self.metrics.record_failed(provider="openai")- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")- return JSONResponse(- status_code=502,- content={- "error": {- "message": "An error occurred while processing your request. Please try again.",- "type": "server_error",- "code": "proxy_error",- }- },- )-- async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:- """WebSocket proxy for /v1/responses (Codex gpt-5.4+).-- Newer Codex versions use WebSocket instead of HTTP POST for the- Responses API. This handler:- 1. Accepts the client WebSocket- 2. Receives the first message (``response.create`` request)- 3. Compresses the ``input`` array using the existing pipeline- 4. Opens an upstream WebSocket to OpenAI- 5. Sends the compressed request upstream- 6. Relays all subsequent messages bidirectionally- """- from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS- from headroom.tokenizers import get_tokenizer- from headroom.utils import extract_user_query-- try:- import websockets- except ImportError:- await websocket.accept()- await websocket.close(- code=1011,- reason="websockets package not installed. pip install websockets",- )- return-- request_id = await self._next_request_id()- session_id = uuid.uuid4().hex-- # Stage-timer — captures per-stage durations for the structured- # log emitted on session close. Unit 2 instrumentation.- stage_timer = StageTimer()- session_started_at = time.perf_counter()-- # Unit 3: initialize registry variables *before* accept so the- # outermost ``finally`` can rely on them existing even if- # registration itself fails for some reason.- ws_sessions: WebSocketSessionRegistry | None = getattr(self, "ws_sessions", None)- session_handle: WSSessionHandle | None = None- termination_cause: TerminationCause = "unknown"-- # Forward client headers to upstream, adding required OpenAI-Beta header- ws_headers = dict(websocket.headers)-- # Extract subprotocol from client — this is an application-level negotiation- # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).- # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.- client_subprotocols: list[str] = []- raw_protocol = ws_headers.get("sec-websocket-protocol", "")- if raw_protocol:- client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]-- # Accept client connection with the requested subprotocol- async with stage_timer.measure("accept"):- if client_subprotocols:- await websocket.accept(subprotocol=client_subprotocols[0])- else:- await websocket.accept()-- # --- Unit 3: register the session as soon as accept succeeds ---- client_addr: str | None = None- client_info = getattr(websocket, "client", None)- if client_info is not None:- host = getattr(client_info, "host", None)- port = getattr(client_info, "port", None)- if host is not None and port is not None:- client_addr = f"{host}:{port}"- elif host is not None:- client_addr = str(host)- if ws_sessions is not None:- session_handle = WSSessionHandle(- session_id=session_id,- request_id=request_id,- client_addr=client_addr,- upstream_url=None, # set below once upstream_url is computed- )- ws_sessions.register(session_handle)- metrics = getattr(self, "metrics", None)- if metrics is not None and hasattr(metrics, "inc_active_ws_sessions"):- try:- metrics.inc_active_ws_sessions()- except Exception: # pragma: no cover - defensive- pass-- # Forward all client headers except hop-by-hop / per-connection headers.- # These are WebSocket handshake mechanics that the `websockets` library- # generates fresh for the upstream connection — forwarding them would conflict.- # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.- _skip_headers = frozenset(- {- "host", # must match upstream, not local proxy- "connection", # hop-by-hop- "upgrade", # hop-by-hop- "sec-websocket-key", # per-connection cryptographic nonce- "sec-websocket-version", # protocol version (websockets lib sets this)- "sec-websocket-extensions", # per-connection negotiation- "sec-websocket-accept", # server-side only- "sec-websocket-protocol", # handled via subprotocols param below- "content-length", # hop-by-hop- "transfer-encoding", # hop-by-hop- }- )- upstream_headers: dict[str, str] = {}- for k, v in ws_headers.items():- if k.lower() not in _skip_headers:- upstream_headers[k] = v-- upstream_headers, is_chatgpt_auth = _resolve_codex_routing_headers(upstream_headers)- _lower_headers = {k.lower(): v for k, v in upstream_headers.items()}-- # Build upstream WebSocket URL based on auth mode- if is_chatgpt_auth:- # ChatGPT session auth → route to chatgpt.com backend- upstream_url = "wss://chatgpt.com/backend-api/codex/responses"- logger.debug(- f"[{request_id}] WS: ChatGPT session auth detected, routing to chatgpt.com"- )- else:- # API key auth → route to configured OpenAI API URL- base = self.OPENAI_API_URL- ws_base = base.replace("https://", "wss://").replace("http://", "ws://")- upstream_url = f"{ws_base}/v1/responses"-- # Unit 3: attach the resolved upstream URL to the session handle.- if session_handle is not None:- session_handle.upstream_url = upstream_url-- # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.- # Safety net for clients that don't forward auth headers via WebSocket upgrade.- if "authorization" not in _lower_headers:- api_key = os.environ.get("OPENAI_API_KEY")- if api_key:- upstream_headers["Authorization"] = f"Bearer {api_key}"- logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")- else:- logger.warning(- f"[{request_id}] WS: no Authorization header from client and "- f"OPENAI_API_KEY not set — upstream will likely reject"- )-- upstream_headers = await apply_copilot_api_auth(upstream_headers, url=upstream_url)-- # Ensure the required beta header is present — OpenAI returns 500 without it.- # Codex sends `responses_websockets=2026-02-06`; only inject if missing entirely.- if "openai-beta" not in _lower_headers:- upstream_headers["OpenAI-Beta"] = "responses_websockets=2026-02-06"-- logger.debug(- f"[{request_id}] WS upstream headers: "- f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "- f"subprotocols={client_subprotocols}"- )-- try:- # Receive the first message from client (the response.create request).- # Bound the wait with WS_FIRST_FRAME_TIMEOUT_SECONDS so a zombie- # client that opens the WS but never sends a frame cannot hold a- # session slot indefinitely. The StageTimer measurement still- # captures the elapsed time up to the timeout so operators can- # see the slow-client pattern in the stage-timings log.- try:- async with stage_timer.measure("first_client_frame"):- first_msg_raw = await asyncio.wait_for(- websocket.receive_text(),- timeout=WS_FIRST_FRAME_TIMEOUT_SECONDS,- )- except asyncio.TimeoutError:- logger.info(- f"[{request_id}] WS first-frame timeout after "- f"{WS_FIRST_FRAME_TIMEOUT_SECONDS:.0f}s; closing session "- f"{session_id} (no client data)"- )- termination_cause = "client_timeout"- with contextlib.suppress(Exception):- # 1001 (going away): server is cleanly terminating a slow- # client, not an internal error.- await websocket.close(code=1001, reason="first-frame timeout")- # Exit the outer try so the session-lifecycle ``finally`` runs- # deregister / metrics / stage-timings emission as usual.- return-- # --- Optional: compress the input in the first message ---- body: dict[str, Any] = {}- try:- body = json.loads(first_msg_raw)- tokens_saved = 0- ws_request_body = body.get("response", body)- input_data = (- ws_request_body.get("input") if isinstance(ws_request_body, dict) else None- )-- should_compress = (- self.config.optimize- and isinstance(input_data, list)- and len(input_data) > 1- and not (- ws_request_body.get("previous_response_id")- if isinstance(ws_request_body, dict)- else None- )- )- if should_compress:- try:- from headroom.proxy.responses_converter import (- messages_to_responses_items,- responses_items_to_messages,- )-- model = ws_request_body.get("model", "gpt-4o")- converted, preserved = responses_items_to_messages(input_data)-- messages: list[dict[str, Any]] = []- instructions = ws_request_body.get("instructions")- if instructions:- messages.append({"role": "system", "content": instructions})- messages.extend(converted)-- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- context_limit = self.openai_provider.get_context_limit(model)- async with stage_timer.measure("compression"):- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != messages:- opt = result.messages- if instructions and opt and opt[0].get("role") == "system":- ws_request_body["instructions"] = opt[0]["content"]- opt = opt[1:]- if result.tokens_after <= original_tokens:- ws_request_body["input"] = messages_to_responses_items(- opt, input_data, preserved- )- else:- logger.warning(- f"[{request_id}] WS optimization inflated tokens "- f"({original_tokens} -> {result.tokens_after}), reverting"- )- tokens_saved = max(0, original_tokens - result.tokens_after)- if "response" in body and isinstance(body["response"], dict):- body["response"] = ws_request_body- else:- body = ws_request_body- first_msg_raw = json.dumps(body)- logger.info(- f"[{request_id}] WS /v1/responses compressed: "- f"saved {tokens_saved} tokens"- )- except Exception as e:- logger.warning(f"[{request_id}] WS compression failed: {e}")-- except json.JSONDecodeError:- # Not JSON — pass through as-is- tokens_saved = 0-- # --- Memory: inject context, tools, and instructions ---- memory_user_id: str | None = None- if self.memory_handler and body:- memory_user_id = ws_headers.get(- "x-headroom-user-id",- os.environ.get("USER", os.environ.get("USERNAME", "default")),- )- try:- # Unwrap response.create envelope to access the response body- ws_response_body = body.get("response", body)-- # Debug: log what Codex sends so we can see the full tool list- existing_tool_names = [- t.get("name") or t.get("function", {}).get("name", "?")- for t in (ws_response_body.get("tools") or [])- ]- instr_preview = (ws_response_body.get("instructions") or "")[:200]- logger.info(- f"[{request_id}] WS Memory: Codex tools={existing_tool_names}, "- f"instructions_len={len(ws_response_body.get('instructions') or '')}, "- f"instructions_preview={instr_preview!r}"- )-- # Inject memory context into instructions- if self.memory_handler.config.inject_context:- ws_input = ws_response_body.get("input", "")- ws_instructions = ws_response_body.get("instructions")- ws_msgs: list[dict[str, Any]] = []- if ws_instructions:- ws_msgs.append({"role": "system", "content": ws_instructions})- if isinstance(ws_input, str) and ws_input:- ws_msgs.append({"role": "user", "content": ws_input})- elif isinstance(ws_input, list):- from headroom.proxy.responses_converter import (- responses_items_to_messages,- )-- converted_msgs, _ = responses_items_to_messages(ws_input)- ws_msgs.extend(converted_msgs)-- try:- async with stage_timer.measure("memory_context"):- memory_context = await asyncio.wait_for(- self.memory_handler.search_and_format_context(- memory_user_id, ws_msgs- ),- timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,- )- except asyncio.TimeoutError:- memory_context = None- logger.info(- f"[{request_id}] WS Memory: Context lookup exceeded "- f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "- f"continuing without it"- )- if memory_context:- existing = ws_response_body.get("instructions") or ""- if existing:- ws_response_body["instructions"] = f"{existing}\n\n{memory_context}"- else:- ws_response_body["instructions"] = memory_context- logger.info(- f"[{request_id}] WS Memory: Injected {len(memory_context)} chars "- f"of context into instructions"- )-- # Inject memory tools (Responses API format)- if self.memory_handler.config.inject_tools:- ws_tools = ws_response_body.get("tools") or []- ws_tools, mem_injected = self.memory_handler.inject_tools(- ws_tools, "openai"- )- if mem_injected:- converted_tools = []- for t in ws_tools:- if t.get("type") == "function" and "function" in t:- fn = t["function"]- converted_tools.append(- {- "type": "function",- "name": fn.get("name"),- "description": fn.get("description", ""),- "parameters": fn.get("parameters", {}),- }- )- else:- converted_tools.append(t)- ws_response_body["tools"] = converted_tools-- # Add memory instruction so the model uses- # memory tools as persistent cross-session knowledge.- mem_instruction = (- "\n\n## Memory\n"- "You have persistent memory via memory_search and "- "memory_save tools. Memory stores knowledge across "- "sessions — user info, project details, org context, "- "decisions, architecture, conventions, anything worth "- "remembering.\n\n"- "- ALWAYS call memory_search BEFORE searching files "- "when the user asks a question that could be answered "- "from prior knowledge.\n"- "- Call memory_save to store important facts, decisions, "- "or context that would be useful in future sessions.\n"- "- Memory is your first source of truth for anything "- "not visible in the current conversation."- )- existing_instr = ws_response_body.get("instructions") or ""- ws_response_body["instructions"] = existing_instr + mem_instruction- logger.info(- f"[{request_id}] WS Memory: Injected memory tools + instruction"- )-- # Write back into envelope if it was wrapped- if "response" in body and isinstance(body["response"], dict):- body["response"] = ws_response_body- else:- body = ws_response_body-- first_msg_raw = json.dumps(body)- except Exception as e:- logger.warning(f"[{request_id}] WS Memory injection failed: {e}")-- # --- Connect to upstream OpenAI WebSocket ---- logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")-- # Use ssl=True to let the websockets library handle SSL natively.- # Manual ssl.create_default_context() + certifi doesn't load the- # Windows system cert store, causing HTTP 500 on wss:// connections.- use_ssl: bool | None = True if upstream_url.startswith("wss://") else None-- ws_connected = False- ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))- ws_last_err: Exception | None = None- _upstream_connect_started = time.perf_counter()- _upstream_connect_recorded = False- _upstream_first_event_started: float | None = None-- for ws_attempt in range(ws_connect_attempts):- try:- async with websockets.connect(- upstream_url,- additional_headers=upstream_headers,- subprotocols=(- [websockets.Subprotocol(p) for p in client_subprotocols]- if client_subprotocols and hasattr(websockets, "Subprotocol")- else client_subprotocols or None- ),- ssl=use_ssl,- open_timeout=max(30, self.config.connect_timeout_seconds * 3),- close_timeout=10,- ping_interval=20,- ping_timeout=20,- ) as upstream:- ws_connected = True- if not _upstream_connect_recorded:- stage_timer.record(- "upstream_connect",- (time.perf_counter() - _upstream_connect_started) * 1000.0,- )- _upstream_connect_recorded = True- _upstream_first_event_started = time.perf_counter()- await upstream.send(first_msg_raw)-- # Unit 3: flag the upstream side flips on seeing- # ``response.completed`` so the outer cause- # classifier can prefer it over the raw- # "upstream iterator ended" default.- response_completed_seen = False- # Captures the first exception surfaced by the- # inner relay ``except`` blocks so the outer- # classifier can still tell ``upstream_error``- # from ``upstream_disconnect`` / ``response_completed``- # even though the halves swallow and log.- upstream_relay_error: BaseException | None = None- client_relay_error: BaseException | None = None-- async def _client_to_upstream() -> None:- nonlocal client_relay_error- try:- while True:- msg = await websocket.receive_text()- await upstream.send(msg)- except asyncio.CancelledError:- # Explicit cancel from the outer- # orchestrator — re-raise so- # ``t.cancelled()`` and ``t.exception()``- # behave correctly in the caller.- raise- except Exception as relay_err:- # Surface real errors to the classifier- # without re-raising (existing fork- # behavior: log and return so the- # partner task can be cancelled- # deterministically).- if "WebSocketDisconnect" not in type(relay_err).__name__:- client_relay_error = relay_err- logger.debug(- f"[{request_id}] WS client→upstream relay ended: {relay_err}"- )- with contextlib.suppress(Exception):- await upstream.close()-- async def _upstream_to_client() -> None:- """Relay upstream→client with transparent memory tool handling.-- Uses a buffer-then-decide approach:- 1. Buffer events until first output item arrives- 2. If first output is a memory tool → suppress entire response,- execute tools silently, send continuation upstream- 3. If first output is non-memory → flush buffer, stream normally- 4. Continuation response events are relayed to Codex seamlessly-- This prevents orphaned response.created events from confusing Codex.- """- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES-- # Unit 3: surface response.completed observation- # to the outer scope so the termination-cause- # classifier can prefer ``response_completed``- # over ``upstream_disconnect``.- nonlocal response_completed_seen- nonlocal upstream_relay_error-- memory_enabled = bool(self.memory_handler and memory_user_id)-- # Per-response state (reset after each response.completed)- event_buffer: list[str] = []- decided = False- suppress_response = False- pending_fcs: list[dict[str, Any]] = []- resp_id: str | None = None-- def _reset() -> None:- nonlocal decided, suppress_response, resp_id- event_buffer.clear()- decided = False- suppress_response = False- pending_fcs.clear()- resp_id = None-- # The retry-loop variable is safe to close over here:- # ``_upstream_to_client`` is defined and awaited within- # a single iteration and never escapes.- _first_event_started_at = _upstream_first_event_started # noqa: B023-- try:- async for msg in upstream:- if (- _first_event_started_at is not None- and "upstream_first_event" not in stage_timer- ):- stage_timer.record(- "upstream_first_event",- (time.perf_counter() - _first_event_started_at)- * 1000.0,- )- if isinstance(msg, bytes):- await websocket.send_bytes(msg)- continue- msg_str = msg if isinstance(msg, str) else str(msg)-- if not memory_enabled:- await websocket.send_text(msg_str)- continue-- # Parse event- try:- event = json.loads(msg_str)- except (json.JSONDecodeError, TypeError):- await websocket.send_text(msg_str)- continue-- event_type = event.get("type", "")-- # --- Phase 1: Buffer until first output item ---- if not decided:- event_buffer.append(msg_str)-- if event_type == "response.output_item.added":- item = event.get("item", {})- if (- item.get("type") == "function_call"- and item.get("name") in MEMORY_TOOL_NAMES- ):- # Memory tool first → suppress entire response- suppress_response = True- decided = True- event_buffer.clear()- logger.info(- f"[{request_id}] WS Memory: Detected "- f"{item.get('name')} — suppressing response"- )- else:- # Non-memory first → flush buffer, pass through- decided = True- for buf in event_buffer:- await websocket.send_text(buf)- event_buffer.clear()-- elif event_type == "response.completed":- # No output items at all — flush- decided = True- for buf in event_buffer:- await websocket.send_text(buf)- event_buffer.clear()- _reset()- response_completed_seen = True-- continue-- # --- Phase 2a: Suppress mode (memory response) ---- if suppress_response:- if event_type == "response.output_item.done":- item = event.get("item", {})- if (- item.get("type") == "function_call"- and item.get("name") in MEMORY_TOOL_NAMES- ):- pending_fcs.append(item)-- elif event_type == "response.completed":- response_completed_seen = True- resp = event.get("response", {})- resp_id = resp.get("id")-- if pending_fcs:- logger.info(- f"[{request_id}] WS Memory: Executing "- f"{len(pending_fcs)} tool(s) transparently"- )-- # Execute memory tool calls- tool_outputs: list[dict[str, Any]] = []- for fc in pending_fcs:- call_id = fc.get("call_id", fc.get("id", ""))- fc_name = fc.get("name", "")- args_str = fc.get("arguments", "{}")- try:- fc_args = json.loads(args_str)- except json.JSONDecodeError:- fc_args = {}-- await self.memory_handler._ensure_initialized()- if self.memory_handler._backend:- result = await self.memory_handler._execute_memory_tool(- fc_name,- fc_args,- memory_user_id,- "openai",- )- else:- result = json.dumps(- {"error": "backend not ready"}- )-- tool_outputs.append(- {- "type": "function_call_output",- "call_id": call_id,- "output": result,- }- )- logger.info(- f"[{request_id}] WS Memory: Executed "- f"{fc_name} for user {memory_user_id}"- )-- # Send continuation upstream- cont: dict[str, Any] = {- "type": "response.create",- "response": {"input": tool_outputs},- }- if resp_id:- cont["response"]["previous_response_id"] = (- resp_id- )- await upstream.send(json.dumps(cont))- logger.info(- f"[{request_id}] WS Memory: Sent continuation "- f"with {len(tool_outputs)} result(s)"- )-- _reset()- # All events suppressed in this mode- continue-- # --- Phase 2b: Pass-through mode ---- await websocket.send_text(msg_str)-- except asyncio.CancelledError:- raise- except Exception as relay_err:- if "WebSocketDisconnect" not in type(relay_err).__name__:- # Capture for the outer classifier- # so ``upstream_error`` can be- # distinguished from a clean- # upstream disconnect.- upstream_relay_error = relay_err- logger.debug(- f"[{request_id}] WS upstream→client relay ended: {relay_err}"- )- finally:- with contextlib.suppress(Exception):- await websocket.close()-- # --- Unit 3: deterministic relay-task cancellation ---- # Spawn each half as a named task so we can:- # (a) attach them to the session registry for- # ``/debug/ws-sessions``,- # (b) cancel the survivor explicitly when the- # first one exits, and- # (c) classify the termination cause for the- # duration histogram.- client_task = asyncio.create_task(- _client_to_upstream(),- name=f"codex-ws-c2u-{session_id}",- )- upstream_task = asyncio.create_task(- _upstream_to_client(),- name=f"codex-ws-u2c-{session_id}",- )- relay_tasks = [client_task, upstream_task]- if ws_sessions is not None:- ws_sessions.attach_tasks(session_id, relay_tasks)- metrics_for_tasks = getattr(self, "metrics", None)- if metrics_for_tasks is not None and hasattr(- metrics_for_tasks, "inc_active_relay_tasks"- ):- try:- metrics_for_tasks.inc_active_relay_tasks(len(relay_tasks))- except Exception: # pragma: no cover - defensive- pass-- try:- done, pending = await asyncio.wait(- {client_task, upstream_task},- return_when=asyncio.FIRST_COMPLETED,- )- # Cancel the survivor so we don't leak the- # partner task. Suppress the CancelledError- # we just raised ourselves — any *other*- # exception from the cancelled task is- # already logged inside its own try/except.- for t in pending:- t.cancel()- if pending:- with contextlib.suppress(asyncio.CancelledError):- await asyncio.gather(*pending, return_exceptions=True)-- # Classify termination cause from whichever- # task completed first. ``CancelledError``- # can show up on the "done" side if the- # handler itself was cancelled from outside- # (e.g. server shutdown).- for t in done:- exc = None- # Cancelled tasks raise CancelledError from- # .exception(); surface it explicitly so the- # downstream ``isinstance(exc, CancelledError)``- # branches actually run. For any other- # unexpected state (``InvalidStateError`` if- # the task somehow isn't done — shouldn't- # happen post-gather but defensive), we- # suppress and leave ``exc`` as ``None``.- if t.cancelled():- exc = asyncio.CancelledError()- else:- with contextlib.suppress(asyncio.InvalidStateError):- exc = t.exception()- task_name = t.get_name() or ""- if t is client_task:- if client_relay_error is not None:- termination_cause = "client_error"- elif exc is None:- termination_cause = "client_disconnect"- elif isinstance(exc, asyncio.CancelledError):- termination_cause = "client_disconnect"- else:- # Distinguish legitimate client- # disconnect exceptions from- # real errors: WebSocketDisconnect- # is a normal client exit.- if "WebSocketDisconnect" in type(exc).__name__:- termination_cause = "client_disconnect"- else:- termination_cause = "client_error"- elif t is upstream_task:- if upstream_relay_error is not None:- termination_cause = "upstream_error"- logger.debug(- f"[{request_id}] WS relay {task_name} "- f"raised: {upstream_relay_error!r}"- )- elif exc is None:- termination_cause = (- "response_completed"- if response_completed_seen- else "upstream_disconnect"- )- elif isinstance(exc, asyncio.CancelledError):- termination_cause = "upstream_disconnect"- else:- termination_cause = "upstream_error"- logger.debug(- f"[{request_id}] WS relay {task_name} raised: {exc!r}"- )- finally:- # In case anything above raised before the- # cancel-and-await loop ran.- for t in relay_tasks:- if not t.done():- t.cancel()- with contextlib.suppress(asyncio.CancelledError):- await asyncio.gather(*relay_tasks, return_exceptions=True)-- logger.info(- f"[{request_id}] WS /v1/responses completed "- f"(tokens_saved={tokens_saved}, cause={termination_cause})"- )- break- except Exception as ws_err:- if ws_connected:- # WS was established but broke mid-stream — re-raise- raise-- ws_last_err = ws_err- if ws_attempt >= ws_connect_attempts - 1:- break-- delay_with_jitter = jitter_delay_ms(- self.config.retry_base_delay_ms,- self.config.retry_max_delay_ms,- ws_attempt,- )- logger.warning(- f"[{request_id}] WS upstream connect failed "- f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "- f"retrying in {delay_with_jitter:.0f}ms"- )- await asyncio.sleep(delay_with_jitter / 1000)-- if not ws_connected:- # WS upgrade failed (HTTP 500 from OpenAI is common).- # Fall back to HTTP POST streaming and relay SSE events- # back over the client WebSocket transparently.- ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")- _ws_detail = str(ws_err)- if hasattr(ws_err, "response"):- resp_body = getattr(getattr(ws_err, "response", None), "body", b"")- if resp_body:- _ws_detail += f" | {resp_body[:300].decode('utf-8', errors='replace')}"- logger.warning(- f"[{request_id}] WS upstream failed ({_ws_detail}), "- f"falling back to HTTP POST streaming"- )- await self._ws_http_fallback(- websocket, body, first_msg_raw, upstream_headers, request_id- )-- # Record metrics- if tokens_saved > 0:- model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"- await self.metrics.record_request(- provider="openai",- model=model_name,- input_tokens=0,- output_tokens=0,- tokens_saved=tokens_saved,- latency_ms=0,- )-- except Exception as e:- if "WebSocketDisconnect" in type(e).__name__:- # Unit 3: client dropped the socket before or during- # relay. The registry classifier may already have set- # ``client_disconnect`` via the relay task exit path;- # preserve that, otherwise set it here.- if termination_cause == "unknown":- termination_cause = "client_disconnect"- else:- # Extract response body from websockets InvalidStatus for better debugging- error_detail = str(e)- if hasattr(e, "response"):- try:- resp = e.response- body_bytes = getattr(resp, "body", None) or b""- if body_bytes:- error_detail += (- f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"- )- except Exception:- pass- logger.error(f"[{request_id}] WS proxy error: {error_detail}")- if termination_cause == "unknown":- termination_cause = "client_error"- with contextlib.suppress(Exception):- await websocket.close(code=1011, reason=str(e)[:120])- finally:- # Unit 2: emit structured per-session stage timings.- stage_timer.record(- "total_session",- (time.perf_counter() - session_started_at) * 1000.0,- )- # Unit 3: deregister the session before (or independently- # of) the stage-timings log so a failure there cannot leak- # the registry entry. ``deregister`` is idempotent, so a- # session that never registered is a no-op.- if ws_sessions is not None and session_handle is not None:- # Use deregister_and_count so the handle pop and the- # relay-task count are read atomically inside the- # registry. Capturing ``len(session_handle.relay_tasks)``- # separately before ``deregister`` would risk drift if- # the registry's bookkeeping ever changes.- _deregistered, released_tasks = ws_sessions.deregister_and_count(- session_id, cause=termination_cause- )- session_duration_ms = (time.perf_counter() - session_started_at) * 1000.0- metrics_for_close = getattr(self, "metrics", None)- if metrics_for_close is not None:- with contextlib.suppress(Exception):- if hasattr(metrics_for_close, "dec_active_ws_sessions"):- metrics_for_close.dec_active_ws_sessions()- if released_tasks and hasattr(metrics_for_close, "dec_active_relay_tasks"):- metrics_for_close.dec_active_relay_tasks(released_tasks)- if hasattr(metrics_for_close, "record_ws_session_duration"):- metrics_for_close.record_ws_session_duration(- session_duration_ms, termination_cause- )- await emit_stage_timings_log(- path="openai_responses_ws",- request_id=request_id,- session_id=session_id,- stage_timer=stage_timer,- expected_stages=(- "accept",- "first_client_frame",- "upstream_connect",- "upstream_first_event",- "memory_context",- "compression",- "total_session",- ),- metrics=getattr(self, "metrics", None),- )-- async def _ws_http_fallback(- self,- websocket: WebSocket,- body: dict[str, Any],- first_msg_raw: str,- upstream_headers: dict[str, str],- request_id: str,- ) -> None:- """Fall back to HTTP POST streaming when upstream WS fails.-- Converts the WS ``response.create`` message to an HTTP POST to- ``/v1/responses?stream=true``, reads SSE events, and relays each- ``data:`` line as a WS text message to the client. This makes- Codex work immediately instead of exhausting its WS retry budget.- """- # Route to correct endpoint based on auth mode- _lower = {k.lower() for k in upstream_headers}- if "chatgpt-account-id" in _lower:- http_url = "https://chatgpt.com/backend-api/codex/responses"- else:- http_url = f"{self.OPENAI_API_URL}/v1/responses"-- # Build HTTP body from the WS response.create payload.- # WS messages use {"type": "response.create", "response": {...}} wrapper.- # The HTTP POST endpoint expects the inner response object directly.- http_body: dict[str, Any]- try:- parsed = json.loads(first_msg_raw) if isinstance(first_msg_raw, str) else body- except (json.JSONDecodeError, TypeError):- parsed = body-- # Normalize WebSocket response.create payload into the HTTP request body.- # Codex may send either:- # 1. {"type":"response.create","response":{...}}- # 2. {"type":"response.create", ...response fields...}- if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):- http_body = dict(parsed["response"])- elif isinstance(parsed, dict):- http_body = dict(parsed)- if http_body.get("type") == "response.create":- http_body.pop("type", None)- else:- http_body = body if isinstance(body, dict) else {}-- # Some clients include response-ish metadata that the HTTP endpoint rejects.- if http_body.get("type") in {"response.create", "response"}:- http_body.pop("type", None)-- # Ensure streaming is enabled so we get SSE events- http_body["stream"] = True-- # Build HTTP headers from the upstream headers (already stripped of WS- # hop-by-hop headers by the caller).- http_headers = dict(upstream_headers)- http_headers["content-type"] = "application/json"-- logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")-- try:- retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))- for http_attempt in range(retry_attempts):- try:- async with self.http_client.stream(- "POST",- http_url,- headers=http_headers,- json=http_body,- timeout=120.0,- ) as response:- if response.status_code != 200:- error_body = b""- async for chunk in response.aiter_bytes():- error_body += chunk- if len(error_body) > 2000:- break- error_text = error_body.decode("utf-8", errors="replace")- logger.error(- f"[{request_id}] WS→HTTP fallback got {response.status_code}: "- f"{error_text[:500]}"- )- # Send error as WS message so client sees it- error_event = {- "type": "error",- "error": {- "type": "server_error",- "message": f"Upstream returned {response.status_code}",- },- }- await websocket.send_text(json.dumps(error_event))- return-- # Relay SSE events as WS text messages- buffer = ""- async for chunk in response.aiter_text():- buffer += chunk- while "\n" in buffer:- line, buffer = buffer.split("\n", 1)- line = line.strip()- if not line:- continue- if line.startswith("data: "):- data = line[6:]- if data == "[DONE]":- continue- try:- await websocket.send_text(data)- except Exception:- return- elif line.startswith("event: "):- # SSE event type — skip, the data line contains the type- continue-- # Flush any remaining data in buffer- for line in buffer.strip().splitlines():- line = line.strip()- if line.startswith("data: ") and line[6:] != "[DONE]":- with contextlib.suppress(Exception):- await websocket.send_text(line[6:])- return- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:- if http_attempt >= retry_attempts - 1:- raise-- delay_with_jitter = jitter_delay_ms(- self.config.retry_base_delay_ms,- self.config.retry_max_delay_ms,- http_attempt,- )- logger.warning(- f"[{request_id}] WS→HTTP fallback connect failed "- f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "- f"retrying in {delay_with_jitter:.0f}ms"- )- await asyncio.sleep(delay_with_jitter / 1000)-- except Exception as http_err:- logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")- error_event = {- "type": "error",- "error": {- "type": "server_error",- "message": f"HTTP fallback failed: {http_err!s}"[:200],- },- }- with contextlib.suppress(Exception):- await websocket.send_text(json.dumps(error_event))- finally:- with contextlib.suppress(Exception):- await websocket.close()-- async def handle_compress(self, request: Request) -> JSONResponse:- """Compress messages without calling an LLM.-- POST /v1/compress- Body: {"messages": [...], "model": "...", "config": {}}- Returns compressed messages + metrics.- """- from fastapi.responses import JSONResponse-- from headroom.proxy.helpers import _read_request_json-- # Check bypass header- if request.headers.get("x-headroom-bypass", "").lower() == "true":- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={"error": f"Invalid request body: {e!s}"},- )- messages = body.get("messages", [])- return JSONResponse(- {- "messages": messages,- "tokens_before": 0,- "tokens_after": 0,- "tokens_saved": 0,- "compression_ratio": 1.0,- "transforms_applied": [],- "ccr_hashes": [],- }- )-- try:- body = await _read_request_json(request)- except Exception:- return JSONResponse(- status_code=400,- content={- "error": {- "type": "invalid_request",- "message": "Invalid JSON in request body.",- }- },- )-- messages = body.get("messages")- model = body.get("model")-- if messages is None:- return JSONResponse(- status_code=400,- content={- "error": {- "type": "invalid_request",- "message": "Missing required field: messages",- }- },- )-- if model is None:- return JSONResponse(- status_code=400,- content={- "error": {- "type": "invalid_request",- "message": "Missing required field: model",- }- },- )-- if not messages:- return JSONResponse(- {- "messages": [],- "tokens_before": 0,- "tokens_after": 0,- "tokens_saved": 0,- "compression_ratio": 1.0,- "transforms_applied": [],- "ccr_hashes": [],- }- )-- try:- # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)- # Allow optional token_budget to override model's context limit- # (used by OpenClaw compact() and other callers that need tighter budgets)- token_budget = body.get("token_budget")- context_limit = (- token_budget- if token_budget and isinstance(token_budget, int)- else self.openai_provider.get_context_limit(model)- )- # Extract CompressConfig options from request body- compress_config = body.get("config", {})- compress_user_messages = compress_config.get("compress_user_messages", False)- target_ratio = compress_config.get("target_ratio")- protect_recent = compress_config.get("protect_recent")- protect_analysis_context = compress_config.get("protect_analysis_context")-- pipeline_kwargs: dict = {"model_limit": context_limit}- if compress_user_messages:- pipeline_kwargs["compress_user_messages"] = True- if target_ratio is not None:- pipeline_kwargs["target_ratio"] = float(target_ratio)- if protect_recent is not None:- pipeline_kwargs["protect_recent"] = int(protect_recent)- if protect_analysis_context is not None:- pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)-- result = self.openai_pipeline.apply(- messages=messages,- model=model,- **pipeline_kwargs,- )-- return JSONResponse(- {- "messages": result.messages,- "tokens_before": result.tokens_before,- "tokens_after": result.tokens_after,- "tokens_saved": result.tokens_before - result.tokens_after,- "compression_ratio": (- result.tokens_after / result.tokens_before- if result.tokens_before > 0- else 1.0- ),- "transforms_applied": result.transforms_applied,- "transforms_summary": result.transforms_summary,- "ccr_hashes": result.markers_inserted,- }- )- except Exception as e:- logger.exception("Compression failed: %s", e)- return JSONResponse(- status_code=503,- content={- "error": {- "type": "compression_error",- "message": str(e),- }- },- )-- async def handle_passthrough(- self,- request: Request,- base_url: str,- endpoint_name: str | None = None,- provider: str | None = None,- ) -> Response:- """Pass through request unchanged.-- Args:- request: The incoming request- base_url: The upstream API base URL- endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")- provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")- """- from fastapi.responses import Response-- start_time = time.time()- path = request.url.path- url = f"{base_url}{path}"-- # Preserve query string parameters- if request.url.query:- url = f"{url}?{request.url.query}"-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("accept-encoding", None)-- body = await request.body()-- headers = await apply_copilot_api_auth(headers, url=url)- response = await self.http_client.request( # type: ignore[union-attr]- method=request.method,- url=url,- headers=headers,- content=body,- )-- # Remove compression headers since httpx already decompressed the response- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression-- # Track stats for passthrough requests- if endpoint_name and provider:- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider=provider,- model=f"passthrough:{endpoint_name}",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- cached=False,- )-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- async def handle_databricks_invocations(- self,- request: Request,- model: str,- ) -> Response | StreamingResponse:- """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.-- This enables using the Databricks CLI directly with Headroom:- databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'-- The request/response format is identical to OpenAI chat completions,- so we inject the model from the path and delegate to handle_openai_chat.- """- from fastapi.responses import JSONResponse-- from headroom.proxy.helpers import _read_request_json-- request_id = await self._next_request_id()-- try:- body = await _read_request_json(request)- except Exception as e:- logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid JSON: {e}",- "type": "invalid_request_error",- }- },- )-- # Inject model from path into body (Databricks CLI passes model in URL, not body)- body["model"] = model-- logger.info(f"[{request_id}] Databricks invocation: model={model}")-- # Create a new request with the modified body- # We reuse the OpenAI chat handler since the format is identical- from starlette.requests import Request as StarletteRequest-- # Build new scope with the body already parsed- scope = dict(request.scope)-- # Create a simple receive function that returns our modified body- body_bytes = json.dumps(body).encode()-- async def receive():- return {"type": "http.request", "body": body_bytes}-- modified_request = StarletteRequest(scope, receive)-- # Delegate to the OpenAI chat handler (same format)- return await self.handle_openai_chat(modified_request)+"""OpenAI handler mixin for HeadroomProxy.++Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.+"""++from __future__ import annotations++import asyncio+import base64+import contextlib+import copy+import json+import logging+import os+import time+import uuid+from datetime import datetime+from typing import TYPE_CHECKING, Any++from headroom.proxy.helpers import jitter_delay_ms+from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log+from headroom.proxy.ws_session_registry import (+ TerminationCause,+ WebSocketSessionRegistry,+ WSSessionHandle,+)++if TYPE_CHECKING:+ from fastapi import Request, WebSocket+ from fastapi.responses import JSONResponse, Response, StreamingResponse++import httpx++from headroom.copilot_auth import apply_copilot_api_auth+from headroom.pipeline import PipelineStage, summarize_routing_markers++logger = logging.getLogger("headroom.proxy")+++# Interactive Responses turns are latency-sensitive. Fail open quickly rather+# than holding the session hostage on memory lookup.+RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0++# Cap the wait for the first client frame after the WS handshake completes.+# A zombie or malicious client that accepts the upgrade but never sends the+# first response.create frame would otherwise hold a slot indefinitely and+# starve the session registry. 60 s is generous for real clients (Codex+# typically sends the first frame within a few hundred milliseconds of the+# accept) but short enough to bound the damage from a hung peer.+WS_FIRST_FRAME_TIMEOUT_SECONDS = 60.0+++def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:+ """Best-effort decode of an OpenAI OAuth bearer token payload.++ OpenClaw's Codex OAuth flow may forward only the bearer token after the+ provider base URL is overridden. In that case the explicit+ ``ChatGPT-Account-ID`` header can be missing even though the JWT still+ carries the account id we need to route to the ChatGPT Codex backend.+ """+ auth = headers.get("authorization") or headers.get("Authorization")+ if not auth:+ return None++ scheme, _, token = auth.partition(" ")+ if scheme.lower() != "bearer" or token.count(".") < 2:+ return None++ payload = token.split(".", 2)[1]+ payload += "=" * (-len(payload) % 4)+ # Intentionally no signature verification here: this is only a best-effort+ # routing hint extractor. Upstream still performs the actual auth/authz checks.+ try:+ decoded = base64.urlsafe_b64decode(payload.encode("ascii"))+ data = json.loads(decoded.decode("utf-8"))+ except (ValueError, UnicodeDecodeError):+ return None++ return data if isinstance(data, dict) else None+++def _resolve_codex_routing_headers(headers: dict[str, str]) -> tuple[dict[str, str], bool]:+ """Resolve ChatGPT Codex routing hints from explicit headers or OAuth JWT."""+ resolved = dict(headers)+ lower_lookup = {k.lower(): k for k in resolved}++ if "chatgpt-account-id" in lower_lookup:+ return resolved, True++ payload = _decode_openai_bearer_payload(resolved)+ auth_claims = payload.get("https://api.openai.com/auth") if isinstance(payload, dict) else None+ account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None+ if isinstance(account_id, str) and account_id.strip():+ resolved["ChatGPT-Account-ID"] = account_id.strip()+ return resolved, True++ return resolved, False+++class OpenAIHandlerMixin:+ """Mixin providing OpenAI API handler methods for HeadroomProxy."""++ @staticmethod+ def _strict_previous_turn_frozen_count(+ messages: list[dict[str, Any]],+ base_frozen_count: int,+ ) -> int:+ """Freeze all prior turns in cache mode; only final user turn is mutable."""+ if not messages:+ return base_frozen_count+ final_idx = len(messages) - 1+ if messages[final_idx].get("role") == "user":+ return max(base_frozen_count, final_idx)+ return len(messages)++ @staticmethod+ def _restore_frozen_prefix(+ original_messages: list[dict[str, Any]],+ candidate_messages: list[dict[str, Any]],+ *,+ frozen_message_count: int,+ ) -> tuple[list[dict[str, Any]], int]:+ """Force frozen prefix bytes to match original request exactly."""+ if frozen_message_count <= 0 or not original_messages:+ return candidate_messages, 0++ frozen = min(frozen_message_count, len(original_messages))+ restored = list(candidate_messages)++ if len(restored) < frozen:+ return list(original_messages[:frozen]) + restored, frozen++ changed = 0+ for idx in range(frozen):+ if restored[idx] != original_messages[idx]:+ restored[idx] = original_messages[idx]+ changed += 1+ return restored, changed++ async def handle_openai_chat(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle OpenAI /v1/chat/completions endpoint."""+ if not hasattr(self, "pipeline_extensions"):+ from headroom.pipeline import PipelineExtensionManager++ self.pipeline_extensions = PipelineExtensionManager(discover=False)++ from fastapi import HTTPException+ from fastapi.responses import JSONResponse, Response++ from headroom.ccr import CCRToolInjector+ from headroom.proxy.helpers import (+ COMPRESSION_TIMEOUT_SECONDS,+ MAX_MESSAGE_ARRAY_LENGTH,+ MAX_REQUEST_BODY_SIZE,+ _read_request_json,+ )+ from headroom.proxy.modes import is_cache_mode, is_token_mode+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "type": "invalid_request_error",+ "code": "request_too_large",+ }+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )+ model = body.get("model", "unknown")+ messages = body.get("messages", [])+ original_client_messages = copy.deepcopy(messages)+ input_event = self.pipeline_extensions.emit(+ PipelineStage.INPUT_RECEIVED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=messages,+ tools=body.get("tools"),+ metadata={"path": "/v1/chat/completions", "stream": body.get("stream", False)},+ )+ if input_event.messages is not None:+ messages = input_event.messages+ original_client_messages = copy.deepcopy(messages)+ if input_event.tools is not None:+ body["tools"] = input_event.tools++ # Validate message array size+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Message array too large ({len(messages)} messages). "+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",+ "type": "invalid_request_error",+ "code": "invalid_request",+ }+ },+ )++ stream = body.get("stream", False)++ # Bypass: skip ALL compression for explicit opt-out+ _bypass = (+ request.headers.get("x-headroom-bypass", "").lower() == "true"+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"+ )+ if _bypass:+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")++ # Image compression: tile alignment + ML-based technique routing+ if self.config.image_optimize and messages and not _bypass:+ from headroom.proxy.helpers import _get_image_compressor++ compressor = _get_image_compressor()+ if compressor and compressor.has_images(messages):+ messages = compressor.compress(messages, provider="openai")+ if compressor.last_result:+ logger.info(+ f"[{request_id}] Image: {compressor.last_result.technique.value} "+ f"({compressor.last_result.savings_percent:.0f}% saved, "+ f"{compressor.last_result.original_tokens} → "+ f"{compressor.last_result.compressed_tokens} tokens)"+ )++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ # Strip accept-encoding so httpx negotiates its own encoding.+ # Cloudflare Workers forward "br, zstd" which OpenAI may honor;+ # if httpx lacks brotli support the response body is undecipherable → 502.+ headers.pop("accept-encoding", None)+ tags = self._extract_tags(headers)++ # Memory: Get user ID when memory is enabled+ memory_user_id: str | None = None+ if self.memory_handler:+ memory_user_id = headers.get(+ "x-headroom-user-id",+ os.environ.get("USER", os.environ.get("USERNAME", "default")),+ )++ # Rate limiting+ if self.rate_limiter:+ rate_key = headers.get("authorization", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited(provider="openai")+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Check cache+ if self.cache and not stream:+ cached = await self.cache.get(messages, model)+ if cached:+ self.pipeline_extensions.emit(+ PipelineStage.INPUT_CACHED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=messages,+ metadata={"cache_hit": True, "path": "/v1/chat/completions"},+ )+ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0, # Savings already counted when response was cached+ latency_ms=(time.time() - start_time) * 1000,+ cached=True,+ )++ # Remove compression headers from cached response+ response_headers = dict(cached.response_headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(content=cached.response_body, headers=response_headers)++ # Token counting+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Hook: pre_compress+ _hook_biases = None+ if self.config.hooks:+ from headroom.hooks import CompressContext++ _hook_ctx = CompressContext(model=model, provider="openai")+ try:+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)+ _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)+ except Exception as e:+ logger.debug(f"[{request_id}] Hook error: {e}")++ # Optimization+ transforms_applied = []+ pipeline_timing: dict[str, float] = {}+ waste_signals_dict: dict[str, int] | None = None+ optimized_messages = messages+ optimized_tokens = original_tokens++ # Get prefix cache tracker for this session+ openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)+ openai_prefix_tracker = self.session_tracker_store.get_or_create(+ openai_session_id, "openai"+ )+ openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()+ if is_cache_mode(self.config.mode):+ openai_frozen_count = self._strict_previous_turn_frozen_count(+ original_client_messages,+ openai_frozen_count,+ )++ _compression_failed = False+ original_messages = messages # Preserve for 400-retry fallback+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True+ if self.config.optimize and messages and not _bypass and _license_ok:+ try:+ context_limit = self.openai_provider.get_context_limit(model)++ if is_token_mode(self.config.mode):+ comp_cache = self._get_compression_cache(openai_session_id)++ # Zone 1: Swap cached compressed versions+ working_messages = comp_cache.apply_cached(messages)++ # Re-freeze boundary+ openai_frozen_count = comp_cache.compute_frozen_count(messages)++ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=working_messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(working_messages),+ frozen_message_count=openai_frozen_count,+ biases=_hook_biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != working_messages:+ comp_cache.update_from_result(messages, result.messages)++ # Always use pipeline result in token mode+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)+ # so tokens_saved captures both Zone 1 + Zone 2 savings.+ optimized_tokens = result.tokens_after+ else:+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ frozen_message_count=openai_frozen_count,+ biases=_hook_biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after++ if result.waste_signals:+ waste_signals_dict = result.waste_signals.to_dict()+ except Exception as e:+ logger.warning(f"Optimization failed: {e}")+ # Flag compression failure for observability+ _compression_failed = True++ # Guard: if "optimization" inflated tokens, revert to originals+ if optimized_tokens > original_tokens:+ logger.warning(+ f"[{request_id}] Optimization inflated tokens "+ f"({original_tokens} -> {optimized_tokens}), reverting to original messages"+ )+ optimized_messages = original_messages+ optimized_tokens = original_tokens+ transforms_applied = []++ tokens_saved = original_tokens - optimized_tokens+ optimization_latency = (time.time() - start_time) * 1000++ routing_markers = summarize_routing_markers(transforms_applied)+ if routing_markers:+ routed_event = self.pipeline_extensions.emit(+ PipelineStage.INPUT_ROUTED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=optimized_messages,+ metadata={+ "routing_markers": routing_markers,+ "transforms_applied": transforms_applied,+ },+ )+ if routed_event.messages is not None:+ optimized_messages = routed_event.messages+ optimized_tokens = tokenizer.count_messages(optimized_messages)+ tokens_saved = original_tokens - optimized_tokens++ compressed_event = self.pipeline_extensions.emit(+ PipelineStage.INPUT_COMPRESSED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=optimized_messages,+ metadata={+ "tokens_before": original_tokens,+ "tokens_after": optimized_tokens,+ "transforms_applied": transforms_applied,+ },+ )+ if compressed_event.messages is not None:+ optimized_messages = compressed_event.messages+ optimized_tokens = tokenizer.count_messages(optimized_messages)+ tokens_saved = original_tokens - optimized_tokens++ # Hook: post_compress+ if self.config.hooks and tokens_saved > 0:+ from headroom.hooks import CompressEvent++ try:+ self.config.hooks.post_compress(+ CompressEvent(+ tokens_before=original_tokens,+ tokens_after=optimized_tokens,+ tokens_saved=tokens_saved,+ compression_ratio=tokens_saved / original_tokens+ if original_tokens > 0+ else 0,+ transforms_applied=transforms_applied,+ model=model,+ provider="openai",+ )+ )+ except Exception as e:+ logger.debug(f"[{request_id}] post_compress hook error: {e}")++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = body.get("tools")+ _original_tools = tools # Preserve for diagnostic / future retry+ if (+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions+ ) and not _bypass:+ injector = CCRToolInjector(+ provider="openai",+ inject_tool=self.config.ccr_inject_tool,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, tools, was_injected = injector.process_request(+ optimized_messages, tools+ )++ if injector.has_compressed_content:+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"+ )+ else:+ logger.debug(+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"+ )++ # Query Echo: disabled — hurts prefix caching in long conversations.+ if is_cache_mode(self.config.mode):+ optimized_messages, restored_count = self._restore_frozen_prefix(+ original_client_messages,+ optimized_messages,+ frozen_message_count=openai_frozen_count,+ )+ if restored_count > 0:+ logger.warning(+ f"[{request_id}] Restored {restored_count} frozen prefix message(s) "+ "to preserve cache stability (openai)"+ )++ # Memory: inject context and tools for OpenAI requests+ memory_context_injected = False+ memory_tools_injected = False+ if self.memory_handler and memory_user_id:+ try:+ # Inject memory context (search similar memories, add as system message)+ if self.memory_handler.config.inject_context:+ memory_context = await self.memory_handler.search_and_format_context(+ memory_user_id, optimized_messages+ )+ if memory_context:+ # Prepend as system message for OpenAI format+ optimized_messages = [+ {"role": "system", "content": memory_context},+ *optimized_messages,+ ]+ memory_context_injected = True+ logger.info(+ f"[{request_id}] Memory: Injected {len(memory_context)} chars "+ f"of context for user {memory_user_id}"+ )++ # Inject memory tools+ if self.memory_handler.config.inject_tools:+ tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "openai")+ if mem_tools_injected:+ memory_tools_injected = True+ logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")+ except Exception as e:+ logger.warning(f"[{request_id}] Memory injection failed: {e}")++ if memory_context_injected or memory_tools_injected:+ remembered_event = self.pipeline_extensions.emit(+ PipelineStage.INPUT_REMEMBERED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=optimized_messages,+ tools=tools,+ metadata={+ "memory_context_injected": memory_context_injected,+ "memory_tools_injected": memory_tools_injected,+ },+ )+ if remembered_event.messages is not None:+ optimized_messages = remembered_event.messages+ if remembered_event.tools is not None:+ tools = remembered_event.tools++ body["messages"] = optimized_messages+ if tools is not None:+ body["tools"] = tools++ presend_event = self.pipeline_extensions.emit(+ PipelineStage.PRE_SEND,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=optimized_messages,+ tools=tools,+ headers=headers,+ metadata={"path": "/v1/chat/completions", "stream": stream},+ )+ if presend_event.messages is not None:+ optimized_messages = presend_event.messages+ body["messages"] = optimized_messages+ if presend_event.tools is not None:+ tools = presend_event.tools+ body["tools"] = tools+ if presend_event.headers is not None:+ headers = presend_event.headers+ optimized_tokens = tokenizer.count_messages(body["messages"])+ tokens_saved = original_tokens - optimized_tokens++ # Route through LiteLLM/any-llm backend if configured+ if self.anthropic_backend is not None:+ try:+ if stream:+ self.pipeline_extensions.emit(+ PipelineStage.POST_SEND,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=body["messages"],+ tools=tools,+ metadata={"path": "/v1/chat/completions", "stream": True},+ )+ # Streaming: use stream_openai_message() → SSE events+ return await self._stream_openai_via_backend(+ body,+ headers,+ model,+ request_id,+ start_time,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ pipeline_timing=pipeline_timing,+ )+ else:+ # Non-streaming: use send_openai_message() → JSON+ backend_response = await self.anthropic_backend.send_openai_message(+ body, headers+ )+ self.pipeline_extensions.emit(+ PipelineStage.POST_SEND,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=body["messages"],+ tools=tools,+ response=backend_response.body,+ metadata={+ "path": "/v1/chat/completions",+ "stream": False,+ "status_code": backend_response.status_code,+ },+ )+ self.pipeline_extensions.emit(+ PipelineStage.RESPONSE_RECEIVED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ response=backend_response.body,+ metadata={+ "path": "/v1/chat/completions",+ "stream": False,+ "status_code": backend_response.status_code,+ },+ )++ if backend_response.error:+ return JSONResponse(+ status_code=backend_response.status_code,+ content=backend_response.body,+ )++ # Track metrics+ total_latency = (time.time() - start_time) * 1000+ usage = backend_response.body.get("usage", {})+ output_tokens = usage.get("completion_tokens", 0)+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)++ await self.metrics.record_request(+ provider=self.anthropic_backend.name,+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cached=False,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"+ )++ return JSONResponse(+ status_code=backend_response.status_code,+ content=backend_response.body,+ )+ except Exception as e:+ logger.error(f"[{request_id}] Backend error: {e}")+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "message": str(e),+ "type": "api_error",+ "code": "backend_error",+ }+ },+ )++ # Direct OpenAI API (no backend configured)+ url = f"{self.OPENAI_API_URL}/v1/chat/completions"++ try:+ if stream:+ # Inject stream_options to get usage stats in streaming response+ # This allows accurate token counting instead of byte-based estimation+ if "stream_options" not in body:+ body["stream_options"] = {"include_usage": True}+ elif isinstance(body.get("stream_options"), dict):+ body["stream_options"]["include_usage"] = True++ self.pipeline_extensions.emit(+ PipelineStage.POST_SEND,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=body["messages"],+ tools=tools,+ metadata={"path": "/v1/chat/completions", "stream": True},+ )+ return await self._stream_response(+ url,+ headers,+ body,+ "openai",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ pipeline_timing=pipeline_timing,+ prefix_tracker=openai_prefix_tracker,+ )+ else:+ headers = await apply_copilot_api_auth(headers, url=url)+ response = await self._retry_request("POST", url, headers, body)+ self.pipeline_extensions.emit(+ PipelineStage.POST_SEND,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ messages=body["messages"],+ tools=tools,+ response=response,+ metadata={+ "path": "/v1/chat/completions",+ "stream": False,+ "status_code": response.status_code,+ },+ )+ self.pipeline_extensions.emit(+ PipelineStage.RESPONSE_RECEIVED,+ operation="proxy.request",+ request_id=request_id,+ provider="openai",+ model=model,+ response=response,+ metadata={+ "path": "/v1/chat/completions",+ "stream": False,+ "status_code": response.status_code,+ },+ )++ # Full diagnostic dump on upstream errors (OpenAI handler)+ if response.status_code >= 400:+ try:+ err_body = response.json()+ err_msg = err_body.get("error", {}).get("message", "")+ err_type = err_body.get("error", {}).get("type", "")+ except Exception:+ err_body = {"raw": response.text[:2000]}+ err_msg = str(response.text[:500])+ err_type = "parse_error"++ logger.warning(+ f"[{request_id}] UPSTREAM_ERROR "+ f"status={response.status_code} "+ f"error_type={err_type} "+ f"error_msg={err_msg!r} "+ f"model={model} "+ f"compressed={'yes' if transforms_applied else 'no'} "+ f"transforms={transforms_applied} "+ f"original_tokens={original_tokens} "+ f"optimized_tokens={optimized_tokens} "+ f"message_count={len(body.get('messages', []))} "+ f"stream={stream}"+ )++ try:+ from headroom import paths as _hr_paths++ debug_dir = _hr_paths.debug_400_dir()+ debug_dir.mkdir(parents=True, exist_ok=True)+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")+ debug_file = debug_dir / f"{ts}_{request_id}.json"++ safe_headers = {}+ for k, v in headers.items():+ if k.lower() in ("x-api-key", "authorization"):+ safe_headers[k] = v[:12] + "..." if v else ""+ else:+ safe_headers[k] = v++ debug_payload = {+ "request_id": request_id,+ "timestamp": datetime.now().isoformat(),+ "status_code": response.status_code,+ "error_response": err_body,+ "model": model,+ "stream": stream,+ "headers": safe_headers,+ "compression": {+ "was_compressed": bool(transforms_applied),+ "transforms": transforms_applied,+ "original_tokens": original_tokens,+ "optimized_tokens": optimized_tokens,+ "tokens_saved": tokens_saved,+ "compression_failed": _compression_failed,+ },+ "tools_sent": body.get("tools"),+ "tool_count": len(body.get("tools") or []),+ "original_tool_count": len(_original_tools or []),+ "messages_sent": body.get("messages"),+ "message_count": len(body.get("messages", [])),+ "original_messages": (+ original_messages+ if original_messages is not body.get("messages")+ else "__same_as_sent__"+ ),+ "original_message_count": len(original_messages),+ "system_prompt": body.get("system"),+ }++ with open(debug_file, "w") as f:+ json.dump(debug_payload, f, indent=2, default=str)++ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")+ except Exception as dump_err:+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")++ total_latency = (time.time() - start_time) * 1000++ total_input_tokens = optimized_tokens # fallback+ output_tokens = 0+ cache_read_tokens = 0+ resp_json = None+ try:+ resp_json = response.json()+ usage = resp_json.get("usage", {})+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)+ output_tokens = usage.get("completion_tokens", 0)+ # OpenAI returns cached_tokens in prompt_tokens_details+ # These are charged at 50% of the input price+ prompt_details = usage.get("prompt_tokens_details") or {}+ cache_read_tokens = prompt_details.get("cached_tokens", 0)+ except (KeyError, TypeError, AttributeError) as e:+ logger.debug(+ f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"+ )++ # Update prefix cache tracker for next turn+ openai_prefix_tracker.update_from_response(+ cache_read_tokens=cache_read_tokens,+ cache_write_tokens=0, # OpenAI doesn't report write tokens+ messages=optimized_messages,+ )++ # OpenAI has no write penalty — uncached = total - cached+ uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)++ if self.cost_tracker:+ self.cost_tracker.record_tokens(+ model,+ tokens_saved,+ optimized_tokens,+ cache_read_tokens=cache_read_tokens,+ uncached_tokens=uncached_input_tokens,+ )++ # Memory: handle memory tool calls in OpenAI Chat Completions response.+ # After executing tools, send a continuation request so the model+ # can produce a final user-facing response (not just tool_calls).+ if (+ self.memory_handler+ and memory_user_id+ and resp_json+ and response.status_code == 200+ and self.memory_handler.has_memory_tool_calls(resp_json, "openai")+ ):+ try:+ tool_results = await self.memory_handler.handle_memory_tool_calls(+ resp_json, memory_user_id, "openai"+ )+ if tool_results:+ # Build continuation: original messages + assistant tool_calls + tool results+ assistant_msg = resp_json.get("choices", [{}])[0].get("message", {})+ continuation_messages = list(optimized_messages)+ continuation_messages.append(assistant_msg)+ continuation_messages.extend(tool_results)++ continuation_body = {+ **body,+ "messages": continuation_messages,+ }++ cont_response = await self._retry_request(+ "POST", url, headers, continuation_body+ )+ if cont_response.status_code == 200:+ resp_json = cont_response.json()+ response = cont_response++ logger.info(+ f"[{request_id}] Memory: Handled {len(tool_results)} "+ f"tool call(s) with continuation for user {memory_user_id}"+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Memory tool handling failed: {e}")++ # Cache+ if self.cache and response.status_code == 200:+ await self.cache.set(+ messages,+ model,+ response.content,+ dict(response.headers),+ tokens_saved,+ )++ # Capture Codex rate-limit window data from response headers+ from headroom.subscription.codex_rate_limits import (+ get_codex_rate_limit_state,+ )++ get_codex_rate_limit_state().update_from_headers(dict(response.headers))++ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ waste_signals=waste_signals_dict,+ cache_read_tokens=cache_read_tokens,+ uncached_input_tokens=uncached_input_tokens,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens)"+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ # Inject Headroom compression metrics (for SaaS metering)+ response_headers["x-headroom-tokens-before"] = str(original_tokens)+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)+ response_headers["x-headroom-model"] = model+ if transforms_applied:+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)+ if cache_read_tokens > 0:+ response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed(provider="openai")+ # Log full error details internally for debugging+ logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")+ # Return sanitized error message to client (don't expose internal details)+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "type": "server_error",+ "code": "proxy_error",+ }+ },+ )++ async def handle_openai_responses(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle OpenAI /v1/responses endpoint (new Responses API).++ The Responses API differs from /v1/chat/completions:+ - Input: `input` (string or array) instead of `messages`+ - System: `instructions` instead of system message+ - Output: `output[]` array instead of `choices[].message`+ - State: `previous_response_id` for multi-turn+ - Built-in tools: web_search, file_search, code_interpreter+ """+ from fastapi import HTTPException+ from fastapi.responses import JSONResponse, Response++ from headroom.proxy.helpers import (+ COMPRESSION_TIMEOUT_SECONDS,+ MAX_REQUEST_BODY_SIZE,+ _read_request_json,+ )+ from headroom.proxy.responses_converter import (+ messages_to_responses_items,+ responses_items_to_messages,+ )+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "type": "invalid_request_error",+ "code": "request_too_large",+ }+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )++ model = body.get("model", "unknown")+ stream = body.get("stream", False)++ # Convert Responses API input to messages format for optimization.+ # The Responses API uses a different item model (function_call,+ # function_call_output, reasoning as top-level items) — we convert to+ # Chat Completions messages for the pipeline, then convert back.++ input_data = body.get("input", "")+ instructions = body.get("instructions")+ previous_response_id = body.get("previous_response_id")++ messages: list[dict[str, Any]] = []+ original_items: list[dict[str, Any]] | None = None+ preserved_indices: list[int] = []++ if instructions:+ messages.append({"role": "system", "content": instructions})++ if isinstance(input_data, str):+ messages.append({"role": "user", "content": input_data})+ elif isinstance(input_data, list):+ original_items = input_data+ converted, preserved_indices = responses_items_to_messages(input_data)+ messages.extend(converted)++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ # Strip accept-encoding so httpx negotiates its own encoding.+ # Cloudflare Workers forward "br, zstd" which OpenAI may honor;+ # if httpx lacks brotli support the response body is undecipherable → 502.+ headers.pop("accept-encoding", None)+ tags = self._extract_tags(headers)++ # Memory: Get user ID when memory is enabled+ memory_user_id: str | None = None+ if self.memory_handler:+ memory_user_id = headers.get(+ "x-headroom-user-id",+ os.environ.get("USER", os.environ.get("USERNAME", "default")),+ )++ # Rate limiting+ if self.rate_limiter:+ rate_key = headers.get("authorization", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited(provider="openai")+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Token counting on converted messages+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Optimize: convert items → compress → convert back+ tokens_saved = 0+ transforms_applied: list[str] = []+ optimized_messages = messages+ optimized_tokens = original_tokens++ _bypass = (+ request.headers.get("x-headroom-bypass", "").lower() == "true"+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"+ )+ _should_compress = (+ self.config.optimize+ and original_items is not None+ and not previous_response_id+ and not _bypass+ and len(messages) > 1+ )+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True++ if _should_compress and _license_ok:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )+ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ except Exception as e:+ logger.warning(f"[{request_id}] Responses API optimization failed: {e}")++ # Guard: if "optimization" inflated tokens, revert to originals+ if optimized_tokens > original_tokens:+ logger.warning(+ f"[{request_id}] Optimization inflated tokens "+ f"({original_tokens} -> {optimized_tokens}), reverting to original messages"+ )+ optimized_messages = messages+ optimized_tokens = original_tokens+ transforms_applied = []++ tokens_saved = original_tokens - optimized_tokens+ optimization_latency = (time.time() - start_time) * 1000++ # Convert compressed messages back to Responses API items+ if optimized_messages is not messages and original_items is not None:+ opt_msgs = optimized_messages+ # Strip system message (instructions) — it's separate in Responses API+ if instructions and opt_msgs and opt_msgs[0].get("role") == "system":+ body["instructions"] = opt_msgs[0]["content"]+ opt_msgs = opt_msgs[1:]++ body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)++ # Memory: inject context and tools for Responses API requests+ if self.memory_handler and memory_user_id:+ try:+ # Inject memory context into instructions (Responses API system prompt)+ if self.memory_handler.config.inject_context:+ try:+ memory_context = await asyncio.wait_for(+ self.memory_handler.search_and_format_context(+ memory_user_id, optimized_messages+ ),+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,+ )+ except asyncio.TimeoutError:+ memory_context = None+ logger.info(+ f"[{request_id}] Memory context lookup exceeded "+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"+ )+ if memory_context:+ existing_instructions = body.get("instructions") or ""+ if existing_instructions:+ body["instructions"] = f"{existing_instructions}\n\n{memory_context}"+ else:+ body["instructions"] = memory_context+ logger.info(+ f"[{request_id}] Memory: Injected {len(memory_context)} chars "+ f"of context into instructions for user {memory_user_id}"+ )++ # Inject memory tools (Responses API format)+ if self.memory_handler.config.inject_tools:+ resp_tools = body.get("tools") or []+ resp_tools, mem_tools_injected = self.memory_handler.inject_tools(+ resp_tools, "openai"+ )+ if mem_tools_injected:+ # Convert Chat Completions format to Responses API format+ converted_tools = []+ for t in resp_tools:+ if t.get("type") == "function" and "function" in t:+ fn = t["function"]+ converted_tools.append(+ {+ "type": "function",+ "name": fn.get("name"),+ "description": fn.get("description", ""),+ "parameters": fn.get("parameters", {}),+ }+ )+ else:+ converted_tools.append(t)+ body["tools"] = converted_tools+ logger.info(+ f"[{request_id}] Memory: Injected memory tools (openai/responses)"+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Memory injection failed (responses): {e}")++ # /v1/responses is OpenAI-specific (Codex) — always routes direct.+ # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.+ if self.anthropic_backend is not None:+ logger.debug(+ f"[{request_id}] /v1/responses always routes to OpenAI direct "+ f"(backend '{self.anthropic_backend.name}' not used for Responses API)"+ )++ headers, is_chatgpt_auth = _resolve_codex_routing_headers(headers)++ # Route to correct endpoint based on auth mode.+ # ChatGPT session auth (codex login) uses chatgpt.com, not api.openai.com.+ if is_chatgpt_auth:+ url = "https://chatgpt.com/backend-api/codex/responses"+ else:+ url = f"{self.OPENAI_API_URL}/v1/responses"++ try:+ if stream:+ # Streaming for Responses API uses semantic events+ return await self._stream_response(+ url,+ headers,+ body,+ "openai",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ memory_user_id=memory_user_id,+ )+ else:+ headers = await apply_copilot_api_auth(headers, url=url)+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ total_input_tokens = original_tokens # fallback+ output_tokens = 0+ try:+ resp_json = response.json()+ usage = resp_json.get("usage", {})+ total_input_tokens = usage.get("input_tokens", original_tokens)+ output_tokens = usage.get("output_tokens", 0)+ except (KeyError, TypeError, AttributeError) as e:+ logger.debug(+ f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"+ )++ # Memory: handle memory tool calls in Responses API response+ if (+ self.memory_handler+ and memory_user_id+ and resp_json+ and response.status_code == 200+ and self.memory_handler.has_memory_tool_calls(resp_json, "openai")+ ):+ try:+ # Extract function_call items from output+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES++ output_items = resp_json.get("output", [])+ memory_fc_items = [+ item+ for item in output_items+ if isinstance(item, dict)+ and item.get("type") == "function_call"+ and item.get("name") in MEMORY_TOOL_NAMES+ ]++ # Execute memory tool calls+ tool_outputs: list[dict[str, Any]] = []+ for fc in memory_fc_items:+ call_id = fc.get("call_id", fc.get("id", ""))+ name = fc.get("name", "")+ args_str = fc.get("arguments", "{}")+ try:+ args = json.loads(args_str)+ except json.JSONDecodeError:+ args = {}++ await self.memory_handler._ensure_initialized()+ if self.memory_handler._backend:+ result = await self.memory_handler._execute_memory_tool(+ name, args, memory_user_id, "openai"+ )+ else:+ result = json.dumps({"error": "Memory backend not initialized"})++ tool_outputs.append(+ {+ "type": "function_call_output",+ "call_id": call_id,+ "output": result,+ }+ )++ if tool_outputs:+ # Make continuation request with tool results+ response_id = resp_json.get("id")+ continuation_body = {+ "model": model,+ "input": tool_outputs,+ }+ if response_id:+ continuation_body["previous_response_id"] = response_id+ existing_tools = body.get("tools")+ if existing_tools:+ continuation_body["tools"] = existing_tools++ cont_response = await self._retry_request(+ "POST", url, headers, continuation_body+ )+ resp_json = cont_response.json()+ response = cont_response+ logger.info(+ f"[{request_id}] Memory: Handled {len(tool_outputs)} "+ f"tool call(s) with continuation for user {memory_user_id} (responses)"+ )+ except Exception as e:+ logger.warning(+ f"[{request_id}] Memory tool handling failed (responses): {e}"+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)++ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ )++ logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")++ # Capture Codex rate-limit window data from response headers+ from headroom.subscription.codex_rate_limits import (+ get_codex_rate_limit_state,+ )++ get_codex_rate_limit_state().update_from_headers(dict(response.headers))++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed(provider="openai")+ logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "type": "server_error",+ "code": "proxy_error",+ }+ },+ )++ async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:+ """WebSocket proxy for /v1/responses (Codex gpt-5.4+).++ Newer Codex versions use WebSocket instead of HTTP POST for the+ Responses API. This handler:+ 1. Accepts the client WebSocket+ 2. Receives the first message (``response.create`` request)+ 3. Compresses the ``input`` array using the existing pipeline+ 4. Opens an upstream WebSocket to OpenAI+ 5. Sends the compressed request upstream+ 6. Relays all subsequent messages bidirectionally+ """+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ try:+ import websockets+ except ImportError:+ await websocket.accept()+ await websocket.close(+ code=1011,+ reason="websockets package not installed. pip install websockets",+ )+ return++ request_id = await self._next_request_id()+ session_id = uuid.uuid4().hex++ # Stage-timer — captures per-stage durations for the structured+ # log emitted on session close. Unit 2 instrumentation.+ stage_timer = StageTimer()+ session_started_at = time.perf_counter()++ # Unit 3: initialize registry variables *before* accept so the+ # outermost ``finally`` can rely on them existing even if+ # registration itself fails for some reason.+ ws_sessions: WebSocketSessionRegistry | None = getattr(self, "ws_sessions", None)+ session_handle: WSSessionHandle | None = None+ termination_cause: TerminationCause = "unknown"++ # Forward client headers to upstream, adding required OpenAI-Beta header+ ws_headers = dict(websocket.headers)++ # Extract subprotocol from client — this is an application-level negotiation+ # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).+ # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.+ client_subprotocols: list[str] = []+ raw_protocol = ws_headers.get("sec-websocket-protocol", "")+ if raw_protocol:+ client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]++ # Accept client connection with the requested subprotocol+ async with stage_timer.measure("accept"):+ if client_subprotocols:+ await websocket.accept(subprotocol=client_subprotocols[0])+ else:+ await websocket.accept()++ # --- Unit 3: register the session as soon as accept succeeds ---+ client_addr: str | None = None+ client_info = getattr(websocket, "client", None)+ if client_info is not None:+ host = getattr(client_info, "host", None)+ port = getattr(client_info, "port", None)+ if host is not None and port is not None:+ client_addr = f"{host}:{port}"+ elif host is not None:+ client_addr = str(host)+ if ws_sessions is not None:+ session_handle = WSSessionHandle(+ session_id=session_id,+ request_id=request_id,+ client_addr=client_addr,+ upstream_url=None, # set below once upstream_url is computed+ )+ ws_sessions.register(session_handle)+ metrics = getattr(self, "metrics", None)+ if metrics is not None and hasattr(metrics, "inc_active_ws_sessions"):+ try:+ metrics.inc_active_ws_sessions()+ except Exception: # pragma: no cover - defensive+ pass++ # Forward all client headers except hop-by-hop / per-connection headers.+ # These are WebSocket handshake mechanics that the `websockets` library+ # generates fresh for the upstream connection — forwarding them would conflict.+ # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.+ _skip_headers = frozenset(+ {+ "host", # must match upstream, not local proxy+ "connection", # hop-by-hop+ "upgrade", # hop-by-hop+ "sec-websocket-key", # per-connection cryptographic nonce+ "sec-websocket-version", # protocol version (websockets lib sets this)+ "sec-websocket-extensions", # per-connection negotiation+ "sec-websocket-accept", # server-side only+ "sec-websocket-protocol", # handled via subprotocols param below+ "content-length", # hop-by-hop+ "transfer-encoding", # hop-by-hop+ }+ )+ upstream_headers: dict[str, str] = {}+ for k, v in ws_headers.items():+ if k.lower() not in _skip_headers:+ upstream_headers[k] = v++ upstream_headers, is_chatgpt_auth = _resolve_codex_routing_headers(upstream_headers)+ _lower_headers = {k.lower(): v for k, v in upstream_headers.items()}++ # Build upstream WebSocket URL based on auth mode+ if is_chatgpt_auth:+ # ChatGPT session auth → route to chatgpt.com backend+ upstream_url = "wss://chatgpt.com/backend-api/codex/responses"+ logger.debug(+ f"[{request_id}] WS: ChatGPT session auth detected, routing to chatgpt.com"+ )+ else:+ # API key auth → route to configured OpenAI API URL+ base = self.OPENAI_API_URL+ ws_base = base.replace("https://", "wss://").replace("http://", "ws://")+ upstream_url = f"{ws_base}/v1/responses"++ # Unit 3: attach the resolved upstream URL to the session handle.+ if session_handle is not None:+ session_handle.upstream_url = upstream_url++ # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.+ # Safety net for clients that don't forward auth headers via WebSocket upgrade.+ if "authorization" not in _lower_headers:+ api_key = os.environ.get("OPENAI_API_KEY")+ if api_key:+ upstream_headers["Authorization"] = f"Bearer {api_key}"+ logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")+ else:+ logger.warning(+ f"[{request_id}] WS: no Authorization header from client and "+ f"OPENAI_API_KEY not set — upstream will likely reject"+ )++ upstream_headers = await apply_copilot_api_auth(upstream_headers, url=upstream_url)++ # Ensure the required beta header is present — OpenAI returns 500 without it.+ # Codex sends `responses_websockets=2026-02-06`; only inject if missing entirely.+ if "openai-beta" not in _lower_headers:+ upstream_headers["OpenAI-Beta"] = "responses_websockets=2026-02-06"++ logger.debug(+ f"[{request_id}] WS upstream headers: "+ f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "+ f"subprotocols={client_subprotocols}"+ )++ try:+ # Receive the first message from client (the response.create request).+ # Bound the wait with WS_FIRST_FRAME_TIMEOUT_SECONDS so a zombie+ # client that opens the WS but never sends a frame cannot hold a+ # session slot indefinitely. The StageTimer measurement still+ # captures the elapsed time up to the timeout so operators can+ # see the slow-client pattern in the stage-timings log.+ try:+ async with stage_timer.measure("first_client_frame"):+ first_msg_raw = await asyncio.wait_for(+ websocket.receive_text(),+ timeout=WS_FIRST_FRAME_TIMEOUT_SECONDS,+ )+ except asyncio.TimeoutError:+ logger.info(+ f"[{request_id}] WS first-frame timeout after "+ f"{WS_FIRST_FRAME_TIMEOUT_SECONDS:.0f}s; closing session "+ f"{session_id} (no client data)"+ )+ termination_cause = "client_timeout"+ with contextlib.suppress(Exception):+ # 1001 (going away): server is cleanly terminating a slow+ # client, not an internal error.+ await websocket.close(code=1001, reason="first-frame timeout")+ # Exit the outer try so the session-lifecycle ``finally`` runs+ # deregister / metrics / stage-timings emission as usual.+ return++ # --- Optional: compress the input in the first message ---+ body: dict[str, Any] = {}+ try:+ body = json.loads(first_msg_raw)+ tokens_saved = 0+ ws_request_body = body.get("response", body)+ input_data = (+ ws_request_body.get("input") if isinstance(ws_request_body, dict) else None+ )++ should_compress = (+ self.config.optimize+ and isinstance(input_data, list)+ and len(input_data) > 1+ and not (+ ws_request_body.get("previous_response_id")+ if isinstance(ws_request_body, dict)+ else None+ )+ )+ if should_compress:+ try:+ from headroom.proxy.responses_converter import (+ messages_to_responses_items,+ responses_items_to_messages,+ )++ model = ws_request_body.get("model", "gpt-4o")+ converted, preserved = responses_items_to_messages(input_data)++ messages: list[dict[str, Any]] = []+ instructions = ws_request_body.get("instructions")+ if instructions:+ messages.append({"role": "system", "content": instructions})+ messages.extend(converted)++ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ context_limit = self.openai_provider.get_context_limit(model)+ async with stage_timer.measure("compression"):+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != messages:+ opt = result.messages+ if instructions and opt and opt[0].get("role") == "system":+ ws_request_body["instructions"] = opt[0]["content"]+ opt = opt[1:]+ if result.tokens_after <= original_tokens:+ ws_request_body["input"] = messages_to_responses_items(+ opt, input_data, preserved+ )+ else:+ logger.warning(+ f"[{request_id}] WS optimization inflated tokens "+ f"({original_tokens} -> {result.tokens_after}), reverting"+ )+ tokens_saved = max(0, original_tokens - result.tokens_after)+ if "response" in body and isinstance(body["response"], dict):+ body["response"] = ws_request_body+ else:+ body = ws_request_body+ first_msg_raw = json.dumps(body)+ logger.info(+ f"[{request_id}] WS /v1/responses compressed: "+ f"saved {tokens_saved} tokens"+ )+ except Exception as e:+ logger.warning(f"[{request_id}] WS compression failed: {e}")++ except json.JSONDecodeError:+ # Not JSON — pass through as-is+ tokens_saved = 0++ # --- Memory: inject context, tools, and instructions ---+ memory_user_id: str | None = None+ if self.memory_handler and body:+ memory_user_id = ws_headers.get(+ "x-headroom-user-id",+ os.environ.get("USER", os.environ.get("USERNAME", "default")),+ )+ try:+ # Unwrap response.create envelope to access the response body+ ws_response_body = body.get("response", body)++ # Debug: log what Codex sends so we can see the full tool list+ existing_tool_names = [+ t.get("name") or t.get("function", {}).get("name", "?")+ for t in (ws_response_body.get("tools") or [])+ ]+ instr_preview = (ws_response_body.get("instructions") or "")[:200]+ logger.info(+ f"[{request_id}] WS Memory: Codex tools={existing_tool_names}, "+ f"instructions_len={len(ws_response_body.get('instructions') or '')}, "+ f"instructions_preview={instr_preview!r}"+ )++ # Inject memory context into instructions+ if self.memory_handler.config.inject_context:+ ws_input = ws_response_body.get("input", "")+ ws_instructions = ws_response_body.get("instructions")+ ws_msgs: list[dict[str, Any]] = []+ if ws_instructions:+ ws_msgs.append({"role": "system", "content": ws_instructions})+ if isinstance(ws_input, str) and ws_input:+ ws_msgs.append({"role": "user", "content": ws_input})+ elif isinstance(ws_input, list):+ from headroom.proxy.responses_converter import (+ responses_items_to_messages,+ )++ converted_msgs, _ = responses_items_to_messages(ws_input)+ ws_msgs.extend(converted_msgs)++ try:+ async with stage_timer.measure("memory_context"):+ memory_context = await asyncio.wait_for(+ self.memory_handler.search_and_format_context(+ memory_user_id, ws_msgs+ ),+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,+ )+ except asyncio.TimeoutError:+ memory_context = None+ logger.info(+ f"[{request_id}] WS Memory: Context lookup exceeded "+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "+ f"continuing without it"+ )+ if memory_context:+ existing = ws_response_body.get("instructions") or ""+ if existing:+ ws_response_body["instructions"] = f"{existing}\n\n{memory_context}"+ else:+ ws_response_body["instructions"] = memory_context+ logger.info(+ f"[{request_id}] WS Memory: Injected {len(memory_context)} chars "+ f"of context into instructions"+ )++ # Inject memory tools (Responses API format)+ if self.memory_handler.config.inject_tools:+ ws_tools = ws_response_body.get("tools") or []+ ws_tools, mem_injected = self.memory_handler.inject_tools(+ ws_tools, "openai"+ )+ if mem_injected:+ converted_tools = []+ for t in ws_tools:+ if t.get("type") == "function" and "function" in t:+ fn = t["function"]+ converted_tools.append(+ {+ "type": "function",+ "name": fn.get("name"),+ "description": fn.get("description", ""),+ "parameters": fn.get("parameters", {}),+ }+ )+ else:+ converted_tools.append(t)+ ws_response_body["tools"] = converted_tools++ # Add memory instruction so the model uses+ # memory tools as persistent cross-session knowledge.+ mem_instruction = (+ "\n\n## Memory\n"+ "You have persistent memory via memory_search and "+ "memory_save tools. Memory stores knowledge across "+ "sessions — user info, project details, org context, "+ "decisions, architecture, conventions, anything worth "+ "remembering.\n\n"+ "- ALWAYS call memory_search BEFORE searching files "+ "when the user asks a question that could be answered "+ "from prior knowledge.\n"+ "- Call memory_save to store important facts, decisions, "+ "or context that would be useful in future sessions.\n"+ "- Memory is your first source of truth for anything "+ "not visible in the current conversation."+ )+ existing_instr = ws_response_body.get("instructions") or ""+ ws_response_body["instructions"] = existing_instr + mem_instruction+ logger.info(+ f"[{request_id}] WS Memory: Injected memory tools + instruction"+ )++ # Write back into envelope if it was wrapped+ if "response" in body and isinstance(body["response"], dict):+ body["response"] = ws_response_body+ else:+ body = ws_response_body++ first_msg_raw = json.dumps(body)+ except Exception as e:+ logger.warning(f"[{request_id}] WS Memory injection failed: {e}")++ # --- Connect to upstream OpenAI WebSocket ---+ logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")++ # Use ssl=True to let the websockets library handle SSL natively.+ # Manual ssl.create_default_context() + certifi doesn't load the+ # Windows system cert store, causing HTTP 500 on wss:// connections.+ use_ssl: bool | None = True if upstream_url.startswith("wss://") else None++ ws_connected = False+ ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))+ ws_last_err: Exception | None = None+ _upstream_connect_started = time.perf_counter()+ _upstream_connect_recorded = False+ _upstream_first_event_started: float | None = None++ for ws_attempt in range(ws_connect_attempts):+ try:+ async with websockets.connect(+ upstream_url,+ additional_headers=upstream_headers,+ subprotocols=(+ [websockets.Subprotocol(p) for p in client_subprotocols]+ if client_subprotocols and hasattr(websockets, "Subprotocol")+ else client_subprotocols or None+ ),+ ssl=use_ssl,+ open_timeout=max(30, self.config.connect_timeout_seconds * 3),+ close_timeout=10,+ ping_interval=20,+ ping_timeout=20,+ ) as upstream:+ ws_connected = True+ if not _upstream_connect_recorded:+ stage_timer.record(+ "upstream_connect",+ (time.perf_counter() - _upstream_connect_started) * 1000.0,+ )+ _upstream_connect_recorded = True+ _upstream_first_event_started = time.perf_counter()+ await upstream.send(first_msg_raw)++ # Unit 3: flag the upstream side flips on seeing+ # ``response.completed`` so the outer cause+ # classifier can prefer it over the raw+ # "upstream iterator ended" default.+ response_completed_seen = False+ # Captures the first exception surfaced by the+ # inner relay ``except`` blocks so the outer+ # classifier can still tell ``upstream_error``+ # from ``upstream_disconnect`` / ``response_completed``+ # even though the halves swallow and log.+ upstream_relay_error: BaseException | None = None+ client_relay_error: BaseException | None = None++ async def _client_to_upstream() -> None:+ nonlocal client_relay_error+ try:+ while True:+ msg = await websocket.receive_text()+ await upstream.send(msg)+ except asyncio.CancelledError:+ # Explicit cancel from the outer+ # orchestrator — re-raise so+ # ``t.cancelled()`` and ``t.exception()``+ # behave correctly in the caller.+ raise+ except Exception as relay_err:+ # Surface real errors to the classifier+ # without re-raising (existing fork+ # behavior: log and return so the+ # partner task can be cancelled+ # deterministically).+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ client_relay_error = relay_err+ logger.debug(+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"+ )+ with contextlib.suppress(Exception):+ await upstream.close()++ async def _upstream_to_client() -> None:+ """Relay upstream→client with transparent memory tool handling.++ Uses a buffer-then-decide approach:+ 1. Buffer events until first output item arrives+ 2. If first output is a memory tool → suppress entire response,+ execute tools silently, send continuation upstream+ 3. If first output is non-memory → flush buffer, stream normally+ 4. Continuation response events are relayed to Codex seamlessly++ This prevents orphaned response.created events from confusing Codex.+ """+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES++ # Unit 3: surface response.completed observation+ # to the outer scope so the termination-cause+ # classifier can prefer ``response_completed``+ # over ``upstream_disconnect``.+ nonlocal response_completed_seen+ nonlocal upstream_relay_error++ memory_enabled = bool(self.memory_handler and memory_user_id)++ # Per-response state (reset after each response.completed)+ event_buffer: list[str] = []+ decided = False+ suppress_response = False+ pending_fcs: list[dict[str, Any]] = []+ resp_id: str | None = None++ def _reset() -> None:+ nonlocal decided, suppress_response, resp_id+ event_buffer.clear()+ decided = False+ suppress_response = False+ pending_fcs.clear()+ resp_id = None++ # The retry-loop variable is safe to close over here:+ # ``_upstream_to_client`` is defined and awaited within+ # a single iteration and never escapes.+ _first_event_started_at = _upstream_first_event_started # noqa: B023++ try:+ async for msg in upstream:+ if (+ _first_event_started_at is not None+ and "upstream_first_event" not in stage_timer+ ):+ stage_timer.record(+ "upstream_first_event",+ (time.perf_counter() - _first_event_started_at)+ * 1000.0,+ )+ if isinstance(msg, bytes):+ await websocket.send_bytes(msg)+ continue+ msg_str = msg if isinstance(msg, str) else str(msg)++ if not memory_enabled:+ await websocket.send_text(msg_str)+ continue++ # Parse event+ try:+ event = json.loads(msg_str)+ except (json.JSONDecodeError, TypeError):+ await websocket.send_text(msg_str)+ continue++ event_type = event.get("type", "")++ # --- Phase 1: Buffer until first output item ---+ if not decided:+ event_buffer.append(msg_str)++ if event_type == "response.output_item.added":+ item = event.get("item", {})+ if (+ item.get("type") == "function_call"+ and item.get("name") in MEMORY_TOOL_NAMES+ ):+ # Memory tool first → suppress entire response+ suppress_response = True+ decided = True+ event_buffer.clear()+ logger.info(+ f"[{request_id}] WS Memory: Detected "+ f"{item.get('name')} — suppressing response"+ )+ else:+ # Non-memory first → flush buffer, pass through+ decided = True+ for buf in event_buffer:+ await websocket.send_text(buf)+ event_buffer.clear()++ elif event_type == "response.completed":+ # No output items at all — flush+ decided = True+ for buf in event_buffer:+ await websocket.send_text(buf)+ event_buffer.clear()+ _reset()+ response_completed_seen = True++ continue++ # --- Phase 2a: Suppress mode (memory response) ---+ if suppress_response:+ if event_type == "response.output_item.done":+ item = event.get("item", {})+ if (+ item.get("type") == "function_call"+ and item.get("name") in MEMORY_TOOL_NAMES+ ):+ pending_fcs.append(item)++ elif event_type == "response.completed":+ response_completed_seen = True+ resp = event.get("response", {})+ resp_id = resp.get("id")++ if pending_fcs:+ logger.info(+ f"[{request_id}] WS Memory: Executing "+ f"{len(pending_fcs)} tool(s) transparently"+ )++ # Execute memory tool calls+ tool_outputs: list[dict[str, Any]] = []+ for fc in pending_fcs:+ call_id = fc.get("call_id", fc.get("id", ""))+ fc_name = fc.get("name", "")+ args_str = fc.get("arguments", "{}")+ try:+ fc_args = json.loads(args_str)+ except json.JSONDecodeError:+ fc_args = {}++ await self.memory_handler._ensure_initialized()+ if self.memory_handler._backend:+ result = await self.memory_handler._execute_memory_tool(+ fc_name,+ fc_args,+ memory_user_id,+ "openai",+ )+ else:+ result = json.dumps(+ {"error": "backend not ready"}+ )++ tool_outputs.append(+ {+ "type": "function_call_output",+ "call_id": call_id,+ "output": result,+ }+ )+ logger.info(+ f"[{request_id}] WS Memory: Executed "+ f"{fc_name} for user {memory_user_id}"+ )++ # Send continuation upstream+ cont: dict[str, Any] = {+ "type": "response.create",+ "response": {"input": tool_outputs},+ }+ if resp_id:+ cont["response"]["previous_response_id"] = (+ resp_id+ )+ await upstream.send(json.dumps(cont))+ logger.info(+ f"[{request_id}] WS Memory: Sent continuation "+ f"with {len(tool_outputs)} result(s)"+ )++ _reset()+ # All events suppressed in this mode+ continue++ # --- Phase 2b: Pass-through mode ---+ await websocket.send_text(msg_str)++ except asyncio.CancelledError:+ raise+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ # Capture for the outer classifier+ # so ``upstream_error`` can be+ # distinguished from a clean+ # upstream disconnect.+ upstream_relay_error = relay_err+ logger.debug(+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"+ )+ finally:+ with contextlib.suppress(Exception):+ await websocket.close()++ # --- Unit 3: deterministic relay-task cancellation ---+ # Spawn each half as a named task so we can:+ # (a) attach them to the session registry for+ # ``/debug/ws-sessions``,+ # (b) cancel the survivor explicitly when the+ # first one exits, and+ # (c) classify the termination cause for the+ # duration histogram.+ client_task = asyncio.create_task(+ _client_to_upstream(),+ name=f"codex-ws-c2u-{session_id}",+ )+ upstream_task = asyncio.create_task(+ _upstream_to_client(),+ name=f"codex-ws-u2c-{session_id}",+ )+ relay_tasks = [client_task, upstream_task]+ if ws_sessions is not None:+ ws_sessions.attach_tasks(session_id, relay_tasks)+ metrics_for_tasks = getattr(self, "metrics", None)+ if metrics_for_tasks is not None and hasattr(+ metrics_for_tasks, "inc_active_relay_tasks"+ ):+ try:+ metrics_for_tasks.inc_active_relay_tasks(len(relay_tasks))+ except Exception: # pragma: no cover - defensive+ pass++ try:+ done, pending = await asyncio.wait(+ {client_task, upstream_task},+ return_when=asyncio.FIRST_COMPLETED,+ )+ # Cancel the survivor so we don't leak the+ # partner task. Suppress the CancelledError+ # we just raised ourselves — any *other*+ # exception from the cancelled task is+ # already logged inside its own try/except.+ for t in pending:+ t.cancel()+ if pending:+ with contextlib.suppress(asyncio.CancelledError):+ await asyncio.gather(*pending, return_exceptions=True)++ # Classify termination cause from whichever+ # task completed first. ``CancelledError``+ # can show up on the "done" side if the+ # handler itself was cancelled from outside+ # (e.g. server shutdown).+ for t in done:+ exc = None+ # Cancelled tasks raise CancelledError from+ # .exception(); surface it explicitly so the+ # downstream ``isinstance(exc, CancelledError)``+ # branches actually run. For any other+ # unexpected state (``InvalidStateError`` if+ # the task somehow isn't done — shouldn't+ # happen post-gather but defensive), we+ # suppress and leave ``exc`` as ``None``.+ if t.cancelled():+ exc = asyncio.CancelledError()+ else:+ with contextlib.suppress(asyncio.InvalidStateError):+ exc = t.exception()+ task_name = t.get_name() or ""+ if t is client_task:+ if client_relay_error is not None:+ termination_cause = "client_error"+ elif exc is None:+ termination_cause = "client_disconnect"+ elif isinstance(exc, asyncio.CancelledError):+ termination_cause = "client_disconnect"+ else:+ # Distinguish legitimate client+ # disconnect exceptions from+ # real errors: WebSocketDisconnect+ # is a normal client exit.+ if "WebSocketDisconnect" in type(exc).__name__:+ termination_cause = "client_disconnect"+ else:+ termination_cause = "client_error"+ elif t is upstream_task:+ if upstream_relay_error is not None:+ termination_cause = "upstream_error"+ logger.debug(+ f"[{request_id}] WS relay {task_name} "+ f"raised: {upstream_relay_error!r}"+ )+ elif exc is None:+ termination_cause = (+ "response_completed"+ if response_completed_seen+ else "upstream_disconnect"+ )+ elif isinstance(exc, asyncio.CancelledError):+ termination_cause = "upstream_disconnect"+ else:+ termination_cause = "upstream_error"+ logger.debug(+ f"[{request_id}] WS relay {task_name} raised: {exc!r}"+ )+ finally:+ # In case anything above raised before the+ # cancel-and-await loop ran.+ for t in relay_tasks:+ if not t.done():+ t.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await asyncio.gather(*relay_tasks, return_exceptions=True)++ logger.info(+ f"[{request_id}] WS /v1/responses completed "+ f"(tokens_saved={tokens_saved}, cause={termination_cause})"+ )+ break+ except Exception as ws_err:+ if ws_connected:+ # WS was established but broke mid-stream — re-raise+ raise++ ws_last_err = ws_err+ if ws_attempt >= ws_connect_attempts - 1:+ break++ delay_with_jitter = jitter_delay_ms(+ self.config.retry_base_delay_ms,+ self.config.retry_max_delay_ms,+ ws_attempt,+ )+ logger.warning(+ f"[{request_id}] WS upstream connect failed "+ f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "+ f"retrying in {delay_with_jitter:.0f}ms"+ )+ await asyncio.sleep(delay_with_jitter / 1000)++ if not ws_connected:+ # WS upgrade failed (HTTP 500 from OpenAI is common).+ # Fall back to HTTP POST streaming and relay SSE events+ # back over the client WebSocket transparently.+ ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")+ _ws_detail = str(ws_err)+ if hasattr(ws_err, "response"):+ resp_body = getattr(getattr(ws_err, "response", None), "body", b"")+ if resp_body:+ _ws_detail += f" | {resp_body[:300].decode('utf-8', errors='replace')}"+ logger.warning(+ f"[{request_id}] WS upstream failed ({_ws_detail}), "+ f"falling back to HTTP POST streaming"+ )+ await self._ws_http_fallback(+ websocket, body, first_msg_raw, upstream_headers, request_id+ )++ # Record metrics+ if tokens_saved > 0:+ model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"+ await self.metrics.record_request(+ provider="openai",+ model=model_name,+ input_tokens=0,+ output_tokens=0,+ tokens_saved=tokens_saved,+ latency_ms=0,+ )++ except Exception as e:+ if "WebSocketDisconnect" in type(e).__name__:+ # Unit 3: client dropped the socket before or during+ # relay. The registry classifier may already have set+ # ``client_disconnect`` via the relay task exit path;+ # preserve that, otherwise set it here.+ if termination_cause == "unknown":+ termination_cause = "client_disconnect"+ else:+ # Extract response body from websockets InvalidStatus for better debugging+ error_detail = str(e)+ if hasattr(e, "response"):+ try:+ resp = e.response+ body_bytes = getattr(resp, "body", None) or b""+ if body_bytes:+ error_detail += (+ f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"+ )+ except Exception:+ pass+ logger.error(f"[{request_id}] WS proxy error: {error_detail}")+ if termination_cause == "unknown":+ termination_cause = "client_error"+ with contextlib.suppress(Exception):+ await websocket.close(code=1011, reason=str(e)[:120])+ finally:+ # Unit 2: emit structured per-session stage timings.+ stage_timer.record(+ "total_session",+ (time.perf_counter() - session_started_at) * 1000.0,+ )+ # Unit 3: deregister the session before (or independently+ # of) the stage-timings log so a failure there cannot leak+ # the registry entry. ``deregister`` is idempotent, so a+ # session that never registered is a no-op.+ if ws_sessions is not None and session_handle is not None:+ # Use deregister_and_count so the handle pop and the+ # relay-task count are read atomically inside the+ # registry. Capturing ``len(session_handle.relay_tasks)``+ # separately before ``deregister`` would risk drift if+ # the registry's bookkeeping ever changes.+ _deregistered, released_tasks = ws_sessions.deregister_and_count(+ session_id, cause=termination_cause+ )+ session_duration_ms = (time.perf_counter() - session_started_at) * 1000.0+ metrics_for_close = getattr(self, "metrics", None)+ if metrics_for_close is not None:+ with contextlib.suppress(Exception):+ if hasattr(metrics_for_close, "dec_active_ws_sessions"):+ metrics_for_close.dec_active_ws_sessions()+ if released_tasks and hasattr(metrics_for_close, "dec_active_relay_tasks"):+ metrics_for_close.dec_active_relay_tasks(released_tasks)+ if hasattr(metrics_for_close, "record_ws_session_duration"):+ metrics_for_close.record_ws_session_duration(+ session_duration_ms, termination_cause+ )+ await emit_stage_timings_log(+ path="openai_responses_ws",+ request_id=request_id,+ session_id=session_id,+ stage_timer=stage_timer,+ expected_stages=(+ "accept",+ "first_client_frame",+ "upstream_connect",+ "upstream_first_event",+ "memory_context",+ "compression",+ "total_session",+ ),+ metrics=getattr(self, "metrics", None),+ )++ async def _ws_http_fallback(+ self,+ websocket: WebSocket,+ body: dict[str, Any],+ first_msg_raw: str,+ upstream_headers: dict[str, str],+ request_id: str,+ ) -> None:+ """Fall back to HTTP POST streaming when upstream WS fails.++ Converts the WS ``response.create`` message to an HTTP POST to+ ``/v1/responses?stream=true``, reads SSE events, and relays each+ ``data:`` line as a WS text message to the client. This makes+ Codex work immediately instead of exhausting its WS retry budget.+ """+ # Route to correct endpoint based on auth mode+ _lower = {k.lower() for k in upstream_headers}+ if "chatgpt-account-id" in _lower:+ http_url = "https://chatgpt.com/backend-api/codex/responses"+ else:+ http_url = f"{self.OPENAI_API_URL}/v1/responses"++ # Build HTTP body from the WS response.create payload.+ # WS messages use {"type": "response.create", "response": {...}} wrapper.+ # The HTTP POST endpoint expects the inner response object directly.+ http_body: dict[str, Any]+ try:+ parsed = json.loads(first_msg_raw) if isinstance(first_msg_raw, str) else body+ except (json.JSONDecodeError, TypeError):+ parsed = body++ # Normalize WebSocket response.create payload into the HTTP request body.+ # Codex may send either:+ # 1. {"type":"response.create","response":{...}}+ # 2. {"type":"response.create", ...response fields...}+ if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):+ http_body = dict(parsed["response"])+ elif isinstance(parsed, dict):+ http_body = dict(parsed)+ if http_body.get("type") == "response.create":+ http_body.pop("type", None)+ else:+ http_body = body if isinstance(body, dict) else {}++ # Some clients include response-ish metadata that the HTTP endpoint rejects.+ if http_body.get("type") in {"response.create", "response"}:+ http_body.pop("type", None)++ # Ensure streaming is enabled so we get SSE events+ http_body["stream"] = True++ # Build HTTP headers from the upstream headers (already stripped of WS+ # hop-by-hop headers by the caller).+ http_headers = dict(upstream_headers)+ http_headers["content-type"] = "application/json"++ logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")++ try:+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))+ for http_attempt in range(retry_attempts):+ try:+ async with self.http_client.stream(+ "POST",+ http_url,+ headers=http_headers,+ json=http_body,+ timeout=120.0,+ ) as response:+ if response.status_code != 200:+ error_body = b""+ async for chunk in response.aiter_bytes():+ error_body += chunk+ if len(error_body) > 2000:+ break+ error_text = error_body.decode("utf-8", errors="replace")+ logger.error(+ f"[{request_id}] WS→HTTP fallback got {response.status_code}: "+ f"{error_text[:500]}"+ )+ # Send error as WS message so client sees it+ error_event = {+ "type": "error",+ "error": {+ "type": "server_error",+ "message": f"Upstream returned {response.status_code}",+ },+ }+ await websocket.send_text(json.dumps(error_event))+ return++ # Relay SSE events as WS text messages+ buffer = ""+ async for chunk in response.aiter_text():+ buffer += chunk+ while "\n" in buffer:+ line, buffer = buffer.split("\n", 1)+ line = line.strip()+ if not line:+ continue+ if line.startswith("data: "):+ data = line[6:]+ if data == "[DONE]":+ continue+ try:+ await websocket.send_text(data)+ except Exception:+ return+ elif line.startswith("event: "):+ # SSE event type — skip, the data line contains the type+ continue++ # Flush any remaining data in buffer+ for line in buffer.strip().splitlines():+ line = line.strip()+ if line.startswith("data: ") and line[6:] != "[DONE]":+ with contextlib.suppress(Exception):+ await websocket.send_text(line[6:])+ return+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:+ if http_attempt >= retry_attempts - 1:+ raise++ delay_with_jitter = jitter_delay_ms(+ self.config.retry_base_delay_ms,+ self.config.retry_max_delay_ms,+ http_attempt,+ )+ logger.warning(+ f"[{request_id}] WS→HTTP fallback connect failed "+ f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "+ f"retrying in {delay_with_jitter:.0f}ms"+ )+ await asyncio.sleep(delay_with_jitter / 1000)++ except Exception as http_err:+ logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")+ error_event = {+ "type": "error",+ "error": {+ "type": "server_error",+ "message": f"HTTP fallback failed: {http_err!s}"[:200],+ },+ }+ with contextlib.suppress(Exception):+ await websocket.send_text(json.dumps(error_event))+ finally:+ with contextlib.suppress(Exception):+ await websocket.close()++ async def handle_compress(self, request: Request) -> JSONResponse:+ """Compress messages without calling an LLM.++ POST /v1/compress+ Body: {"messages": [...], "model": "...", "config": {}}+ Returns compressed messages + metrics.+ """+ from fastapi.responses import JSONResponse++ from headroom.proxy.helpers import _read_request_json++ # Check bypass header+ if request.headers.get("x-headroom-bypass", "").lower() == "true":+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={"error": f"Invalid request body: {e!s}"},+ )+ messages = body.get("messages", [])+ return JSONResponse(+ {+ "messages": messages,+ "tokens_before": 0,+ "tokens_after": 0,+ "tokens_saved": 0,+ "compression_ratio": 1.0,+ "transforms_applied": [],+ "ccr_hashes": [],+ }+ )++ try:+ body = await _read_request_json(request)+ except Exception:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "type": "invalid_request",+ "message": "Invalid JSON in request body.",+ }+ },+ )++ messages = body.get("messages")+ model = body.get("model")++ if messages is None:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "type": "invalid_request",+ "message": "Missing required field: messages",+ }+ },+ )++ if model is None:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "type": "invalid_request",+ "message": "Missing required field: model",+ }+ },+ )++ if not messages:+ return JSONResponse(+ {+ "messages": [],+ "tokens_before": 0,+ "tokens_after": 0,+ "tokens_saved": 0,+ "compression_ratio": 1.0,+ "transforms_applied": [],+ "ccr_hashes": [],+ }+ )++ try:+ # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)+ # Allow optional token_budget to override model's context limit+ # (used by OpenClaw compact() and other callers that need tighter budgets)+ token_budget = body.get("token_budget")+ context_limit = (+ token_budget+ if token_budget and isinstance(token_budget, int)+ else self.openai_provider.get_context_limit(model)+ )+ # Extract CompressConfig options from request body+ compress_config = body.get("config", {})+ compress_user_messages = compress_config.get("compress_user_messages", False)+ target_ratio = compress_config.get("target_ratio")+ protect_recent = compress_config.get("protect_recent")+ protect_analysis_context = compress_config.get("protect_analysis_context")++ pipeline_kwargs: dict = {"model_limit": context_limit}+ if compress_user_messages:+ pipeline_kwargs["compress_user_messages"] = True+ if target_ratio is not None:+ pipeline_kwargs["target_ratio"] = float(target_ratio)+ if protect_recent is not None:+ pipeline_kwargs["protect_recent"] = int(protect_recent)+ if protect_analysis_context is not None:+ pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)++ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ **pipeline_kwargs,+ )++ return JSONResponse(+ {+ "messages": result.messages,+ "tokens_before": result.tokens_before,+ "tokens_after": result.tokens_after,+ "tokens_saved": result.tokens_before - result.tokens_after,+ "compression_ratio": (+ result.tokens_after / result.tokens_before+ if result.tokens_before > 0+ else 1.0+ ),+ "transforms_applied": result.transforms_applied,+ "transforms_summary": result.transforms_summary,+ "ccr_hashes": result.markers_inserted,+ }+ )+ except Exception as e:+ logger.exception("Compression failed: %s", e)+ return JSONResponse(+ status_code=503,+ content={+ "error": {+ "type": "compression_error",+ "message": str(e),+ }+ },+ )++ async def handle_passthrough(+ self,+ request: Request,+ base_url: str,+ endpoint_name: str | None = None,+ provider: str | None = None,+ ) -> Response:+ """Pass through request unchanged.++ Args:+ request: The incoming request+ base_url: The upstream API base URL+ endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")+ provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")+ """+ from fastapi.responses import Response++ start_time = time.time()+ path = request.url.path+ url = f"{base_url}{path}"++ # Preserve query string parameters+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("accept-encoding", None)++ body = await request.body()++ headers = await apply_copilot_api_auth(headers, url=url)+ response = await self.http_client.request( # type: ignore[union-attr]+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ # Track stats for passthrough requests+ if endpoint_name and provider:+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider=provider,+ model=f"passthrough:{endpoint_name}",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ cached=False,+ )++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def handle_databricks_invocations(+ self,+ request: Request,+ model: str,+ ) -> Response | StreamingResponse:+ """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.++ This enables using the Databricks CLI directly with Headroom:+ databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'++ The request/response format is identical to OpenAI chat completions,+ so we inject the model from the path and delegate to handle_openai_chat.+ """+ from fastapi.responses import JSONResponse++ from headroom.proxy.helpers import _read_request_json++ request_id = await self._next_request_id()++ try:+ body = await _read_request_json(request)+ except Exception as e:+ logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON: {e}",+ "type": "invalid_request_error",+ }+ },+ )++ # Inject model from path into body (Databricks CLI passes model in URL, not body)+ body["model"] = model++ logger.info(f"[{request_id}] Databricks invocation: model={model}")++ # Create a new request with the modified body+ # We reuse the OpenAI chat handler since the format is identical+ from starlette.requests import Request as StarletteRequest++ # Build new scope with the body already parsed+ scope = dict(request.scope)++ # Create a simple receive function that returns our modified body+ body_bytes = json.dumps(body).encode()++ async def receive():+ return {"type": "http.request", "body": body_bytes}++ modified_request = StarletteRequest(scope, receive)++ # Delegate to the OpenAI chat handler (same format)+ return await self.handle_openai_chat(modified_request)diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex cee782ecf..398210e8e 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1,2577 +1,2578 @@-"""Headroom Proxy Server - Production Ready.--A full-featured LLM proxy with optimization, caching, rate limiting,-and observability.--Features:-- Context optimization (SmartCrusher, CacheAligner, RollingWindow)-- Semantic caching (save costs on repeated queries)-- Rate limiting (token bucket)-- Retry with exponential backoff-- Cost tracking and budgets-- Request tagging and metadata-- Provider fallback-- Prometheus metrics-- Full request/response logging--Usage:- python -m headroom.proxy.server --port 8787-- # With Claude Code:- ANTHROPIC_BASE_URL=http://localhost:8787 claude-"""--from __future__ import annotations--import argparse-import asyncio-import json-import logging-import os-import sys-import time-from datetime import datetime, timezone-from pathlib import Path-from typing import TYPE_CHECKING, Any, Literal--if TYPE_CHECKING:- from ..backends.base import Backend- from ..cache.compression_cache import CompressionCache- from ..memory.tracker import MemoryTracker---import httpx--try:- import uvicorn- from fastapi import Depends, FastAPI, HTTPException, Request, Response- from fastapi.middleware.cors import CORSMiddleware- from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse-- FASTAPI_AVAILABLE = True-except ImportError:- FASTAPI_AVAILABLE = False--# Add parent to path for imports-sys.path.insert(0, str(Path(__file__).parent.parent.parent))--from headroom._version import __version__-from headroom.cache.compression_feedback import get_compression_feedback-from headroom.cache.compression_store import get_compression_store-from headroom.ccr import (- CCR_TOOL_NAME,- # Batch processing- CCRResponseHandler,- CCRToolInjector,- ContextTracker,- ContextTrackerConfig,- ResponseHandlerConfig,- parse_tool_call,-)-from headroom.config import (- CacheAlignerConfig,- CCRConfig,- IntelligentContextConfig,- ReadLifecycleConfig,- RollingWindowConfig,- SmartCrusherConfig,-)-from headroom.dashboard import get_dashboard_html-from headroom.observability import (- LangfuseTracingConfig,- OTelMetricsConfig,- configure_langfuse_tracing,- configure_otel_metrics,- get_langfuse_tracing_status,- get_otel_metrics_status,- shutdown_headroom_tracing,- shutdown_otel_metrics,-)-from headroom.pipeline import PipelineExtensionManager, PipelineStage-from headroom.providers.proxy_routes import register_provider_routes-from headroom.providers.registry import (- DEFAULT_ANTHROPIC_API_URL,- DEFAULT_CLOUDCODE_API_URL,- DEFAULT_GEMINI_API_URL,- DEFAULT_OPENAI_API_URL,- build_proxy_provider_runtime,- create_proxy_backend,- format_backend_status,-)--# =============================================================================-# Extracted modules (re-exported for backward compatibility)-# =============================================================================-from headroom.proxy.cost import (- _CACHE_ECONOMICS, # noqa: F401- CostTracker, # noqa: F401- _summarize_transforms, # noqa: F401- build_prefix_cache_stats, # noqa: F401- build_session_summary, # noqa: F401- merge_cost_stats, # noqa: F401-)-from headroom.proxy.helpers import (- COMPRESSION_TIMEOUT_SECONDS, # noqa: F401- MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401- MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401- MAX_REQUEST_BODY_SIZE, # noqa: F401- MAX_SSE_BUFFER_SIZE, # noqa: F401- _get_image_compressor, # noqa: F401- _get_rtk_stats, # noqa: F401- _read_request_json, # noqa: F401- _setup_file_logging, # noqa: F401- is_anthropic_auth, # noqa: F401- jitter_delay_ms,-)-from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler--# Data models (extracted to headroom/proxy/models.py for maintainability)-from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401-from headroom.proxy.modes import (- PROXY_MODE_CACHE,- PROXY_MODE_TOKEN,- is_token_mode,- normalize_proxy_mode,-)-from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401-from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401-from headroom.proxy.request_logger import RequestLogger # noqa: F401-from headroom.proxy.semantic_cache import SemanticCache # noqa: F401-from headroom.proxy.warmup import WarmupRegistry-from headroom.proxy.ws_session_registry import WebSocketSessionRegistry-from headroom.subscription.base import get_quota_registry, reset_quota_registry-from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state-from headroom.subscription.copilot_quota import get_copilot_quota_tracker-from headroom.subscription.tracker import (- configure_subscription_tracker,- get_subscription_tracker,-)-from headroom.telemetry import get_telemetry_collector-from headroom.telemetry.beacon import is_telemetry_enabled-from headroom.telemetry.toin import get_toin-from headroom.transforms import (- CacheAligner,- CodeAwareCompressor,- CodeCompressorConfig,- ContentRouter,- ContentRouterConfig,- IntelligentContextManager,- RollingWindow,- SmartCrusher,- Transform,- TransformPipeline,- is_tree_sitter_available,-)--AnyLLMBackend: Any = None-LiteLLMBackend: Any = None--fcntl: Any = None-try:- import fcntl as _fcntl-- fcntl = _fcntl- HAS_FCNTL = True-except ImportError:- HAS_FCNTL = False--_build_prefix_cache_stats = build_prefix_cache_stats-_build_session_summary = build_session_summary-_merge_cost_stats = merge_cost_stats---logging.basicConfig(- level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"-)-logger = logging.getLogger("headroom.proxy")--# Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis-_setup_file_logging()---# Compression pipeline timeout in seconds---from headroom.proxy.handlers import ( # noqa: E402- AnthropicHandlerMixin,- BatchHandlerMixin,- GeminiHandlerMixin,- OpenAIHandlerMixin,- StreamingMixin,-)---class HeadroomProxy(- StreamingMixin,- AnthropicHandlerMixin,- OpenAIHandlerMixin,- GeminiHandlerMixin,- BatchHandlerMixin,-):- """Production-ready Headroom optimization proxy."""-- ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL- OPENAI_API_URL = DEFAULT_OPENAI_API_URL- GEMINI_API_URL = DEFAULT_GEMINI_API_URL- CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL-- def __init__(self, config: ProxyConfig):- self.config = config- self.config.mode = normalize_proxy_mode(self.config.mode)- self.pipeline_extensions = PipelineExtensionManager(- hooks=config.hooks,- extensions=config.pipeline_extensions,- discover=config.discover_pipeline_extensions,- )-- self.provider_runtime = build_proxy_provider_runtime(config)- api_targets = self.provider_runtime.api_targets-- # Preserve the long-standing proxy compatibility surface while keeping- # provider_runtime as the source of truth for resolved upstream targets.- HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic- HeadroomProxy.OPENAI_API_URL = api_targets.openai- HeadroomProxy.GEMINI_API_URL = api_targets.gemini- HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode- self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")- self.openai_provider = self.provider_runtime.pipeline_provider("openai")-- # Initialize transforms based on routing mode- # Choose context manager: IntelligentContextManager (smart) or RollingWindow (legacy)- context_manager: Transform # Can be either IntelligentContextManager or RollingWindow- if config.intelligent_context:- # Get TOIN instance for learned pattern integration- toin = get_toin() if config.intelligent_context_scoring else None- context_manager = IntelligentContextManager(- config=IntelligentContextConfig(- enabled=True,- keep_system=True,- keep_last_turns=config.keep_last_turns,- use_importance_scoring=config.intelligent_context_scoring,- toin_integration=config.intelligent_context_scoring,- compress_threshold=0.10 if config.intelligent_context_compress_first else 0.0,- ),- toin=toin,- )- self._context_manager_status = "intelligent"- else:- context_manager = RollingWindow(- RollingWindowConfig(- enabled=True,- keep_system=True,- keep_last_turns=config.keep_last_turns,- )- )- self._context_manager_status = "rolling_window"-- if config.smart_routing:- # Smart routing: ContentRouter handles all content types intelligently- # It lazy-loads compressors only when needed- router_config = ContentRouterConfig(- enable_code_aware=config.code_aware_enabled,- tool_profiles=config.tool_profiles,- read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),- )- # Token mode: allow compression of older excluded-tool results- if is_token_mode(config.mode):- router_config.protect_recent_reads_fraction = 0.3- transforms = [- CacheAligner(CacheAlignerConfig(enabled=False)),- ContentRouter(router_config),- context_manager,- ]- self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"- else:- # Legacy mode: sequential pipeline- transforms = [- CacheAligner(CacheAlignerConfig(enabled=False)),- SmartCrusher(- SmartCrusherConfig( # type: ignore[arg-type]- enabled=True,- min_tokens_to_crush=config.min_tokens_to_crush,- max_items_after_crush=config.max_items_after_crush,- ),- ccr_config=CCRConfig(- enabled=config.ccr_inject_tool,- inject_retrieval_marker=config.ccr_inject_tool, # Add CCR markers- ),- ),- context_manager,- ]- # Add CodeAware if enabled and available- self._code_aware_status = self._setup_code_aware(config, transforms)-- self.anthropic_pipeline = TransformPipeline(- transforms=transforms,- provider=self.anthropic_provider,- )- self.openai_pipeline = TransformPipeline(- transforms=transforms,- provider=self.openai_provider,- )-- # Initialize components- self.cache = (- SemanticCache(- max_entries=config.cache_max_entries,- ttl_seconds=config.cache_ttl_seconds,- )- if config.cache_enabled- else None- )-- self.rate_limiter = (- TokenBucketRateLimiter(- requests_per_minute=config.rate_limit_requests_per_minute,- tokens_per_minute=config.rate_limit_tokens_per_minute,- )- if config.rate_limit_enabled- else None- )-- self.cost_tracker = (- CostTracker(- budget_limit_usd=config.budget_limit_usd,- budget_period=config.budget_period,- )- if config.cost_tracking_enabled- else None- )-- self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)-- # Prefix cache tracking: freeze already-cached messages to avoid- # invalidating the provider's prefix cache with our transforms- from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore-- self.session_tracker_store = SessionTrackerStore(- default_config=PrefixFreezeConfig(- enabled=config.prefix_freeze_enabled,- session_ttl_seconds=config.prefix_freeze_session_ttl,- )- )-- # Compression cache store for token mode (session-scoped)- self._compression_caches: dict[str, CompressionCache] = {}-- self.logger = (- RequestLogger(- log_file=config.log_file,- log_full_messages=config.log_full_messages,- )- if config.log_requests- else None- )-- # Enterprise security plugin (loaded dynamically if available + licensed)- self.security = None-- # HTTP client- self.http_client: httpx.AsyncClient | None = None-- # Shared cold-start warmup registry (populated by startup()).- # Holds typed slots with loaded / loading / null / error status for- # each preloaded heavy asset. Exposed as ``proxy.warmup`` and- # serialized by the /debug/warmup route (Unit 5).- self.warmup: WarmupRegistry = WarmupRegistry()- # Unit 3: live registry of Codex WS sessions. Populated by- # ``handle_openai_responses_ws`` on accept; drained in its- # outermost ``finally``. Consumed by ``/debug/ws-sessions``.- self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()-- # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP- # path. Caps how many ``handle_anthropic_messages`` calls may be- # running deep-copy / first-stage compression / memory-context- # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,- # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are- # intentionally NOT gated by this semaphore.- #- # A value of ``0`` or negative disables the semaphore (unbounded- # mode); this is useful for the Unit 6 counter-factual where we- # deliberately reproduce the original starvation. The default is- # ``max(2, min(8, os.cpu_count() or 4))``.- _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency- if _pre_upstream_cfg is None:- _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))- else:- _pre_upstream_resolved = _pre_upstream_cfg- self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved- self.anthropic_pre_upstream_acquire_timeout_seconds = float(- config.anthropic_pre_upstream_acquire_timeout_seconds- )- self.anthropic_pre_upstream_memory_context_timeout_seconds = float(- config.anthropic_pre_upstream_memory_context_timeout_seconds- )- if _pre_upstream_resolved > 0:- self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = asyncio.Semaphore(- _pre_upstream_resolved- )- else:- self.anthropic_pre_upstream_sem = None-- # Backend for Anthropic API (direct, LiteLLM, or any-llm)- # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"- self.anthropic_backend: Backend | None = create_proxy_backend(- backend=config.backend,- anyllm_provider=config.anyllm_provider,- bedrock_region=config.bedrock_region,- logger=logger,- anyllm_backend_cls=AnyLLMBackend,- litellm_backend_cls=LiteLLMBackend,- )-- # Request counter for IDs- self._request_counter = 0- self._request_counter_lock = asyncio.Lock()-- # CCR tool injectors (one per provider)- self.anthropic_tool_injector = CCRToolInjector(- provider="anthropic",- inject_tool=config.ccr_inject_tool,- inject_system_instructions=config.ccr_inject_system_instructions,- )- self.openai_tool_injector = CCRToolInjector(- provider="openai",- inject_tool=config.ccr_inject_tool,- inject_system_instructions=config.ccr_inject_system_instructions,- )-- # CCR Response Handler (handles CCR tool calls automatically)- self.ccr_response_handler = (- CCRResponseHandler(- ResponseHandlerConfig(- enabled=True,- max_retrieval_rounds=config.ccr_max_retrieval_rounds,- )- )- if config.ccr_handle_responses- else None- )-- # CCR Context Tracker (tracks compressed content across turns)- self.ccr_context_tracker = (- ContextTracker(- ContextTrackerConfig(- enabled=True,- proactive_expansion=config.ccr_proactive_expansion,- max_proactive_expansions=config.ccr_max_proactive_expansions,- )- )- if config.ccr_context_tracking- else None- )-- # Turn counter for context tracking- self._turn_counter = 0-- # Memory Handler (persistent user memory)- self.memory_handler: MemoryHandler | None = None- if config.memory_enabled:- # Resolve memory DB path: empty → project-scoped default- _mem_db_path = config.memory_db_path- if not _mem_db_path:- _mem_dir = Path.cwd() / ".headroom"- _mem_dir.mkdir(parents=True, exist_ok=True)- _mem_db_path = str(_mem_dir / "memory.db")- logger.info(f"Memory: Project-scoped DB at {_mem_db_path}")-- memory_config = MemoryConfig(- enabled=True,- backend=config.memory_backend,- db_path=_mem_db_path,- inject_tools=config.memory_inject_tools,- use_native_tool=config.memory_use_native_tool,- inject_context=config.memory_inject_context,- top_k=config.memory_top_k,- min_similarity=config.memory_min_similarity,- qdrant_host=config.memory_qdrant_host,- qdrant_port=config.memory_qdrant_port,- neo4j_uri=config.memory_neo4j_uri,- neo4j_user=config.memory_neo4j_user,- neo4j_password=config.memory_neo4j_password,- bridge_enabled=config.memory_bridge_enabled,- bridge_md_paths=config.memory_bridge_md_paths,- bridge_md_format=config.memory_bridge_md_format,- bridge_auto_import=config.memory_bridge_auto_import,- bridge_export_path=config.memory_bridge_export_path,- )- self.memory_handler = MemoryHandler(- memory_config,- agent_type=config.traffic_learning_agent_type,- )-- # Usage Reporter (license validation + phone-home for managed/enterprise)- self.usage_reporter: UsageReporter | None = None- if config.license_key:- from headroom.telemetry.reporter import UsageReporter-- self.usage_reporter = UsageReporter(- license_key=config.license_key,- cloud_url=config.license_cloud_url,- report_interval=config.license_report_interval,- )-- # Traffic Learner (live pattern extraction from proxy traffic)- # Only activates with --learn flag; requires --memory for backend- self.traffic_learner: TrafficLearner | None = None- self.traffic_learning_agent_type: str = config.traffic_learning_agent_type- if config.traffic_learning_enabled:- from headroom.memory.traffic_learner import TrafficLearner-- self.traffic_learner = TrafficLearner(- user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),- agent_type=config.traffic_learning_agent_type,- )-- # Code graph file watcher (live reindex on file changes)- self.code_graph_watcher: CodeGraphWatcher | None = None # type: ignore[annotation-unchecked]- if config.code_graph_watcher:- from headroom.graph.watcher import CodeGraphWatcher-- self.code_graph_watcher = CodeGraphWatcher(project_dir=Path.cwd())- if self.code_graph_watcher.start():- logger.info("Code graph: file watcher started")- else:- self.code_graph_watcher = None-- self.pipeline_extensions.emit(- PipelineStage.SETUP,- operation="proxy.setup",- metadata={- "mode": self.config.mode,- "optimize": self.config.optimize,- "backend": self.config.backend,- "memory_enabled": self.config.memory_enabled,- },- )-- def _get_compression_cache(self, session_id: str) -> CompressionCache:- """Get or create a CompressionCache for a session."""- if session_id not in self._compression_caches:- from headroom.cache.compression_cache import CompressionCache-- # Evict oldest caches if at capacity- if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:- # Remove oldest quarter to amortize cleanup cost- oldest_keys = list(self._compression_caches.keys())[- : MAX_COMPRESSION_CACHE_SESSIONS // 4- ]- for key in oldest_keys:- del self._compression_caches[key]- logger.info(- "Evicted %d compression caches (exceeded %d max sessions)",- len(oldest_keys),- MAX_COMPRESSION_CACHE_SESSIONS,- )-- self._compression_caches[session_id] = CompressionCache()- return self._compression_caches[session_id]-- def _setup_code_aware(self, config: ProxyConfig, transforms: list) -> str:- """Set up code-aware compression if enabled.-- Args:- config: Proxy configuration- transforms: Transform list to append to-- Returns:- Status string for logging: 'enabled', 'disabled', 'available', 'unavailable'- """- if config.code_aware_enabled:- if is_tree_sitter_available():- code_config = CodeCompressorConfig(- preserve_imports=True,- preserve_signatures=True,- preserve_type_annotations=True,- )- # Insert before RollingWindow (which should be last)- transforms.insert(-1, CodeAwareCompressor(code_config))- return "enabled"- else:- logger.warning(- "Code-aware compression requested but tree-sitter not installed. "- "Install with: pip install headroom-ai[code]"- )- return "unavailable"- else:- if is_tree_sitter_available():- return "available" # Available but not enabled- return "disabled"-- async def startup(self):- """Initialize async resources."""- self.pipeline_extensions.emit(- PipelineStage.PRE_START,- operation="proxy.startup",- metadata={"port": self.config.port, "host": self.config.host},- )- self.http_client = httpx.AsyncClient(- timeout=httpx.Timeout(- connect=self.config.connect_timeout_seconds,- read=self.config.request_timeout_seconds,- write=self.config.request_timeout_seconds,- pool=self.config.connect_timeout_seconds,- ),- limits=httpx.Limits(- max_connections=self.config.max_connections,- max_keepalive_connections=self.config.max_keepalive_connections,- ),- http2=self.config.http2,- )- logger.info("Headroom Proxy started")- logger.info(f"Optimization: {'ENABLED' if self.config.optimize else 'DISABLED'}")- self.config.mode = normalize_proxy_mode(self.config.mode)- logger.info(f"Mode: {self.config.mode}")- if self.config.mode == PROXY_MODE_TOKEN:- logger.info(" Prefix freeze: re-freeze after compression")- logger.info(" Read protection window: 30%% of excluded-tool messages")- logger.info(" CCR TTL: extended for session lifetime")- logger.info(" Compression cache: active")- if self.config.mode == PROXY_MODE_CACHE:- logger.info(" Prefix freeze: strict (all prior turns immutable)")- logger.info(" Mutations: latest turn only")- logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}")- logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}")- logger.info(- f"Connection Pool: max_connections={self.config.max_connections}, "- f"max_keepalive={self.config.max_keepalive_connections}, "- f"http2={'ENABLED' if self.config.http2 else 'DISABLED'}"- )-- # Unit 4 pre-upstream concurrency announcement. Report the resolved- # value (auto-detected vs. explicit) so operators can correlate- # ``pre_upstream_wait_ms`` log lines with the configured cap.- if self.anthropic_pre_upstream_sem is None:- logger.info("Anthropic pre-upstream concurrency: unbounded (explicitly disabled)")- else:- _explicit = self.config.anthropic_pre_upstream_concurrency- _origin = "auto-detected" if _explicit is None else "explicit"- logger.info(- "Anthropic pre-upstream concurrency: %d (%s)",- self.anthropic_pre_upstream_concurrency,- _origin,- )- logger.info(- "Anthropic pre-upstream timeouts: acquire=%.1fs compression=%.1fs memory_context=%.1fs",- self.anthropic_pre_upstream_acquire_timeout_seconds,- float(COMPRESSION_TIMEOUT_SECONDS),- self.anthropic_pre_upstream_memory_context_timeout_seconds,- )-- # Smart routing status- if self.config.smart_routing:- logger.info("Smart Routing: ENABLED (intelligent content detection)")- else:- logger.info("Smart Routing: DISABLED (legacy sequential mode)")-- # Eagerly load ALL compressors, parsers, and detectors at startup- # This eliminates cold-start latency spikes on first requests.- # Iterate BOTH pipelines (Anthropic + OpenAI) and dedupe transforms- # by id() so shared-transform instances never load twice. The- # resulting status dict is merged into ``self.warmup`` so /debug/warmup- # (Unit 5) and /readyz have a single source of truth.- self._kompress_status = "not installed"- eager_status: dict[str, str] = {}-- if self.config.optimize:- logger.info("Pre-loading compressors and parsers...")- seen_transform_ids: set[int] = set()- pipelines = (self.anthropic_pipeline, self.openai_pipeline)- for pipeline in pipelines:- for transform in pipeline.transforms:- if id(transform) in seen_transform_ids:- continue- seen_transform_ids.add(id(transform))- if not hasattr(transform, "eager_load_compressors"):- continue- try:- transform_status = transform.eager_load_compressors()- except Exception as exc:- logger.warning(- "Eager preload failed for %s: %s",- type(transform).__name__,- exc,- )- continue- if not isinstance(transform_status, dict):- continue- # Merge: later writers win only if the key wasn't set.- # Preload a transform ONCE — if another pipeline also has- # ``eager_load_compressors`` it contributes only new keys.- for key, value in transform_status.items():- eager_status.setdefault(key, value)- self.warmup.merge_transform_status(transform_status)-- # Update internal status from eager loading results- if eager_status.get("kompress") == "enabled":- self._kompress_status = "enabled"- if eager_status.get("code_aware") == "enabled":- self._code_aware_status = "enabled"-- # Log component status- if self._kompress_status == "enabled":- logger.info("Kompress: ENABLED (ModernBERT token compressor)")- elif self.config.optimize:- logger.info("Kompress: not installed (pip install headroom-ai[ml] for ML compression)")-- if self._code_aware_status == "enabled":- logger.info("Code-Aware: ENABLED (AST-based compression)")- if "tree_sitter" in eager_status:- logger.info(f"Tree-Sitter: {eager_status['tree_sitter']}")- elif self._code_aware_status == "lazy":- logger.info("Code-Aware: LAZY (will load when code content detected)")- elif self._code_aware_status == "available":- logger.info("Code-Aware: available but disabled (use --code-aware)")- elif self._code_aware_status == "unavailable":- logger.info("Code-Aware: not installed (pip install headroom-ai[code])")- elif self._code_aware_status == "disabled":- logger.info("Code-Aware: DISABLED")-- if eager_status.get("magika") == "enabled":- logger.info("Magika: ENABLED (ML content detection)")-- if self.memory_handler:- self.warmup.memory_backend.mark_loading()- try:- await self.memory_handler.ensure_initialized()- except Exception as exc: # pragma: no cover - defensive- self.warmup.memory_backend.mark_error(str(exc))- logger.warning("Memory: backend initialization failed (startup continues): %s", exc)- memory_status = self.memory_handler.health_status()- if memory_status.get("initialized"):- self.warmup.memory_backend.mark_loaded(- handle=self.memory_handler,- backend=memory_status.get("backend"),- )- # Force one embed call so the ONNX graph is compiled now,- # not lazily during the first request. Best-effort — any- # failure is swallowed inside warmup_embedder.- self.warmup.memory_embedder.mark_loading()- warmed = await self.memory_handler.warmup_embedder()- if warmed:- self.warmup.memory_embedder.mark_loaded()- else:- # Not an error — e.g. qdrant-neo4j has no embedder slot- # we can reach, or the backend simply exposes no handle.- self.warmup.memory_embedder.mark_null()- else:- if self.warmup.memory_backend.status != "error":- self.warmup.memory_backend.mark_null()- self.warmup.memory_embedder.mark_null()- logger.info(- "Memory: ENABLED "- f"(backend={memory_status['backend']}, initialized={memory_status['initialized']})"- )- else:- logger.info("Memory: DISABLED")-- # CCR status- ccr_features = []- if self.config.ccr_inject_tool:- ccr_features.append("tool_injection")- if self.config.ccr_handle_responses:- ccr_features.append("response_handling")- if self.config.ccr_context_tracking:- ccr_features.append("context_tracking")- if self.config.ccr_proactive_expansion:- ccr_features.append("proactive_expansion")- if ccr_features:- logger.info(f"CCR (Compress-Cache-Retrieve): ENABLED ({', '.join(ccr_features)})")- else:- logger.info("CCR: DISABLED")- logger.info(f"Savings history: {self.metrics.savings_tracker.storage_path}")-- # Reset and rebuild the quota tracker registry for this server instance.- # reset_quota_registry() ensures a clean slate when the proxy is restarted- # (e.g. in tests that spin up multiple app instances in the same process).- reset_quota_registry()- registry = get_quota_registry()- tracker = configure_subscription_tracker(- poll_interval_s=self.config.subscription_poll_interval_s,- active_window_s=self.config.subscription_active_window_s,- enabled=self.config.subscription_tracking_enabled,- )- registry.register(tracker)- registry.register(get_codex_rate_limit_state())- registry.register(get_copilot_quota_tracker())- await registry.start_all()-- if self.config.subscription_tracking_enabled:- logger.info(- "Subscription tracking: ENABLED "- f"(poll_interval={self.config.subscription_poll_interval_s}s, "- f"active_window={self.config.subscription_active_window_s}s)"- )- else:- logger.info("Subscription tracking: DISABLED")-- copilot_tracker = get_copilot_quota_tracker()- if copilot_tracker.is_available():- logger.info("GitHub Copilot quota tracking: ENABLED")- else:- logger.info(- "GitHub Copilot quota tracking: DISABLED "- "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)"- )-- # Log anonymous telemetry status so operators can see it in the log stream- if is_telemetry_enabled():- logger.info(- "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). "- "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry"- )- else:- logger.info("Anonymous telemetry: DISABLED")-- self.pipeline_extensions.emit(- PipelineStage.POST_START,- operation="proxy.startup",- metadata={- "port": self.config.port,- "host": self.config.host,- "warmup": self.warmup.to_dict(),- },- )-- async def shutdown(self):- """Cleanup async resources."""- if self.http_client:- await self.http_client.aclose()- self.http_client = None-- if self.memory_handler and hasattr(self.memory_handler, "close"):- await self.memory_handler.close()-- # Stop all quota trackers via the registry- await get_quota_registry().stop_all()-- # Print final stats- self._print_summary()-- def _print_summary(self):- """Print session summary."""- m = self.metrics- logger.info("=" * 70)- logger.info("HEADROOM PROXY SESSION SUMMARY")- logger.info("=" * 70)- logger.info(f"Total requests: {m.requests_total}")- logger.info(f"Cached responses: {m.requests_cached}")- logger.info(f"Rate limited: {m.requests_rate_limited}")- logger.info(f"Failed: {m.requests_failed}")- logger.info(f"Input tokens: {m.tokens_input_total:,}")- logger.info(f"Output tokens: {m.tokens_output_total:,}")- logger.info(f"Tokens saved: {m.tokens_saved_total:,}")- if m.tokens_input_total > 0:- savings_pct = (- m.tokens_saved_total / (m.tokens_input_total + m.tokens_saved_total)- ) * 100- logger.info(f"Token savings: {savings_pct:.1f}%")- if m.latency_count > 0:- avg_latency = m.latency_sum_ms / m.latency_count- logger.info(f"Avg latency: {avg_latency:.0f}ms")- logger.info("=" * 70)-- async def _next_request_id(self) -> str:- """Generate unique request ID."""- async with self._request_counter_lock:- self._request_counter += 1- return f"hr_{int(time.time())}_{self._request_counter:06d}"-- def _extract_tags(self, headers: dict) -> dict[str, str]:- """Extract Headroom tags from headers."""- tags = {}- for key, value in headers.items():- if key.lower().startswith("x-headroom-"):- tag_name = key.lower().replace("x-headroom-", "")- tags[tag_name] = value- return tags-- def _inject_system_context(- self,- messages: list[dict[str, Any]],- context: str,- body: dict[str, Any] | None = None,- ) -> list[dict[str, Any]]:- """Inject context into the system message/parameter.-- For Anthropic API: Uses top-level 'system' parameter (not messages array).- For OpenAI API: Uses system role in messages array.-- Args:- messages: The messages list.- context: Context to inject.- body: Optional request body to update system parameter (for Anthropic).-- Returns:- Updated messages list.- """- messages = list(messages) # Copy to avoid mutation-- # For Anthropic API: use top-level 'system' parameter- if body is not None:- existing_system = body.get("system", "")- if isinstance(existing_system, str):- body["system"] = (existing_system + "\n\n" + context).strip()- elif isinstance(existing_system, list):- # system is a list of content blocks (e.g., with cache_control).- # Append memory context as a new text block — never overwrite.- body["system"] = existing_system + [{"type": "text", "text": context}]- else:- # No existing system prompt — set as string- body["system"] = context- return messages-- # For OpenAI API: use system role in messages- for i, msg in enumerate(messages):- if msg.get("role") == "system":- content = msg.get("content", "")- if isinstance(content, str):- messages[i] = {**msg, "content": content + "\n\n" + context}- return messages-- # No system message found - prepend one- messages.insert(0, {"role": "system", "content": context})- return messages-- async def _retry_request(- self,- method: str,- url: str,- headers: dict,- body: dict,- stream: bool = False,- ) -> httpx.Response:- """Make request with retry and exponential backoff."""- last_error = None-- for attempt in range(self.config.retry_max_attempts):- try:- if stream:- # For streaming, we return early - retry happens at higher level- return await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]- else:- response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]-- # Don't retry client errors (4xx)- if 400 <= response.status_code < 500:- return response-- # Retry server errors (5xx)- if response.status_code >= 500:- raise httpx.HTTPStatusError(- f"Server error: {response.status_code}",- request=response.request,- response=response,- )-- return response-- except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:- last_error = e-- if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:- raise-- # Exponential backoff with jitter- delay_with_jitter = jitter_delay_ms(- self.config.retry_base_delay_ms,- self.config.retry_max_delay_ms,- attempt,- )-- logger.warning(- f"Request failed (attempt {attempt + 1}), retrying in {delay_with_jitter:.0f}ms: {e}"- )- await asyncio.sleep(delay_with_jitter / 1000)-- raise last_error # type: ignore[misc]---async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:- """Background task that logs TOIN stats periodically.-- Args:- interval_seconds: How often to log stats (default: 5 minutes).- """- while True:- await asyncio.sleep(interval_seconds)- try:- toin = get_toin()- stats = toin.get_stats()- total_compressions = stats.get("total_compressions", 0)- if total_compressions > 0:- patterns = stats.get("patterns_tracked", 0)- retrievals = stats.get("total_retrievals", 0)- retrieval_rate = stats.get("global_retrieval_rate", 0.0)- logger.info(- "TOIN: %d patterns, %d compressions, %d retrievals, %.1f%% retrieval rate",- patterns,- total_compressions,- retrievals,- retrieval_rate * 100,- )- except Exception as e:- logger.debug("Failed to log TOIN stats: %s", e)---def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:- """Register all memory-tracked components with the tracker.-- This function is idempotent - it checks if components are already registered.-- Args:- proxy: The HeadroomProxy instance.- tracker: The MemoryTracker instance.- """- # Register compression store (global singleton)- if "compression_store" not in tracker.registered_components:- store = get_compression_store()- tracker.register("compression_store", store.get_memory_stats)-- # Register semantic cache (instance on proxy)- if proxy.cache and "semantic_cache" not in tracker.registered_components:- tracker.register("semantic_cache", proxy.cache.get_memory_stats)-- # Register request logger (instance on proxy)- if proxy.logger and "request_logger" not in tracker.registered_components:- tracker.register("request_logger", proxy.logger.get_memory_stats)-- # Register batch context store (global singleton)- if "batch_context_store" not in tracker.registered_components:- try:- from ..ccr.batch_store import get_batch_context_store-- batch_store = get_batch_context_store()- if hasattr(batch_store, "get_memory_stats"):- tracker.register("batch_context_store", batch_store.get_memory_stats)- except ImportError:- pass-- # Note: graph_store and vector_index are created per-user within the- # LocalMemoryBackend, not as global singletons. They would need to be- # registered when the memory system is initialized with specific backends.---def create_app(config: ProxyConfig | None = None) -> FastAPI:- """Create FastAPI application."""- if not FASTAPI_AVAILABLE:- raise ImportError("FastAPI required. Install: pip install fastapi uvicorn httpx")-- from contextlib import asynccontextmanager-- config = config or ProxyConfig()- proxy = HeadroomProxy(config)-- # Telemetry beacon (anonymous aggregate stats).- # With uvicorn workers > 1, each worker runs the lifespan independently.- # We must ensure only ONE beacon runs across all workers — otherwise each- # worker creates its own beacon, spamming the telemetry table with N rows- # per cycle instead of 1 (all reading the same /stats from the same port).- #- # Strategy: use a file lock to ensure only the first worker starts the- # beacon. Other workers see the lock and skip.- from headroom.telemetry.beacon import TelemetryBeacon-- _beacon = TelemetryBeacon(- port=config.port if hasattr(config, "port") else 8787,- sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",- backend=config.backend if hasattr(config, "backend") else "anthropic",- )- from headroom import paths as _hr_paths-- _beacon_lock_path = _hr_paths.beacon_lock_path(config.port)- _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor- _beacon_is_owner: list = [False]-- def _try_acquire_beacon_lock() -> bool:- """Try to acquire the beacon file lock (non-blocking).-- Returns True if this process is the beacon owner.- """- if not HAS_FCNTL:- return True-- fd = None- try:- _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)- fd = open(_beacon_lock_path, "w") # noqa: SIM115- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)- fd.write(str(os.getpid()))- fd.flush()- _beacon_lock_fd[0] = fd- return True- except OSError:- if fd is not None:- fd.close()- return False-- def _release_beacon_lock() -> None:- """Release the beacon file lock."""- fd = _beacon_lock_fd[0]- if fd:- try:- if HAS_FCNTL:- fcntl.flock(fd, fcntl.LOCK_UN)- fd.close()- except Exception:- pass- _beacon_lock_fd[0] = None- try:- _beacon_lock_path.unlink(missing_ok=True)- except Exception:- pass-- @asynccontextmanager- async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]- configure_otel_metrics(OTelMetricsConfig.from_env(default_service_name="headroom-proxy"))- configure_langfuse_tracing(- LangfuseTracingConfig.from_env(default_service_name="headroom-proxy")- )-- app.state.started_at = time.time()- app.state.ready = False- app.state.startup_error = None-- try:- try:- # Startup- await proxy.startup()- asyncio.create_task(_log_toin_stats_periodically())- if proxy.usage_reporter:- await proxy.usage_reporter.start(proxy)- if proxy.traffic_learner:- await proxy.traffic_learner.start()-- # Only start beacon if we acquire the lock (first worker wins)- _beacon_is_owner[0] = _try_acquire_beacon_lock()- if _beacon_is_owner[0]:- await _beacon.start()- else:- logger.debug("Beacon: skipping (another worker owns the lock)")-- app.state.ready = True- yield- except Exception as exc:- app.state.startup_error = str(exc)- raise- finally:- app.state.ready = False- # Shutdown- if _beacon_is_owner[0]:- await _beacon.stop()- _release_beacon_lock()- if proxy.usage_reporter:- await proxy.usage_reporter.stop()- if proxy.traffic_learner:- await proxy.traffic_learner.stop()- if proxy.code_graph_watcher:- proxy.code_graph_watcher.stop()- await proxy.shutdown()- shutdown_headroom_tracing()- shutdown_otel_metrics()-- app = FastAPI(- title="Headroom Proxy",- description="Production-ready LLM optimization proxy",- version=__version__,- lifespan=lifespan,- )- app.state.proxy = proxy- app.state.started_at = None- app.state.ready = False- app.state.startup_error = None-- def _iso_utc_now() -> str:- return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")-- def _uptime_seconds() -> float:- started_at = getattr(app.state, "started_at", None)- if not isinstance(started_at, int | float):- return 0.0- return round(max(0.0, time.time() - float(started_at)), 3)-- def _component_health(- *,- enabled: bool,- ready: bool,- **details: Any,- ) -> dict[str, Any]:- status = "disabled" if not enabled else ("healthy" if ready else "unhealthy")- return {- "enabled": enabled,- "ready": (ready if enabled else True),- "status": status,- **details,- }-- def _health_checks() -> dict[str, dict[str, Any]]:- memory_status = (- proxy.memory_handler.health_status()- if proxy.memory_handler- else {- "enabled": False,- "backend": None,- "initialized": False,- "native_tool": False,- "bridge_enabled": False,- }- )- memory_enabled = bool(memory_status.get("enabled", False))- memory_initialized = bool(memory_status.get("initialized", False))- return {- "startup": _component_health(- enabled=True,- ready=bool(getattr(app.state, "ready", False)),- error=getattr(app.state, "startup_error", None),- ),- "http_client": _component_health(- enabled=True,- ready=proxy.http_client is not None,- ),- "cache": _component_health(- enabled=config.cache_enabled,- ready=(proxy.cache is not None),- ),- "rate_limiter": _component_health(- enabled=config.rate_limit_enabled,- ready=(proxy.rate_limiter is not None),- ),- "memory": _component_health(- enabled=memory_enabled,- ready=memory_initialized,- backend=memory_status["backend"],- initialized=memory_initialized,- native_tool=bool(memory_status.get("native_tool", False)),- bridge_enabled=bool(memory_status.get("bridge_enabled", False)),- ),- }-- def _runtime_payload() -> dict[str, Any]:- ws_registry = getattr(proxy, "ws_sessions", None)- ws_active_sessions = ws_registry.active_count() if ws_registry is not None else 0- ws_active_relay_tasks = (- ws_registry.active_relay_task_count() if ws_registry is not None else 0- )- return {- "anthropic_pre_upstream": {- "enabled": proxy.anthropic_pre_upstream_sem is not None,- "resolved_concurrency": proxy.anthropic_pre_upstream_concurrency,- "source": (- "auto" if config.anthropic_pre_upstream_concurrency is None else "explicit"- ),- "acquire_timeout_seconds": proxy.anthropic_pre_upstream_acquire_timeout_seconds,- "compression_timeout_seconds": float(COMPRESSION_TIMEOUT_SECONDS),- "memory_context_timeout_seconds": (- proxy.anthropic_pre_upstream_memory_context_timeout_seconds- ),- "codex_ws_gated": False,- },- "websocket_sessions": {- "active_sessions": ws_active_sessions,- "active_relay_tasks": ws_active_relay_tasks,- },- }-- def _health_payload(*, include_config: bool) -> dict[str, Any]:- checks = _health_checks()- ready = all(check["ready"] for check in checks.values())- payload: dict[str, Any] = {- "service": "headroom-proxy",- "status": "healthy" if ready else "unhealthy",- "ready": ready,- "version": __version__,- "timestamp": _iso_utc_now(),- "uptime_seconds": _uptime_seconds(),- "checks": checks,- "runtime": _runtime_payload(),- }- deployment_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE")- if deployment_profile:- payload["deployment"] = {- "profile": deployment_profile,- "preset": os.environ.get("HEADROOM_DEPLOYMENT_PRESET"),- "runtime": os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME"),- "supervisor": os.environ.get("HEADROOM_DEPLOYMENT_SUPERVISOR"),- "scope": os.environ.get("HEADROOM_DEPLOYMENT_SCOPE"),- }- if include_config:- payload["config"] = {- "backend": config.backend,- "optimize": config.optimize,- "cache": config.cache_enabled,- "rate_limit": config.rate_limit_enabled,- "memory": config.memory_enabled,- "learn": config.traffic_learning_enabled,- "code_graph": config.code_graph_watcher,- "pid": os.getpid(),- }- return payload-- # CORS- app.add_middleware(- CORSMiddleware,- allow_origins=["*"],- allow_credentials=True,- allow_methods=["*"],- allow_headers=["*"],- )-- # X-Headroom-Stack: SDK adapters (TS openai/anthropic/etc.) tag their- # requests so telemetry can segment by integration surface. Registered- # before extension middleware so any extension-level auth/guards run- # outermost and we don't count requests they reject.- @app.middleware("http")- async def _record_headroom_stack(request, call_next):- if request.url.path.startswith("/v1/"):- stack = request.headers.get("x-headroom-stack")- if stack:- try:- proxy.metrics.record_stack(stack)- except Exception:- logger.debug("record_stack failed", exc_info=True)- return await call_next(request)-- # Third-party proxy extensions (Enterprise, custom plugins). Discovered via- # the `headroom.proxy_extension` entry-point group. An extension that raises- # from its install() is a deliberate fail-closed signal and aborts startup.- from headroom.proxy.extensions import install_all as _install_extensions-- _install_extensions(app, config)-- # Health & Metrics- @app.get("/livez")- async def livez():- return JSONResponse(- status_code=200,- content={- "service": "headroom-proxy",- "status": "healthy",- "alive": True,- "version": __version__,- "timestamp": _iso_utc_now(),- "uptime_seconds": _uptime_seconds(),- },- )-- @app.get("/readyz")- async def readyz():- payload = _health_payload(include_config=False)- return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)-- @app.get("/health")- async def health():- payload = _health_payload(include_config=True)- return JSONResponse(status_code=200, content=payload)-- # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —- # debug endpoints are invisible to external scanners.- from headroom.proxy.debug_introspection import (- collect_tasks as _collect_tasks,- )- from headroom.proxy.loopback_guard import require_loopback as _require_loopback-- @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])- async def debug_tasks(stack: bool = False):- """Enumerate running asyncio tasks.-- Default is cheap — ``stack_depth`` is ``null`` in every entry so- a storm snapshot does not walk 50+ coroutine frames synchronously.- Pass ``?stack=true`` to compute ``stack_depth`` for each task- (useful for single-shot human debugging).- """- ws_registry = getattr(proxy, "ws_sessions", None)- return JSONResponse(- status_code=200,- content=_collect_tasks(ws_registry, with_stack_depth=stack),- )-- @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])- async def debug_ws_sessions():- ws_registry = getattr(proxy, "ws_sessions", None)- snapshot = ws_registry.snapshot() if ws_registry is not None else []- return JSONResponse(status_code=200, content=snapshot)-- @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])- async def debug_warmup():- warmup_registry = getattr(proxy, "warmup", None)- payload = warmup_registry.to_dict() if warmup_registry is not None else {}- payload["runtime"] = _runtime_payload()- return JSONResponse(status_code=200, content=payload)-- @app.get("/dashboard", response_class=HTMLResponse)- async def dashboard():- """Serve the Headroom dashboard UI."""- return get_dashboard_html()-- @app.get("/stats")- async def stats():- """Get comprehensive proxy statistics.-- This is the main stats endpoint - it aggregates data from all subsystems:- - Request metrics (total, cached, failed, by model/provider)- - Token usage and savings- - Cost tracking- - Canonical persisted display_session metrics for downstream dashboards- - Compression (CCR) statistics- - Telemetry/TOIN (data flywheel) statistics- - Cache and rate limiter stats- """- m = proxy.metrics-- # Calculate average latency- avg_latency_ms = round(m.latency_sum_ms / m.latency_count, 2) if m.latency_count > 0 else 0- min_latency_ms = (- round(m.latency_min_ms, 2)- if m.latency_count > 0 and m.latency_min_ms != float("inf")- else 0- )- max_latency_ms = round(m.latency_max_ms, 2) if m.latency_count > 0 else 0-- # Calculate Headroom overhead (optimization time only, excludes pass-through requests)- avg_overhead_ms = (- round(m.overhead_sum_ms / m.overhead_count, 2) if m.overhead_count > 0 else 0- )- min_overhead_ms = (- round(m.overhead_min_ms, 2)- if m.overhead_count > 0 and m.overhead_min_ms != float("inf")- else 0- )- max_overhead_ms = round(m.overhead_max_ms, 2) if m.overhead_count > 0 else 0-- # Calculate TTFB (time to first byte)- avg_ttfb_ms = round(m.ttfb_sum_ms / m.ttfb_count, 2) if m.ttfb_count > 0 else 0- min_ttfb_ms = (- round(m.ttfb_min_ms, 2) if m.ttfb_count > 0 and m.ttfb_min_ms != float("inf") else 0- )- max_ttfb_ms = round(m.ttfb_max_ms, 2) if m.ttfb_count > 0 else 0-- # Get compression store stats- store = get_compression_store()- compression_stats = store.get_stats()-- # Get telemetry/TOIN stats- telemetry = get_telemetry_collector()- telemetry_stats = telemetry.get_stats()-- # Get feedback loop stats- feedback = get_compression_feedback()- feedback_stats = feedback.get_stats()-- # Build prefix cache stats once (used in both prefix_cache and cost)- prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)-- # Fetch CLI filtering savings (rtk — tokens avoided before reaching context)- cli_filtering_stats = _get_rtk_stats()- cli_tokens_avoided = (- cli_filtering_stats.get("tokens_saved", 0) if cli_filtering_stats else 0- )-- # Calculate total tokens before compression- total_tokens_before = m.tokens_input_total + m.tokens_saved_total-- # Build human-readable summary- summary = _build_session_summary(- proxy, m, prefix_cache_stats, cli_tokens_avoided, total_tokens_before- )- # DEBUG: log the summary payload for external upsert consumers- try:- logger.debug("/stats summary data: %r", summary)- except Exception:- logger.warning("Failed to log /stats summary payload")-- # Compression cache stats (token mode)- compression_cache_stats: dict = {}- if proxy.config.mode == PROXY_MODE_TOKEN and proxy._compression_caches:- total_entries = 0- total_hits = 0- total_misses = 0- total_tokens_saved = 0- for cache in proxy._compression_caches.values():- s = cache.get_stats()- total_entries += s.get("entries", 0)- total_hits += s.get("hits", 0)- total_misses += s.get("misses", 0)- total_tokens_saved += s.get("total_tokens_saved", 0)- compression_cache_stats = {- "mode": PROXY_MODE_TOKEN,- "active_sessions": len(proxy._compression_caches),- "total_entries": total_entries,- "total_hits": total_hits,- "total_misses": total_misses,- "hit_rate": round(total_hits / max(1, total_hits + total_misses) * 100, 1),- "total_tokens_saved": total_tokens_saved,- }- else:- compression_cache_stats = {"mode": proxy.config.mode}-- # Build unified savings summary (all layers)- compression_tokens = m.tokens_saved_total- cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)- total_tokens_all_layers = compression_tokens + cli_tokens_avoided- persistent_savings = m.savings_tracker.stats_preview()- display_session = persistent_savings.get("display_session", {})-- return {- "summary": summary,- "savings": {- "total_tokens": total_tokens_all_layers,- "by_layer": {- "cli_filtering": {- "tokens": cli_tokens_avoided,- "description": "Tokens avoided by CLI output filtering (rtk) before reaching context",- },- "compression": {- "tokens": compression_tokens,- "description": "Tokens removed by proxy compression (SmartCrusher, ContentRouter, etc.)",- },- "prefix_cache": {- "discount_usd": round(cache_net_usd, 4),- "description": (- "Cost discount from provider prefix caching. "- "Headroom's CacheAligner improves hit rates; "- "baseline caching is provider-native."- ),- },- },- },- "requests": {- "total": m.requests_total,- "cached": m.requests_cached,- "rate_limited": m.requests_rate_limited,- "failed": m.requests_failed,- "by_provider": dict(m.requests_by_provider),- "by_model": dict(m.requests_by_model),- "by_stack": dict(m.requests_by_stack),- },- "tokens": {- "input": m.tokens_input_total,- "output": m.tokens_output_total,- "saved": m.tokens_saved_total,- "cli_tokens_avoided": cli_tokens_avoided,- "total_before_compression": total_tokens_before,- "savings_percent": round(- (m.tokens_saved_total / total_tokens_before * 100)- if total_tokens_before > 0- else 0,- 2,- ),- },- "latency": {- "average_ms": avg_latency_ms,- "min_ms": min_latency_ms,- "max_ms": max_latency_ms,- "total_requests": m.latency_count,- },- "overhead": {- "average_ms": avg_overhead_ms,- "min_ms": min_overhead_ms,- "max_ms": max_overhead_ms,- },- "ttfb": {- "average_ms": avg_ttfb_ms,- "min_ms": min_ttfb_ms,- "max_ms": max_ttfb_ms,- },- "pipeline_timing": {- name: {- "average_ms": round(- m.transform_timing_sum[name] / m.transform_timing_count[name], 2- ),- "max_ms": round(m.transform_timing_max[name], 2),- "count": m.transform_timing_count[name],- }- for name in sorted(m.transform_timing_sum.keys())- }- if m.transform_timing_sum- else {},- "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},- "savings_history": m.savings_history[-100:], # Last 100 data points- "display_session": display_session,- "persistent_savings": persistent_savings,- "prefix_cache": prefix_cache_stats,- "cost": _merge_cost_stats(- proxy.cost_tracker.stats() if proxy.cost_tracker else None,- prefix_cache_stats,- cli_tokens_avoided=cli_tokens_avoided,- ),- "compression": {- "ccr_entries": compression_stats.get("entry_count", 0),- "ccr_max_entries": compression_stats.get("max_entries", 0),- "original_tokens_cached": compression_stats.get("total_original_tokens", 0),- "compressed_tokens_cached": compression_stats.get("total_compressed_tokens", 0),- "ccr_retrievals": compression_stats.get("total_retrievals", 0),- },- "compression_cache": compression_cache_stats,- "anon_telemetry_shipping": is_telemetry_enabled(),- "telemetry": {- "enabled": telemetry_stats.get("enabled", False),- "total_compressions": telemetry_stats.get("total_compressions", 0),- "total_retrievals": telemetry_stats.get("total_retrievals", 0),- "global_retrieval_rate": round(telemetry_stats.get("global_retrieval_rate", 0), 4),- "tool_signatures_tracked": telemetry_stats.get("tool_signatures_tracked", 0),- "avg_compression_ratio": round(telemetry_stats.get("avg_compression_ratio", 0), 4),- "avg_token_reduction": round(telemetry_stats.get("avg_token_reduction", 0), 4),- },- "otel": get_otel_metrics_status(),- "langfuse": get_langfuse_tracing_status(),- "feedback_loop": {- "tools_tracked": feedback_stats.get("tools_tracked", 0),- "total_compressions": feedback_stats.get("total_compressions", 0),- "total_retrievals": feedback_stats.get("total_retrievals", 0),- "global_retrieval_rate": round(feedback_stats.get("global_retrieval_rate", 0), 4),- "tools_with_high_retrieval": sum(- 1- for p in feedback_stats.get("tool_patterns", {}).values()- if p.get("retrieval_rate", 0) > 0.3- ),- },- "toin": get_toin().get_stats(),- "cli_filtering": cli_filtering_stats,- "cache": await proxy.cache.stats() if proxy.cache else None,- "rate_limiter": await proxy.rate_limiter.stats() if proxy.rate_limiter else None,- "recent_requests": proxy.logger.get_recent(10) if proxy.logger else [],- "log_full_messages": proxy.config.log_full_messages if proxy else False,- **get_quota_registry().get_all_stats(),- }-- @app.get("/stats-history")- async def stats_history(- format: Literal["json", "csv"] = "json",- series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",- ):- """Get durable proxy compression history plus display-session state."""- if format == "csv":- filename = f"headroom-stats-history-{series}.csv"- return Response(- content=proxy.metrics.savings_tracker.export_csv(series=series),- media_type="text/csv; charset=utf-8",- headers={"Content-Disposition": f'attachment; filename="{filename}"'},- )-- return proxy.metrics.savings_tracker.history_response()-- @app.get("/transformations/feed")- async def transformations_feed(limit: int = 20):- """Get recent message transformations for the live feed.-- Returns empty list if log_full_messages is disabled (messages are not stored).- """- if limit > 100:- limit = 100-- transformations = []- log_full_messages = proxy.config.log_full_messages if proxy else False-- if proxy and proxy.logger:- logs = proxy.logger.get_recent_with_messages(limit)- for log in logs:- transformations.append(- {- "request_id": log.get("request_id"),- "timestamp": log.get("timestamp"),- "provider": log.get("provider"),- "model": log.get("model"),- "input_tokens_original": log.get("input_tokens_original"),- "input_tokens_optimized": log.get("input_tokens_optimized"),- "tokens_saved": log.get("tokens_saved"),- "savings_percent": log.get("savings_percent"),- "transforms_applied": log.get("transforms_applied", []),- "request_messages": log.get("request_messages"),- "response_content": log.get("response_content"),- }- )-- return {"transformations": transformations, "log_full_messages": log_full_messages}-- @app.get("/subscription-window")- async def subscription_window():- """Current Anthropic subscription window utilisation and Headroom contribution."""- tracker = get_subscription_tracker()- if tracker is None:- return JSONResponse(- status_code=503,- content={"error": "Subscription tracking is not enabled"},- )- return JSONResponse(content=tracker.state)-- @app.get("/quota")- async def quota():- """Unified quota/rate-limit stats for all registered providers (Anthropic, Codex, Copilot)."""- return JSONResponse(content=get_quota_registry().get_all_stats())-- @app.get("/metrics")- async def metrics():- """Prometheus metrics endpoint."""- return PlainTextResponse(- await proxy.metrics.export(),- media_type="text/plain; version=0.0.4",- )-- # Debug endpoints- @app.get("/debug/memory")- async def debug_memory():- """Get detailed memory usage statistics.-- Returns memory usage for all tracked components including:- - Process-level memory (RSS, VMS, percent)- - Per-component memory usage and budgets- - Cache hit/miss statistics- - Total tracked vs target budget-- This endpoint is useful for debugging memory issues and- monitoring memory budgets.- """- from ..memory.tracker import MemoryTracker-- tracker = MemoryTracker.get()-- # Register components if not already registered- _register_memory_components(proxy, tracker)-- report = tracker.get_report()- return report.to_dict()-- @app.post("/cache/clear")- async def clear_cache():- """Clear the response cache."""- if proxy.cache:- await proxy.cache.clear()- return {"status": "cleared"}- return {"status": "cache disabled"}-- # CCR (Compress-Cache-Retrieve) endpoints- @app.post("/v1/retrieve")- async def ccr_retrieve(request: Request):- """Retrieve original content from CCR compression cache.-- This is the "Retrieve" part of CCR (Compress-Cache-Retrieve).- When SmartCrusher compresses tool outputs, the original data is cached.- LLMs can call this endpoint to get more data if needed.-- Request body:- hash (str): Hash key from compression marker (required)- query (str): Optional search query to filter results-- Response:- Full retrieval: {"hash": "...", "original_content": "...", ...}- Search: {"hash": "...", "query": "...", "results": [...], "count": N}- """- data = await request.json()- hash_key = data.get("hash")- query = data.get("query")-- if not hash_key:- raise HTTPException(status_code=400, detail="hash required")-- store = get_compression_store()-- if query:- # Search within cached content- results = store.search(hash_key, query)- return {- "hash": hash_key,- "query": query,- "results": results,- "count": len(results),- }- else:- # Return full original content- entry = store.retrieve(hash_key)- if entry:- return {- "hash": hash_key,- "original_content": entry.original_content,- "original_tokens": entry.original_tokens,- "original_item_count": entry.original_item_count,- "compressed_item_count": entry.compressed_item_count,- "tool_name": entry.tool_name,- "retrieval_count": entry.retrieval_count,- }- raise HTTPException(- status_code=404, detail="Entry not found or expired (TTL: 5 minutes)"- )-- @app.get("/v1/retrieve/stats")- async def ccr_stats():- """Get CCR compression store statistics."""- store = get_compression_store()- stats = store.get_stats()- events = store.get_retrieval_events(limit=20)- return {- "store": stats,- "recent_retrievals": [- {- "hash": e.hash,- "query": e.query,- "items_retrieved": e.items_retrieved,- "total_items": e.total_items,- "tool_name": e.tool_name,- "retrieval_type": e.retrieval_type,- }- for e in events- ],- }-- @app.get("/v1/feedback")- async def ccr_feedback():- """Get CCR feedback loop statistics and learned patterns.-- This endpoint exposes the feedback loop's learned patterns for monitoring- and debugging. It shows:- - Per-tool retrieval rates (high = compress less aggressively)- - Common search queries per tool- - Queried fields (suggest what to preserve)-- Use this to understand how well compression is working and whether- the feedback loop is adjusting appropriately.- """- feedback = get_compression_feedback()- stats = feedback.get_stats()- return {- "feedback": stats,- "hints_example": {- tool_name: {- "hints": {- "max_items": hints.max_items- if (hints := feedback.get_compression_hints(tool_name))- else 15,- "suggested_items": hints.suggested_items if hints else None,- "skip_compression": hints.skip_compression if hints else False,- "preserve_fields": hints.preserve_fields if hints else [],- "reason": hints.reason if hints else "",- }- }- for tool_name in list(stats.get("tool_patterns", {}).keys())[:5]- },- }-- @app.get("/v1/feedback/{tool_name}")- async def ccr_feedback_for_tool(tool_name: str):- """Get compression hints for a specific tool.-- Returns feedback-based hints that would be used for compressing- this tool's output.- """- feedback = get_compression_feedback()- hints = feedback.get_compression_hints(tool_name)- patterns = feedback.get_all_patterns().get(tool_name)-- return {- "tool_name": tool_name,- "hints": {- "max_items": hints.max_items,- "min_items": hints.min_items,- "suggested_items": hints.suggested_items,- "aggressiveness": hints.aggressiveness,- "skip_compression": hints.skip_compression,- "preserve_fields": hints.preserve_fields,- "reason": hints.reason,- },- "pattern": {- "total_compressions": patterns.total_compressions if patterns else 0,- "total_retrievals": patterns.total_retrievals if patterns else 0,- "retrieval_rate": patterns.retrieval_rate if patterns else 0.0,- "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0,- "search_rate": patterns.search_rate if patterns else 0.0,- "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [],- "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [],- }- if patterns- else None,- }-- # Telemetry endpoints (Data Flywheel)- @app.get("/v1/telemetry")- async def telemetry_stats():- """Get telemetry statistics for the data flywheel.-- This endpoint exposes privacy-preserving telemetry data that powers- the data flywheel - learning optimal compression strategies across- tool types based on usage patterns.-- What's collected (anonymized):- - Tool output structure patterns (field types, not values)- - Compression decisions and ratios- - Retrieval patterns (rate, type, not content)- - Strategy effectiveness-- What's NOT collected:- - Actual data values- - User identifiers- - Queries or search terms- - File paths or tool names (hashed by default)- """- telemetry = get_telemetry_collector()- return telemetry.get_stats()-- @app.get("/v1/telemetry/export")- async def telemetry_export():- """Export full telemetry data for aggregation.-- This endpoint exports all telemetry data in a format suitable for- cross-user aggregation. The data is privacy-preserving - no actual- values are included, only structural patterns and statistics.-- Use this for:- - Building a central learning service- - Sharing learned patterns across instances- - Analysis and debugging- """- telemetry = get_telemetry_collector()- return telemetry.export_stats()-- @app.post("/v1/telemetry/import")- async def telemetry_import(request: Request):- """Import telemetry data from another source.-- This allows merging telemetry from multiple sources for cross-user- learning. The imported data is merged with existing statistics.-- Request body: Telemetry export data from /v1/telemetry/export- """- telemetry = get_telemetry_collector()- data = await request.json()- telemetry.import_stats(data)- return {"status": "imported", "current_stats": telemetry.get_stats()}-- @app.get("/v1/telemetry/tools")- async def telemetry_tools():- """Get telemetry statistics for all tracked tool signatures.-- Returns statistics per tool signature (anonymized), including:- - Compression ratios and strategy usage- - Retrieval rates (high = compression too aggressive)- - Learned recommendations- """- telemetry = get_telemetry_collector()- all_stats = telemetry.get_all_tool_stats()- return {- "tool_count": len(all_stats),- "tools": {sig_hash: stats.to_dict() for sig_hash, stats in all_stats.items()},- }-- @app.get("/v1/telemetry/tools/{signature_hash}")- async def telemetry_tool_detail(signature_hash: str):- """Get detailed telemetry for a specific tool signature.-- Includes learned recommendations if enough data has been collected.- """- telemetry = get_telemetry_collector()- stats = telemetry.get_tool_stats(signature_hash)- recommendations = telemetry.get_recommendations(signature_hash)-- if stats is None:- raise HTTPException(- status_code=404, detail=f"No telemetry found for signature: {signature_hash}"- )-- return {- "signature_hash": signature_hash,- "stats": stats.to_dict(),- "recommendations": recommendations,- }-- # TOIN (Tool Output Intelligence Network) endpoints- @app.get("/v1/toin/stats")- async def toin_stats():- """Get overall TOIN statistics.-- Returns aggregated statistics from the Tool Output Intelligence Network,- which learns optimal compression strategies across all tool types.-- Response includes:- - enabled: Whether TOIN is enabled- - patterns_tracked: Number of unique tool patterns being tracked- - total_compressions: Total compression events recorded- - total_retrievals: Total retrieval events recorded- - global_retrieval_rate: Overall retrieval rate (high = compression too aggressive)- - patterns_with_recommendations: Patterns with enough data for recommendations- """- toin = get_toin()- return toin.get_stats()-- @app.get("/v1/toin/patterns")- async def toin_patterns(limit: int = 20):- """List TOIN patterns with most samples.-- Returns patterns sorted by sample_size descending. Use this to see- which tool types have the most data and their learned behaviors.-- Query params:- limit: Maximum number of patterns to return (default 20)-- Response includes for each pattern:- - hash: Truncated tool signature hash (12 chars)- - compressions: Total compression events- - retrievals: Total retrieval events- - retrieval_rate: Percentage of compressions that triggered retrieval- - confidence: Confidence level in recommendations (0.0-1.0)- - skip_recommended: Whether TOIN recommends skipping compression- - optimal_max_items: Learned optimal max_items setting- """- toin = get_toin()- exported = toin.export_patterns()- patterns_data = exported.get("patterns", {})-- # Convert to list and sort by sample_size- patterns_list = []- for sig_hash, pattern_dict in patterns_data.items():- sample_size = pattern_dict.get("sample_size", 0)- total_compressions = pattern_dict.get("total_compressions", 0)- total_retrievals = pattern_dict.get("total_retrievals", 0)- retrieval_rate = (- total_retrievals / total_compressions if total_compressions > 0 else 0.0- )-- patterns_list.append(- {- "hash": sig_hash[:12],- "compressions": total_compressions,- "retrievals": total_retrievals,- "retrieval_rate": f"{retrieval_rate:.1%}",- "confidence": round(pattern_dict.get("confidence", 0.0), 3),- "skip_recommended": pattern_dict.get("skip_compression_recommended", False),- "optimal_max_items": pattern_dict.get("optimal_max_items", 20),- "sample_size": sample_size,- }- )-- # Sort by sample_size descending- patterns_list.sort(key=lambda p: p["sample_size"], reverse=True)-- # Remove sample_size from output (used only for sorting)- for p in patterns_list:- del p["sample_size"]-- return patterns_list[:limit]-- @app.get("/v1/toin/pattern/{hash_prefix}")- async def toin_pattern_detail(hash_prefix: str):- """Get detailed TOIN pattern info by hash prefix.-- Searches for a pattern where the tool signature hash starts with- the provided prefix. Returns full pattern details if found.-- Path params:- hash_prefix: Beginning of the tool signature hash (min 4 chars recommended)-- Response: Full pattern.to_dict() with all learned statistics and recommendations.- """- toin = get_toin()- exported = toin.export_patterns()- patterns_data = exported.get("patterns", {})-- # Search for pattern with matching hash prefix- for sig_hash, pattern_dict in patterns_data.items():- if sig_hash.startswith(hash_prefix):- return pattern_dict-- raise HTTPException(- status_code=404, detail=f"No TOIN pattern found with hash starting with: {hash_prefix}"- )-- @app.get("/v1/retrieve/{hash_key}")- async def ccr_retrieve_get(hash_key: str, query: str | None = None):- """GET version of CCR retrieve for easier testing."""- store = get_compression_store()-- if query:- results = store.search(hash_key, query)- return {- "hash": hash_key,- "query": query,- "results": results,- "count": len(results),- }- else:- entry = store.retrieve(hash_key)- if entry:- return {- "hash": hash_key,- "original_content": entry.original_content,- "original_tokens": entry.original_tokens,- "original_item_count": entry.original_item_count,- "compressed_item_count": entry.compressed_item_count,- "tool_name": entry.tool_name,- "retrieval_count": entry.retrieval_count,- }- raise HTTPException(status_code=404, detail="Entry not found or expired")-- # CCR Tool Call Handler - for agent frameworks to call when LLM uses headroom_retrieve- @app.post("/v1/retrieve/tool_call")- async def ccr_handle_tool_call(request: Request):- """Handle a CCR tool call from an LLM response.-- This endpoint accepts tool call formats from various providers and returns- a properly formatted tool result. Agent frameworks can use this to handle- CCR tool calls without implementing the retrieval logic themselves.-- Request body (Anthropic format):- {- "tool_call": {- "id": "toolu_123",- "name": "headroom_retrieve",- "input": {"hash": "abc123", "query": "optional search"}- },- "provider": "anthropic"- }-- Request body (OpenAI format):- {- "tool_call": {- "id": "call_123",- "function": {- "name": "headroom_retrieve",- "arguments": "{\"hash\": \"abc123\"}"- }- },- "provider": "openai"- }-- Response:- {- "tool_result": {...}, # Formatted for the provider- "success": true,- "data": {...} # Raw retrieval data- }- """- data = await request.json()- tool_call = data.get("tool_call", {})- provider = data.get("provider", "anthropic")-- # Parse the tool call- hash_key, query = parse_tool_call(tool_call, provider)-- if hash_key is None:- raise HTTPException(- status_code=400, detail=f"Invalid tool call or not a {CCR_TOOL_NAME} call"- )-- # Perform retrieval- store = get_compression_store()-- if query:- results = store.search(hash_key, query)- retrieval_data = {- "hash": hash_key,- "query": query,- "results": results,- "count": len(results),- }- else:- entry = store.retrieve(hash_key)- if entry:- retrieval_data = {- "hash": hash_key,- "original_content": entry.original_content,- "original_item_count": entry.original_item_count,- "compressed_item_count": entry.compressed_item_count,- }- else:- retrieval_data = {- "error": "Entry not found or expired (TTL: 5 minutes)",- "hash": hash_key,- }-- # Format tool result for provider- tool_call_id = tool_call.get("id", "")- result_content = json.dumps(retrieval_data, indent=2)-- if provider == "anthropic":- tool_result = {- "type": "tool_result",- "tool_use_id": tool_call_id,- "content": result_content,- }- elif provider == "openai":- tool_result = {- "role": "tool",- "tool_call_id": tool_call_id,- "content": result_content,- }- else:- tool_result = {- "tool_call_id": tool_call_id,- "content": result_content,- }-- return {- "tool_result": tool_result,- "success": "error" not in retrieval_data,- "data": retrieval_data,- }-- # Compression-only endpoint (for TypeScript SDK and other HTTP clients)- @app.post("/v1/compress")- async def compress_messages(request: Request):- return await proxy.handle_compress(request)-- register_provider_routes(app, proxy)-- return app---def _get_code_aware_banner_status(config: ProxyConfig) -> str:- """Get code-aware compression status line for banner."""- if config.code_aware_enabled:- if is_tree_sitter_available():- return "ENABLED (AST-based)"- else:- return "NOT INSTALLED (pip install headroom-ai[code])"- else:- if is_tree_sitter_available():- return "DISABLED (remove --no-code-aware to enable)"- return "DISABLED"---def run_server(- config: ProxyConfig | None = None,- workers: int = 1,- limit_concurrency: int = 1000,-):- """Run the proxy server.-- Args:- config: Proxy configuration- workers: Number of worker processes (use N for multi-core scaling)- limit_concurrency: Max concurrent connections before 503 response- """- if not FASTAPI_AVAILABLE:- print("ERROR: FastAPI required. Install: pip install fastapi uvicorn httpx")- sys.exit(1)-- config = config or ProxyConfig()- app = create_app(config)-- code_aware_status = _get_code_aware_banner_status(config)-- # Format connection pool info- pool_info = f"max={config.max_connections}, keepalive={config.max_keepalive_connections}"- http2_status = "ENABLED" if config.http2 else "DISABLED"-- backend_status = format_backend_status(- backend=config.backend,- anyllm_provider=config.anyllm_provider,- bedrock_region=config.bedrock_region,- )-- print(f"""-╔══════════════════════════════════════════════════════════════════════╗-║ HEADROOM PROXY SERVER ║-╠══════════════════════════════════════════════════════════════════════╣-║ Version: 1.0.0 ║-║ Listening: http://{config.host}:{config.port:<5} ║-║ Workers: {workers:<3} Concurrency Limit: {limit_concurrency:<5} ║-║ Backend: {backend_status:<59}║-╠══════════════════════════════════════════════════════════════════════╣-║ FEATURES: ║-║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║-║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║-║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║-║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║-║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║-║ Code-Aware: {code_aware_status:<52}║-║ HTTP/2: {http2_status:<52}║-║ Conn Pool: {pool_info:<52}║-╠══════════════════════════════════════════════════════════════════════╣-║ USAGE: ║-║ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude ║-║ Cursor: Set base URL in settings ║-╠══════════════════════════════════════════════════════════════════════╣-║ ENDPOINTS: ║-║ /livez Process liveness ║-║ /readyz Traffic readiness ║-║ /health Aggregate health ║-║ /stats Detailed statistics ║-║ /metrics Prometheus metrics ║-║ /cache/clear Clear response cache ║-║ /v1/retrieve CCR: Retrieve compressed content ║-║ /v1/retrieve/stats CCR: Compression store stats ║-║ /v1/retrieve/tool_call CCR: Handle LLM tool calls ║-║ /v1/feedback CCR: Feedback loop stats & patterns ║-║ /v1/feedback/{{tool}} CCR: Compression hints for a tool ║-║ /v1/telemetry Data flywheel: Telemetry stats ║-║ /v1/telemetry/export Data flywheel: Export for aggregation ║-║ /v1/telemetry/tools Data flywheel: Per-tool stats ║-║ /v1/toin/stats TOIN: Overall intelligence stats ║-║ /v1/toin/patterns TOIN: List learned patterns ║-║ /v1/toin/pattern/{{hash}} TOIN: Pattern details by hash ║-╚══════════════════════════════════════════════════════════════════════╝-""")-- uvicorn.run(- app,- host=config.host,- port=config.port,- log_level="warning",- workers=workers if workers > 1 else None, # None = single process (default)- limit_concurrency=limit_concurrency,- # Defense-in-depth: the loopback guard for /debug/* endpoints trusts- # request.client.host. uvicorn's ProxyHeadersMiddleware rewrites that- # from X-Forwarded-For when FORWARDED_ALLOW_IPS is broader than the- # default. Disabling proxy_headers here guarantees the guard sees the- # real peer address regardless of env.- proxy_headers=False,- )---def _get_env_bool(name: str, default: bool) -> bool:- """Get boolean from environment variable."""- val = os.environ.get(name)- if val is None:- return default- return val.lower() in ("true", "1", "yes", "on")---def _get_env_int(name: str, default: int) -> int:- """Get integer from environment variable."""- val = os.environ.get(name)- if val is None:- return default- try:- return int(val)- except ValueError:- return default---def _get_env_float(name: str, default: float) -> float:- """Get float from environment variable."""- val = os.environ.get(name)- if val is None:- return default- try:- return float(val)- except ValueError:- return default---def _get_env_str(name: str, default: str) -> str:- """Get string from environment variable."""- return os.environ.get(name, default)---def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:- """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.-- Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)- Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")-- Returns:- Dict mapping tool names to CompressionProfile instances.- """- from headroom.config import PROFILE_PRESETS, CompressionProfile-- profiles: dict[str, CompressionProfile] = {}- raw_entries: list[str] = list(cli_profiles)-- # Also check env var- env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")- if env_val:- raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())-- for entry in raw_entries:- if ":" not in entry:- logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)- continue- tool_name, level = entry.split(":", 1)- tool_name = tool_name.strip()- level = level.strip().lower()-- if level in PROFILE_PRESETS:- profiles[tool_name] = PROFILE_PRESETS[level]- else:- logger.warning(- "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",- level,- tool_name,- )-- return profiles---if __name__ == "__main__":- parser = argparse.ArgumentParser(description="Headroom Proxy Server")-- # Server- parser.add_argument("--host", default="127.0.0.1")- parser.add_argument("--port", type=int, default=8787)- parser.add_argument(- "--openai-api-url", help=f"Custom OpenAI API URL (default: {DEFAULT_OPENAI_API_URL})"- )- parser.add_argument(- "--anthropic-api-url",- help=f"Custom Anthropic API URL (default: {DEFAULT_ANTHROPIC_API_URL})",- )-- # Backend (anthropic direct, bedrock, openrouter, anyllm, or litellm-<provider>)- parser.add_argument(- "--backend",- default="anthropic",- help=(- "Backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', "- "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-hosted_vllm, litellm-vertex)"- ),- )- parser.add_argument(- "--bedrock-region",- default="us-west-2",- help="AWS region for Bedrock backend (default: us-west-2)",- )- parser.add_argument(- "--bedrock-profile",- help="AWS profile for Bedrock backend (default: use default credentials)",- )- parser.add_argument(- "--openrouter-api-key",- help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",- )- parser.add_argument(- "--anyllm-provider",- default="openai",- help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",- )-- # Connection pool (scalability)- parser.add_argument(- "--max-connections",- type=int,- default=500,- help="Max connections to upstream APIs (default: 500)",- )- parser.add_argument(- "--max-keepalive", type=int, default=100, help="Max keepalive connections (default: 100)"- )- parser.add_argument(- "--no-http2",- action="store_true",- help="Disable HTTP/2 (enabled by default for better throughput)",- )- parser.add_argument(- "--workers",- type=int,- default=1,- help="Number of worker processes (default: 1, use N for multi-core)",- )- parser.add_argument(- "--limit-concurrency",- type=int,- default=1000,- help="Max concurrent connections before 503 (default: 1000)",- )-- # Optimization- parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")- parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")- parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")- parser.add_argument(- "--tool-profile",- action="append",- default=[],- help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "- "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",- )-- # Caching- parser.add_argument("--no-cache", action="store_true", help="Disable caching")- parser.add_argument("--cache-ttl", type=int, default=3600, help="Cache TTL seconds")-- # Rate limiting- parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")- parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")- parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")-- # Cost- parser.add_argument("--budget", type=float, help="Budget limit in USD")- parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")-- # Logging- parser.add_argument("--log-file", help="Log file path")- parser.add_argument("--log-messages", action="store_true", help="Log full messages")-- # Smart routing (content-aware compression)- parser.add_argument(- "--no-smart-routing",- action="store_true",- help="Disable smart routing (use legacy sequential pipeline)",- )-- # Code-aware compression- parser.add_argument(- "--code-aware",- action="store_true",- help="Enable AST-based code compression (requires: pip install headroom-ai[code])",- )- parser.add_argument(- "--no-code-aware",- action="store_true",- help="Disable code-aware compression",- )-- args = parser.parse_args()-- # Environment variable defaults (HEADROOM_* prefix)- # CLI args override env vars, env vars override ProxyConfig defaults- env_smart_routing = _get_env_bool("HEADROOM_SMART_ROUTING", True)- env_code_aware = _get_env_bool("HEADROOM_CODE_AWARE_ENABLED", True)- env_optimize = _get_env_bool("HEADROOM_OPTIMIZE", True)- env_cache = _get_env_bool("HEADROOM_CACHE_ENABLED", True)- env_rate_limit = _get_env_bool("HEADROOM_RATE_LIMIT_ENABLED", True)-- # Determine settings: CLI flags override env vars- # --no-X explicitly disables, --X explicitly enables, neither uses env var- smart_routing = env_smart_routing if not args.no_smart_routing else False- code_aware_enabled = (- env_code_aware- if not (args.code_aware or args.no_code_aware)- else (args.code_aware or not args.no_code_aware)- )- optimize = env_optimize if not args.no_optimize else False- cache_enabled = env_cache if not args.no_cache else False- rate_limit_enabled = env_rate_limit if not args.no_rate_limit else False-- # Set OpenRouter API key from CLI if provided- if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:- os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key-- # Parse per-tool compression profiles from CLI and env var- tool_profiles = _parse_tool_profiles(args.tool_profile)-- config = ProxyConfig(- host=_get_env_str("HEADROOM_HOST", args.host),- port=_get_env_int("HEADROOM_PORT", args.port),- openai_api_url=_get_env_str("OPENAI_TARGET_API_URL", args.openai_api_url),- anthropic_api_url=_get_env_str("ANTHROPIC_TARGET_API_URL", args.anthropic_api_url),- # Backend settings- backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]- bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),- bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),- anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),- optimize=optimize,- min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),- max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),- cache_enabled=cache_enabled,- cache_ttl_seconds=_get_env_int("HEADROOM_CACHE_TTL", args.cache_ttl),- rate_limit_enabled=rate_limit_enabled,- rate_limit_requests_per_minute=_get_env_int("HEADROOM_RPM", args.rpm),- rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),- budget_limit_usd=args.budget,- budget_period=args.budget_period,- log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)- if args.log_file- else os.environ.get("HEADROOM_LOG_FILE"),- log_full_messages=args.log_messages or _get_env_bool("HEADROOM_LOG_MESSAGES", False),- smart_routing=smart_routing,- code_aware_enabled=code_aware_enabled,- # Connection pool settings- max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),- max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),- http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),- tool_profiles=tool_profiles if tool_profiles else None,- mode=normalize_proxy_mode(_get_env_str("HEADROOM_MODE", PROXY_MODE_TOKEN)),- )-- # Get worker and concurrency settings- workers = _get_env_int("HEADROOM_WORKERS", args.workers)- limit_concurrency = _get_env_int("HEADROOM_LIMIT_CONCURRENCY", args.limit_concurrency)-- run_server(config, workers=workers, limit_concurrency=limit_concurrency)+"""Headroom Proxy Server - Production Ready.++A full-featured LLM proxy with optimization, caching, rate limiting,+and observability.++Features:+- Context optimization (SmartCrusher, CacheAligner, RollingWindow)+- Semantic caching (save costs on repeated queries)+- Rate limiting (token bucket)+- Retry with exponential backoff+- Cost tracking and budgets+- Request tagging and metadata+- Provider fallback+- Prometheus metrics+- Full request/response logging++Usage:+ python -m headroom.proxy.server --port 8787++ # With Claude Code:+ ANTHROPIC_BASE_URL=http://localhost:8787 claude+"""++from __future__ import annotations++import argparse+import asyncio+import json+import logging+import os+import sys+import time+from datetime import datetime, timezone+from pathlib import Path+from typing import TYPE_CHECKING, Any, Literal++if TYPE_CHECKING:+ from ..backends.base import Backend+ from ..cache.compression_cache import CompressionCache+ from ..memory.tracker import MemoryTracker+++import httpx++try:+ import uvicorn+ from fastapi import Depends, FastAPI, HTTPException, Request, Response+ from fastapi.middleware.cors import CORSMiddleware+ from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse++ FASTAPI_AVAILABLE = True+except ImportError:+ FASTAPI_AVAILABLE = False++# Add parent to path for imports+sys.path.insert(0, str(Path(__file__).parent.parent.parent))++from headroom._version import __version__+from headroom.cache.compression_feedback import get_compression_feedback+from headroom.cache.compression_store import get_compression_store+from headroom.ccr import (+ CCR_TOOL_NAME,+ # Batch processing+ CCRResponseHandler,+ CCRToolInjector,+ ContextTracker,+ ContextTrackerConfig,+ ResponseHandlerConfig,+ parse_tool_call,+)+from headroom.config import (+ CacheAlignerConfig,+ CCRConfig,+ IntelligentContextConfig,+ ReadLifecycleConfig,+ RollingWindowConfig,+ SmartCrusherConfig,+)+from headroom.dashboard import get_dashboard_html+from headroom.observability import (+ LangfuseTracingConfig,+ OTelMetricsConfig,+ configure_langfuse_tracing,+ configure_otel_metrics,+ get_langfuse_tracing_status,+ get_otel_metrics_status,+ shutdown_headroom_tracing,+ shutdown_otel_metrics,+)+from headroom.pipeline import PipelineExtensionManager, PipelineStage+from headroom.providers.proxy_routes import register_provider_routes+from headroom.providers.registry import (+ DEFAULT_ANTHROPIC_API_URL,+ DEFAULT_CLOUDCODE_API_URL,+ DEFAULT_GEMINI_API_URL,+ DEFAULT_OPENAI_API_URL,+ build_proxy_provider_runtime,+ create_proxy_backend,+ format_backend_status,+)++# =============================================================================+# Extracted modules (re-exported for backward compatibility)+# =============================================================================+from headroom.proxy.cost import (+ _CACHE_ECONOMICS, # noqa: F401+ CostTracker, # noqa: F401+ _summarize_transforms, # noqa: F401+ build_prefix_cache_stats, # noqa: F401+ build_session_summary, # noqa: F401+ merge_cost_stats, # noqa: F401+)+from headroom.proxy.helpers import (+ COMPRESSION_TIMEOUT_SECONDS, # noqa: F401+ MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401+ MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401+ MAX_REQUEST_BODY_SIZE, # noqa: F401+ MAX_SSE_BUFFER_SIZE, # noqa: F401+ _get_image_compressor, # noqa: F401+ _get_rtk_stats, # noqa: F401+ _read_request_json, # noqa: F401+ _setup_file_logging, # noqa: F401+ is_anthropic_auth, # noqa: F401+ jitter_delay_ms,+)+from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler++# Data models (extracted to headroom/proxy/models.py for maintainability)+from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401+from headroom.proxy.modes import (+ PROXY_MODE_CACHE,+ PROXY_MODE_TOKEN,+ is_token_mode,+ normalize_proxy_mode,+)+from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401+from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401+from headroom.proxy.request_logger import RequestLogger # noqa: F401+from headroom.proxy.semantic_cache import SemanticCache # noqa: F401+from headroom.proxy.warmup import WarmupRegistry+from headroom.proxy.ws_session_registry import WebSocketSessionRegistry+from headroom.subscription.base import get_quota_registry, reset_quota_registry+from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state+from headroom.subscription.copilot_quota import get_copilot_quota_tracker+from headroom.subscription.tracker import (+ configure_subscription_tracker,+ get_subscription_tracker,+)+from headroom.telemetry import get_telemetry_collector+from headroom.telemetry.beacon import is_telemetry_enabled+from headroom.telemetry.toin import get_toin+from headroom.transforms import (+ CacheAligner,+ CodeAwareCompressor,+ CodeCompressorConfig,+ ContentRouter,+ ContentRouterConfig,+ IntelligentContextManager,+ RollingWindow,+ SmartCrusher,+ Transform,+ TransformPipeline,+ is_tree_sitter_available,+)++AnyLLMBackend: Any = None+LiteLLMBackend: Any = None++fcntl: Any = None+try:+ import fcntl as _fcntl++ fcntl = _fcntl+ HAS_FCNTL = True+except ImportError:+ HAS_FCNTL = False++_build_prefix_cache_stats = build_prefix_cache_stats+_build_session_summary = build_session_summary+_merge_cost_stats = merge_cost_stats+++logging.basicConfig(+ level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"+)+logger = logging.getLogger("headroom.proxy")++# Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis+_setup_file_logging()+++# Compression pipeline timeout in seconds+++from headroom.proxy.handlers import ( # noqa: E402+ AnthropicHandlerMixin,+ BatchHandlerMixin,+ GeminiHandlerMixin,+ OpenAIHandlerMixin,+ StreamingMixin,+)+++class HeadroomProxy(+ StreamingMixin,+ AnthropicHandlerMixin,+ OpenAIHandlerMixin,+ GeminiHandlerMixin,+ BatchHandlerMixin,+):+ """Production-ready Headroom optimization proxy."""++ ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL+ OPENAI_API_URL = DEFAULT_OPENAI_API_URL+ GEMINI_API_URL = DEFAULT_GEMINI_API_URL+ CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL++ def __init__(self, config: ProxyConfig):+ self.config = config+ self.config.mode = normalize_proxy_mode(self.config.mode)+ self.pipeline_extensions = PipelineExtensionManager(+ hooks=config.hooks,+ extensions=config.pipeline_extensions,+ discover=config.discover_pipeline_extensions,+ )++ self.provider_runtime = build_proxy_provider_runtime(config)+ api_targets = self.provider_runtime.api_targets++ # Preserve the long-standing proxy compatibility surface while keeping+ # provider_runtime as the source of truth for resolved upstream targets.+ HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic+ HeadroomProxy.OPENAI_API_URL = api_targets.openai+ HeadroomProxy.GEMINI_API_URL = api_targets.gemini+ HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode+ self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")+ self.openai_provider = self.provider_runtime.pipeline_provider("openai")++ # Initialize transforms based on routing mode+ # Choose context manager: IntelligentContextManager (smart) or RollingWindow (legacy)+ context_manager: Transform # Can be either IntelligentContextManager or RollingWindow+ if config.intelligent_context:+ # Get TOIN instance for learned pattern integration+ toin = get_toin() if config.intelligent_context_scoring else None+ context_manager = IntelligentContextManager(+ config=IntelligentContextConfig(+ enabled=True,+ keep_system=True,+ keep_last_turns=config.keep_last_turns,+ use_importance_scoring=config.intelligent_context_scoring,+ toin_integration=config.intelligent_context_scoring,+ compress_threshold=0.10 if config.intelligent_context_compress_first else 0.0,+ ),+ toin=toin,+ )+ self._context_manager_status = "intelligent"+ else:+ context_manager = RollingWindow(+ RollingWindowConfig(+ enabled=True,+ keep_system=True,+ keep_last_turns=config.keep_last_turns,+ )+ )+ self._context_manager_status = "rolling_window"++ if config.smart_routing:+ # Smart routing: ContentRouter handles all content types intelligently+ # It lazy-loads compressors only when needed+ router_config = ContentRouterConfig(+ enable_code_aware=config.code_aware_enabled,+ tool_profiles=config.tool_profiles,+ read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),+ )+ # Token mode: allow compression of older excluded-tool results+ if is_token_mode(config.mode):+ router_config.protect_recent_reads_fraction = 0.3+ transforms = [+ CacheAligner(CacheAlignerConfig(enabled=False)),+ ContentRouter(router_config),+ context_manager,+ ]+ self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"+ else:+ # Legacy mode: sequential pipeline+ transforms = [+ CacheAligner(CacheAlignerConfig(enabled=False)),+ SmartCrusher(+ SmartCrusherConfig( # type: ignore[arg-type]+ enabled=True,+ min_tokens_to_crush=config.min_tokens_to_crush,+ max_items_after_crush=config.max_items_after_crush,+ ),+ ccr_config=CCRConfig(+ enabled=config.ccr_inject_tool,+ inject_retrieval_marker=config.ccr_inject_tool, # Add CCR markers+ ),+ ),+ context_manager,+ ]+ # Add CodeAware if enabled and available+ self._code_aware_status = self._setup_code_aware(config, transforms)++ self.anthropic_pipeline = TransformPipeline(+ transforms=transforms,+ provider=self.anthropic_provider,+ )+ self.openai_pipeline = TransformPipeline(+ transforms=transforms,+ provider=self.openai_provider,+ )++ # Initialize components+ self.cache = (+ SemanticCache(+ max_entries=config.cache_max_entries,+ ttl_seconds=config.cache_ttl_seconds,+ )+ if config.cache_enabled+ else None+ )++ self.rate_limiter = (+ TokenBucketRateLimiter(+ requests_per_minute=config.rate_limit_requests_per_minute,+ tokens_per_minute=config.rate_limit_tokens_per_minute,+ )+ if config.rate_limit_enabled+ else None+ )++ self.cost_tracker = (+ CostTracker(+ budget_limit_usd=config.budget_limit_usd,+ budget_period=config.budget_period,+ )+ if config.cost_tracking_enabled+ else None+ )++ self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)++ # Prefix cache tracking: freeze already-cached messages to avoid+ # invalidating the provider's prefix cache with our transforms+ from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore++ self.session_tracker_store = SessionTrackerStore(+ default_config=PrefixFreezeConfig(+ enabled=config.prefix_freeze_enabled,+ session_ttl_seconds=config.prefix_freeze_session_ttl,+ )+ )++ # Compression cache store for token mode (session-scoped)+ self._compression_caches: dict[str, CompressionCache] = {}++ self.logger = (+ RequestLogger(+ log_file=config.log_file,+ log_full_messages=config.log_full_messages,+ )+ if config.log_requests+ else None+ )++ # Enterprise security plugin (loaded dynamically if available + licensed)+ self.security = None++ # HTTP client+ self.http_client: httpx.AsyncClient | None = None++ # Shared cold-start warmup registry (populated by startup()).+ # Holds typed slots with loaded / loading / null / error status for+ # each preloaded heavy asset. Exposed as ``proxy.warmup`` and+ # serialized by the /debug/warmup route (Unit 5).+ self.warmup: WarmupRegistry = WarmupRegistry()+ # Unit 3: live registry of Codex WS sessions. Populated by+ # ``handle_openai_responses_ws`` on accept; drained in its+ # outermost ``finally``. Consumed by ``/debug/ws-sessions``.+ self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()++ # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP+ # path. Caps how many ``handle_anthropic_messages`` calls may be+ # running deep-copy / first-stage compression / memory-context+ # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,+ # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are+ # intentionally NOT gated by this semaphore.+ #+ # A value of ``0`` or negative disables the semaphore (unbounded+ # mode); this is useful for the Unit 6 counter-factual where we+ # deliberately reproduce the original starvation. The default is+ # ``max(2, min(8, os.cpu_count() or 4))``.+ _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency+ if _pre_upstream_cfg is None:+ _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))+ else:+ _pre_upstream_resolved = _pre_upstream_cfg+ self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved+ self.anthropic_pre_upstream_acquire_timeout_seconds = float(+ config.anthropic_pre_upstream_acquire_timeout_seconds+ )+ self.anthropic_pre_upstream_memory_context_timeout_seconds = float(+ config.anthropic_pre_upstream_memory_context_timeout_seconds+ )+ if _pre_upstream_resolved > 0:+ self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = asyncio.Semaphore(+ _pre_upstream_resolved+ )+ else:+ self.anthropic_pre_upstream_sem = None++ # Backend for Anthropic API (direct, LiteLLM, or any-llm)+ # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"+ self.anthropic_backend: Backend | None = create_proxy_backend(+ backend=config.backend,+ anyllm_provider=config.anyllm_provider,+ bedrock_region=config.bedrock_region,+ logger=logger,+ anyllm_backend_cls=AnyLLMBackend,+ litellm_backend_cls=LiteLLMBackend,+ )++ # Request counter for IDs+ self._request_counter = 0+ self._request_counter_lock = asyncio.Lock()++ # CCR tool injectors (one per provider)+ self.anthropic_tool_injector = CCRToolInjector(+ provider="anthropic",+ inject_tool=config.ccr_inject_tool,+ inject_system_instructions=config.ccr_inject_system_instructions,+ )+ self.openai_tool_injector = CCRToolInjector(+ provider="openai",+ inject_tool=config.ccr_inject_tool,+ inject_system_instructions=config.ccr_inject_system_instructions,+ )++ # CCR Response Handler (handles CCR tool calls automatically)+ self.ccr_response_handler = (+ CCRResponseHandler(+ ResponseHandlerConfig(+ enabled=True,+ max_retrieval_rounds=config.ccr_max_retrieval_rounds,+ )+ )+ if config.ccr_handle_responses+ else None+ )++ # CCR Context Tracker (tracks compressed content across turns)+ self.ccr_context_tracker = (+ ContextTracker(+ ContextTrackerConfig(+ enabled=True,+ proactive_expansion=config.ccr_proactive_expansion,+ max_proactive_expansions=config.ccr_max_proactive_expansions,+ )+ )+ if config.ccr_context_tracking+ else None+ )++ # Turn counter for context tracking+ self._turn_counter = 0++ # Memory Handler (persistent user memory)+ self.memory_handler: MemoryHandler | None = None+ if config.memory_enabled:+ # Resolve memory DB path: empty → project-scoped default+ _mem_db_path = config.memory_db_path+ if not _mem_db_path:+ _mem_dir = Path.cwd() / ".headroom"+ _mem_dir.mkdir(parents=True, exist_ok=True)+ _mem_db_path = str(_mem_dir / "memory.db")+ logger.info(f"Memory: Project-scoped DB at {_mem_db_path}")++ memory_config = MemoryConfig(+ enabled=True,+ backend=config.memory_backend,+ db_path=_mem_db_path,+ inject_tools=config.memory_inject_tools,+ use_native_tool=config.memory_use_native_tool,+ inject_context=config.memory_inject_context,+ top_k=config.memory_top_k,+ min_similarity=config.memory_min_similarity,+ qdrant_host=config.memory_qdrant_host,+ qdrant_port=config.memory_qdrant_port,+ neo4j_uri=config.memory_neo4j_uri,+ neo4j_user=config.memory_neo4j_user,+ neo4j_password=config.memory_neo4j_password,+ bridge_enabled=config.memory_bridge_enabled,+ bridge_md_paths=config.memory_bridge_md_paths,+ bridge_md_format=config.memory_bridge_md_format,+ bridge_auto_import=config.memory_bridge_auto_import,+ bridge_export_path=config.memory_bridge_export_path,+ )+ self.memory_handler = MemoryHandler(+ memory_config,+ agent_type=config.traffic_learning_agent_type,+ )++ # Usage Reporter (license validation + phone-home for managed/enterprise)+ self.usage_reporter: UsageReporter | None = None+ if config.license_key:+ from headroom.telemetry.reporter import UsageReporter++ self.usage_reporter = UsageReporter(+ license_key=config.license_key,+ cloud_url=config.license_cloud_url,+ report_interval=config.license_report_interval,+ )++ # Traffic Learner (live pattern extraction from proxy traffic)+ # Only activates with --learn flag; requires --memory for backend+ self.traffic_learner: TrafficLearner | None = None+ self.traffic_learning_agent_type: str = config.traffic_learning_agent_type+ if config.traffic_learning_enabled:+ from headroom.memory.traffic_learner import TrafficLearner++ self.traffic_learner = TrafficLearner(+ user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),+ agent_type=config.traffic_learning_agent_type,+ )++ # Code graph file watcher (live reindex on file changes)+ self.code_graph_watcher: CodeGraphWatcher | None = None # type: ignore[annotation-unchecked]+ if config.code_graph_watcher:+ from headroom.graph.watcher import CodeGraphWatcher++ self.code_graph_watcher = CodeGraphWatcher(project_dir=Path.cwd())+ if self.code_graph_watcher.start():+ logger.info("Code graph: file watcher started")+ else:+ self.code_graph_watcher = None++ self.pipeline_extensions.emit(+ PipelineStage.SETUP,+ operation="proxy.setup",+ metadata={+ "mode": self.config.mode,+ "optimize": self.config.optimize,+ "backend": self.config.backend,+ "memory_enabled": self.config.memory_enabled,+ },+ )++ def _get_compression_cache(self, session_id: str) -> CompressionCache:+ """Get or create a CompressionCache for a session."""+ if session_id not in self._compression_caches:+ from headroom.cache.compression_cache import CompressionCache++ # Evict oldest caches if at capacity+ if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:+ # Remove oldest quarter to amortize cleanup cost+ oldest_keys = list(self._compression_caches.keys())[+ : MAX_COMPRESSION_CACHE_SESSIONS // 4+ ]+ for key in oldest_keys:+ del self._compression_caches[key]+ logger.info(+ "Evicted %d compression caches (exceeded %d max sessions)",+ len(oldest_keys),+ MAX_COMPRESSION_CACHE_SESSIONS,+ )++ self._compression_caches[session_id] = CompressionCache()+ return self._compression_caches[session_id]++ def _setup_code_aware(self, config: ProxyConfig, transforms: list) -> str:+ """Set up code-aware compression if enabled.++ Args:+ config: Proxy configuration+ transforms: Transform list to append to++ Returns:+ Status string for logging: 'enabled', 'disabled', 'available', 'unavailable'+ """+ if config.code_aware_enabled:+ if is_tree_sitter_available():+ code_config = CodeCompressorConfig(+ preserve_imports=True,+ preserve_signatures=True,+ preserve_type_annotations=True,+ )+ # Insert before RollingWindow (which should be last)+ transforms.insert(-1, CodeAwareCompressor(code_config))+ return "enabled"+ else:+ logger.warning(+ "Code-aware compression requested but tree-sitter not installed. "+ "Install with: pip install headroom-ai[code]"+ )+ return "unavailable"+ else:+ if is_tree_sitter_available():+ return "available" # Available but not enabled+ return "disabled"++ async def startup(self):+ """Initialize async resources."""+ self.pipeline_extensions.emit(+ PipelineStage.PRE_START,+ operation="proxy.startup",+ metadata={"port": self.config.port, "host": self.config.host},+ )+ self.http_client = httpx.AsyncClient(+ timeout=httpx.Timeout(+ connect=self.config.connect_timeout_seconds,+ read=self.config.request_timeout_seconds,+ write=self.config.request_timeout_seconds,+ pool=self.config.connect_timeout_seconds,+ ),+ limits=httpx.Limits(+ max_connections=self.config.max_connections,+ max_keepalive_connections=self.config.max_keepalive_connections,+ ),+ http2=self.config.http2,+ )+ logger.info("Headroom Proxy started")+ logger.info(f"Optimization: {'ENABLED' if self.config.optimize else 'DISABLED'}")+ self.config.mode = normalize_proxy_mode(self.config.mode)+ logger.info(f"Mode: {self.config.mode}")+ if self.config.mode == PROXY_MODE_TOKEN:+ logger.info(" Prefix freeze: re-freeze after compression")+ logger.info(" Read protection window: 30%% of excluded-tool messages")+ logger.info(" CCR TTL: extended for session lifetime")+ logger.info(" Compression cache: active")+ if self.config.mode == PROXY_MODE_CACHE:+ logger.info(" Prefix freeze: strict (all prior turns immutable)")+ logger.info(" Mutations: latest turn only")+ logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}")+ logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}")+ logger.info(+ f"Connection Pool: max_connections={self.config.max_connections}, "+ f"max_keepalive={self.config.max_keepalive_connections}, "+ f"http2={'ENABLED' if self.config.http2 else 'DISABLED'}"+ )++ # Unit 4 pre-upstream concurrency announcement. Report the resolved+ # value (auto-detected vs. explicit) so operators can correlate+ # ``pre_upstream_wait_ms`` log lines with the configured cap.+ if self.anthropic_pre_upstream_sem is None:+ logger.info("Anthropic pre-upstream concurrency: unbounded (explicitly disabled)")+ else:+ _explicit = self.config.anthropic_pre_upstream_concurrency+ _origin = "auto-detected" if _explicit is None else "explicit"+ logger.info(+ "Anthropic pre-upstream concurrency: %d (%s)",+ self.anthropic_pre_upstream_concurrency,+ _origin,+ )+ logger.info(+ "Anthropic pre-upstream timeouts: acquire=%.1fs compression=%.1fs memory_context=%.1fs",+ self.anthropic_pre_upstream_acquire_timeout_seconds,+ float(COMPRESSION_TIMEOUT_SECONDS),+ self.anthropic_pre_upstream_memory_context_timeout_seconds,+ )++ # Smart routing status+ if self.config.smart_routing:+ logger.info("Smart Routing: ENABLED (intelligent content detection)")+ else:+ logger.info("Smart Routing: DISABLED (legacy sequential mode)")++ # Eagerly load ALL compressors, parsers, and detectors at startup+ # This eliminates cold-start latency spikes on first requests.+ # Iterate BOTH pipelines (Anthropic + OpenAI) and dedupe transforms+ # by id() so shared-transform instances never load twice. The+ # resulting status dict is merged into ``self.warmup`` so /debug/warmup+ # (Unit 5) and /readyz have a single source of truth.+ self._kompress_status = "not installed"+ eager_status: dict[str, str] = {}++ if self.config.optimize:+ logger.info("Pre-loading compressors and parsers...")+ seen_transform_ids: set[int] = set()+ pipelines = (self.anthropic_pipeline, self.openai_pipeline)+ for pipeline in pipelines:+ for transform in pipeline.transforms:+ if id(transform) in seen_transform_ids:+ continue+ seen_transform_ids.add(id(transform))+ if not hasattr(transform, "eager_load_compressors"):+ continue+ try:+ transform_status = transform.eager_load_compressors()+ except Exception as exc:+ logger.warning(+ "Eager preload failed for %s: %s",+ type(transform).__name__,+ exc,+ )+ continue+ if not isinstance(transform_status, dict):+ continue+ # Merge: later writers win only if the key wasn't set.+ # Preload a transform ONCE — if another pipeline also has+ # ``eager_load_compressors`` it contributes only new keys.+ for key, value in transform_status.items():+ eager_status.setdefault(key, value)+ self.warmup.merge_transform_status(transform_status)++ # Update internal status from eager loading results+ if eager_status.get("kompress") == "enabled":+ self._kompress_status = "enabled"+ if eager_status.get("code_aware") == "enabled":+ self._code_aware_status = "enabled"++ # Log component status+ if self._kompress_status == "enabled":+ logger.info("Kompress: ENABLED (ModernBERT token compressor)")+ elif self.config.optimize:+ logger.info("Kompress: not installed (pip install headroom-ai[ml] for ML compression)")++ if self._code_aware_status == "enabled":+ logger.info("Code-Aware: ENABLED (AST-based compression)")+ if "tree_sitter" in eager_status:+ logger.info(f"Tree-Sitter: {eager_status['tree_sitter']}")+ elif self._code_aware_status == "lazy":+ logger.info("Code-Aware: LAZY (will load when code content detected)")+ elif self._code_aware_status == "available":+ logger.info("Code-Aware: available but disabled (use --code-aware)")+ elif self._code_aware_status == "unavailable":+ logger.info("Code-Aware: not installed (pip install headroom-ai[code])")+ elif self._code_aware_status == "disabled":+ logger.info("Code-Aware: DISABLED")++ if eager_status.get("magika") == "enabled":+ logger.info("Magika: ENABLED (ML content detection)")++ if self.memory_handler:+ self.warmup.memory_backend.mark_loading()+ try:+ await self.memory_handler.ensure_initialized()+ except Exception as exc: # pragma: no cover - defensive+ self.warmup.memory_backend.mark_error(str(exc))+ logger.warning("Memory: backend initialization failed (startup continues): %s", exc)+ memory_status = self.memory_handler.health_status()+ if memory_status.get("initialized"):+ self.warmup.memory_backend.mark_loaded(+ handle=self.memory_handler,+ backend=memory_status.get("backend"),+ )+ # Force one embed call so the ONNX graph is compiled now,+ # not lazily during the first request. Best-effort — any+ # failure is swallowed inside warmup_embedder.+ self.warmup.memory_embedder.mark_loading()+ warmed = await self.memory_handler.warmup_embedder()+ if warmed:+ self.warmup.memory_embedder.mark_loaded()+ else:+ # Not an error — e.g. qdrant-neo4j has no embedder slot+ # we can reach, or the backend simply exposes no handle.+ self.warmup.memory_embedder.mark_null()+ else:+ if self.warmup.memory_backend.status != "error":+ self.warmup.memory_backend.mark_null()+ self.warmup.memory_embedder.mark_null()+ logger.info(+ "Memory: ENABLED "+ f"(backend={memory_status['backend']}, initialized={memory_status['initialized']})"+ )+ else:+ logger.info("Memory: DISABLED")++ # CCR status+ ccr_features = []+ if self.config.ccr_inject_tool:+ ccr_features.append("tool_injection")+ if self.config.ccr_handle_responses:+ ccr_features.append("response_handling")+ if self.config.ccr_context_tracking:+ ccr_features.append("context_tracking")+ if self.config.ccr_proactive_expansion:+ ccr_features.append("proactive_expansion")+ if ccr_features:+ logger.info(f"CCR (Compress-Cache-Retrieve): ENABLED ({', '.join(ccr_features)})")+ else:+ logger.info("CCR: DISABLED")+ logger.info(f"Savings history: {self.metrics.savings_tracker.storage_path}")++ # Reset and rebuild the quota tracker registry for this server instance.+ # reset_quota_registry() ensures a clean slate when the proxy is restarted+ # (e.g. in tests that spin up multiple app instances in the same process).+ reset_quota_registry()+ registry = get_quota_registry()+ tracker = configure_subscription_tracker(+ poll_interval_s=self.config.subscription_poll_interval_s,+ active_window_s=self.config.subscription_active_window_s,+ enabled=self.config.subscription_tracking_enabled,+ )+ registry.register(tracker)+ registry.register(get_codex_rate_limit_state())+ registry.register(get_copilot_quota_tracker())+ await registry.start_all()++ if self.config.subscription_tracking_enabled:+ logger.info(+ "Subscription tracking: ENABLED "+ f"(poll_interval={self.config.subscription_poll_interval_s}s, "+ f"active_window={self.config.subscription_active_window_s}s)"+ )+ else:+ logger.info("Subscription tracking: DISABLED")++ copilot_tracker = get_copilot_quota_tracker()+ if copilot_tracker.is_available():+ logger.info("GitHub Copilot quota tracking: ENABLED")+ else:+ logger.info(+ "GitHub Copilot quota tracking: DISABLED "+ "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)"+ )++ # Log anonymous telemetry status so operators can see it in the log stream+ if is_telemetry_enabled():+ logger.info(+ "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). "+ "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry"+ )+ else:+ logger.info("Anonymous telemetry: DISABLED")++ self.pipeline_extensions.emit(+ PipelineStage.POST_START,+ operation="proxy.startup",+ metadata={+ "port": self.config.port,+ "host": self.config.host,+ "warmup": self.warmup.to_dict(),+ },+ )++ async def shutdown(self):+ """Cleanup async resources."""+ if self.http_client:+ await self.http_client.aclose()+ self.http_client = None++ if self.memory_handler and hasattr(self.memory_handler, "close"):+ await self.memory_handler.close()++ # Stop all quota trackers via the registry+ await get_quota_registry().stop_all()++ # Print final stats+ self._print_summary()++ def _print_summary(self):+ """Print session summary."""+ m = self.metrics+ logger.info("=" * 70)+ logger.info("HEADROOM PROXY SESSION SUMMARY")+ logger.info("=" * 70)+ logger.info(f"Total requests: {m.requests_total}")+ logger.info(f"Cached responses: {m.requests_cached}")+ logger.info(f"Rate limited: {m.requests_rate_limited}")+ logger.info(f"Failed: {m.requests_failed}")+ logger.info(f"Input tokens: {m.tokens_input_total:,}")+ logger.info(f"Output tokens: {m.tokens_output_total:,}")+ logger.info(f"Tokens saved: {m.tokens_saved_total:,}")+ if m.tokens_input_total > 0:+ savings_pct = (+ m.tokens_saved_total / (m.tokens_input_total + m.tokens_saved_total)+ ) * 100+ logger.info(f"Token savings: {savings_pct:.1f}%")+ if m.latency_count > 0:+ avg_latency = m.latency_sum_ms / m.latency_count+ logger.info(f"Avg latency: {avg_latency:.0f}ms")+ logger.info("=" * 70)++ async def _next_request_id(self) -> str:+ """Generate unique request ID."""+ async with self._request_counter_lock:+ self._request_counter += 1+ return f"hr_{int(time.time())}_{self._request_counter:06d}"++ def _extract_tags(self, headers: dict) -> dict[str, str]:+ """Extract Headroom tags from headers."""+ tags = {}+ for key, value in headers.items():+ if key.lower().startswith("x-headroom-"):+ tag_name = key.lower().replace("x-headroom-", "")+ tags[tag_name] = value+ return tags++ def _inject_system_context(+ self,+ messages: list[dict[str, Any]],+ context: str,+ body: dict[str, Any] | None = None,+ ) -> list[dict[str, Any]]:+ """Inject context into the system message/parameter.++ For Anthropic API: Uses top-level 'system' parameter (not messages array).+ For OpenAI API: Uses system role in messages array.++ Args:+ messages: The messages list.+ context: Context to inject.+ body: Optional request body to update system parameter (for Anthropic).++ Returns:+ Updated messages list.+ """+ messages = list(messages) # Copy to avoid mutation++ # For Anthropic API: use top-level 'system' parameter+ if body is not None:+ existing_system = body.get("system", "")+ if isinstance(existing_system, str):+ body["system"] = (existing_system + "\n\n" + context).strip()+ elif isinstance(existing_system, list):+ # system is a list of content blocks (e.g., with cache_control).+ # Append memory context as a new text block — never overwrite.+ body["system"] = existing_system + [{"type": "text", "text": context}]+ else:+ # No existing system prompt — set as string+ body["system"] = context+ return messages++ # For OpenAI API: use system role in messages+ for i, msg in enumerate(messages):+ if msg.get("role") == "system":+ content = msg.get("content", "")+ if isinstance(content, str):+ messages[i] = {**msg, "content": content + "\n\n" + context}+ return messages++ # No system message found - prepend one+ messages.insert(0, {"role": "system", "content": context})+ return messages++ async def _retry_request(+ self,+ method: str,+ url: str,+ headers: dict,+ body: dict,+ stream: bool = False,+ ) -> httpx.Response:+ """Make request with retry and exponential backoff."""+ last_error = None++ for attempt in range(self.config.retry_max_attempts):+ try:+ if stream:+ # For streaming, we return early - retry happens at higher level+ return await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]+ else:+ response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]++ # Don't retry client errors (4xx)+ if 400 <= response.status_code < 500:+ return response++ # Retry server errors (5xx)+ if response.status_code >= 500:+ raise httpx.HTTPStatusError(+ f"Server error: {response.status_code}",+ request=response.request,+ response=response,+ )++ return response++ except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:+ last_error = e++ if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:+ raise++ # Exponential backoff with jitter+ delay_with_jitter = jitter_delay_ms(+ self.config.retry_base_delay_ms,+ self.config.retry_max_delay_ms,+ attempt,+ )++ logger.warning(+ f"Request failed (attempt {attempt + 1}), retrying in {delay_with_jitter:.0f}ms: {e}"+ )+ await asyncio.sleep(delay_with_jitter / 1000)++ raise last_error # type: ignore[misc]+++async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:+ """Background task that logs TOIN stats periodically.++ Args:+ interval_seconds: How often to log stats (default: 5 minutes).+ """+ while True:+ await asyncio.sleep(interval_seconds)+ try:+ toin = get_toin()+ stats = toin.get_stats()+ total_compressions = stats.get("total_compressions", 0)+ if total_compressions > 0:+ patterns = stats.get("patterns_tracked", 0)+ retrievals = stats.get("total_retrievals", 0)+ retrieval_rate = stats.get("global_retrieval_rate", 0.0)+ logger.info(+ "TOIN: %d patterns, %d compressions, %d retrievals, %.1f%% retrieval rate",+ patterns,+ total_compressions,+ retrievals,+ retrieval_rate * 100,+ )+ except Exception as e:+ logger.debug("Failed to log TOIN stats: %s", e)+++def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:+ """Register all memory-tracked components with the tracker.++ This function is idempotent - it checks if components are already registered.++ Args:+ proxy: The HeadroomProxy instance.+ tracker: The MemoryTracker instance.+ """+ # Register compression store (global singleton)+ if "compression_store" not in tracker.registered_components:+ store = get_compression_store()+ tracker.register("compression_store", store.get_memory_stats)++ # Register semantic cache (instance on proxy)+ if proxy.cache and "semantic_cache" not in tracker.registered_components:+ tracker.register("semantic_cache", proxy.cache.get_memory_stats)++ # Register request logger (instance on proxy)+ if proxy.logger and "request_logger" not in tracker.registered_components:+ tracker.register("request_logger", proxy.logger.get_memory_stats)++ # Register batch context store (global singleton)+ if "batch_context_store" not in tracker.registered_components:+ try:+ from ..ccr.batch_store import get_batch_context_store++ batch_store = get_batch_context_store()+ if hasattr(batch_store, "get_memory_stats"):+ tracker.register("batch_context_store", batch_store.get_memory_stats)+ except ImportError:+ pass++ # Note: graph_store and vector_index are created per-user within the+ # LocalMemoryBackend, not as global singletons. They would need to be+ # registered when the memory system is initialized with specific backends.+++def create_app(config: ProxyConfig | None = None) -> FastAPI:+ """Create FastAPI application."""+ if not FASTAPI_AVAILABLE:+ raise ImportError("FastAPI required. Install: pip install fastapi uvicorn httpx")++ from contextlib import asynccontextmanager++ config = config or ProxyConfig()+ proxy = HeadroomProxy(config)++ # Telemetry beacon (anonymous aggregate stats).+ # With uvicorn workers > 1, each worker runs the lifespan independently.+ # We must ensure only ONE beacon runs across all workers — otherwise each+ # worker creates its own beacon, spamming the telemetry table with N rows+ # per cycle instead of 1 (all reading the same /stats from the same port).+ #+ # Strategy: use a file lock to ensure only the first worker starts the+ # beacon. Other workers see the lock and skip.+ from headroom.telemetry.beacon import TelemetryBeacon++ _beacon = TelemetryBeacon(+ port=config.port if hasattr(config, "port") else 8787,+ sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",+ backend=config.backend if hasattr(config, "backend") else "anthropic",+ )+ from headroom import paths as _hr_paths++ _beacon_lock_path = _hr_paths.beacon_lock_path(config.port)+ _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor+ _beacon_is_owner: list = [False]++ def _try_acquire_beacon_lock() -> bool:+ """Try to acquire the beacon file lock (non-blocking).++ Returns True if this process is the beacon owner.+ """+ if not HAS_FCNTL:+ return True++ fd = None+ try:+ _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)+ fd = open(_beacon_lock_path, "w") # noqa: SIM115+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)+ fd.write(str(os.getpid()))+ fd.flush()+ _beacon_lock_fd[0] = fd+ return True+ except OSError:+ if fd is not None:+ fd.close()+ return False++ def _release_beacon_lock() -> None:+ """Release the beacon file lock."""+ fd = _beacon_lock_fd[0]+ if fd:+ try:+ if HAS_FCNTL:+ fcntl.flock(fd, fcntl.LOCK_UN)+ fd.close()+ except Exception:+ pass+ _beacon_lock_fd[0] = None+ try:+ _beacon_lock_path.unlink(missing_ok=True)+ except Exception:+ pass++ @asynccontextmanager+ async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]+ configure_otel_metrics(OTelMetricsConfig.from_env(default_service_name="headroom-proxy"))+ configure_langfuse_tracing(+ LangfuseTracingConfig.from_env(default_service_name="headroom-proxy")+ )++ app.state.started_at = time.time()+ app.state.ready = False+ app.state.startup_error = None++ try:+ try:+ # Startup+ await proxy.startup()+ asyncio.create_task(_log_toin_stats_periodically())+ if proxy.usage_reporter:+ await proxy.usage_reporter.start(proxy)+ if proxy.traffic_learner:+ await proxy.traffic_learner.start()++ # Only start beacon if we acquire the lock (first worker wins)+ _beacon_is_owner[0] = _try_acquire_beacon_lock()+ if _beacon_is_owner[0]:+ await _beacon.start()+ else:+ logger.debug("Beacon: skipping (another worker owns the lock)")++ app.state.ready = True+ yield+ except Exception as exc:+ app.state.startup_error = str(exc)+ raise+ finally:+ app.state.ready = False+ # Shutdown+ if _beacon_is_owner[0]:+ await _beacon.stop()+ _release_beacon_lock()+ if proxy.usage_reporter:+ await proxy.usage_reporter.stop()+ if proxy.traffic_learner:+ await proxy.traffic_learner.stop()+ if proxy.code_graph_watcher:+ proxy.code_graph_watcher.stop()+ await proxy.shutdown()+ shutdown_headroom_tracing()+ shutdown_otel_metrics()++ app = FastAPI(+ title="Headroom Proxy",+ description="Production-ready LLM optimization proxy",+ version=__version__,+ lifespan=lifespan,+ )+ app.state.proxy = proxy+ app.state.started_at = None+ app.state.ready = False+ app.state.startup_error = None++ def _iso_utc_now() -> str:+ return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")++ def _uptime_seconds() -> float:+ started_at = getattr(app.state, "started_at", None)+ if not isinstance(started_at, int | float):+ return 0.0+ return round(max(0.0, time.time() - float(started_at)), 3)++ def _component_health(+ *,+ enabled: bool,+ ready: bool,+ **details: Any,+ ) -> dict[str, Any]:+ status = "disabled" if not enabled else ("healthy" if ready else "unhealthy")+ return {+ "enabled": enabled,+ "ready": (ready if enabled else True),+ "status": status,+ **details,+ }++ def _health_checks() -> dict[str, dict[str, Any]]:+ memory_status = (+ proxy.memory_handler.health_status()+ if proxy.memory_handler+ else {+ "enabled": False,+ "backend": None,+ "initialized": False,+ "native_tool": False,+ "bridge_enabled": False,+ }+ )+ memory_enabled = bool(memory_status.get("enabled", False))+ memory_initialized = bool(memory_status.get("initialized", False))+ return {+ "startup": _component_health(+ enabled=True,+ ready=bool(getattr(app.state, "ready", False)),+ error=getattr(app.state, "startup_error", None),+ ),+ "http_client": _component_health(+ enabled=True,+ ready=proxy.http_client is not None,+ ),+ "cache": _component_health(+ enabled=config.cache_enabled,+ ready=(proxy.cache is not None),+ ),+ "rate_limiter": _component_health(+ enabled=config.rate_limit_enabled,+ ready=(proxy.rate_limiter is not None),+ ),+ "memory": _component_health(+ enabled=memory_enabled,+ ready=memory_initialized,+ backend=memory_status["backend"],+ initialized=memory_initialized,+ native_tool=bool(memory_status.get("native_tool", False)),+ bridge_enabled=bool(memory_status.get("bridge_enabled", False)),+ ),+ }++ def _runtime_payload() -> dict[str, Any]:+ ws_registry = getattr(proxy, "ws_sessions", None)+ ws_active_sessions = ws_registry.active_count() if ws_registry is not None else 0+ ws_active_relay_tasks = (+ ws_registry.active_relay_task_count() if ws_registry is not None else 0+ )+ return {+ "anthropic_pre_upstream": {+ "enabled": proxy.anthropic_pre_upstream_sem is not None,+ "resolved_concurrency": proxy.anthropic_pre_upstream_concurrency,+ "source": (+ "auto" if config.anthropic_pre_upstream_concurrency is None else "explicit"+ ),+ "acquire_timeout_seconds": proxy.anthropic_pre_upstream_acquire_timeout_seconds,+ "compression_timeout_seconds": float(COMPRESSION_TIMEOUT_SECONDS),+ "memory_context_timeout_seconds": (+ proxy.anthropic_pre_upstream_memory_context_timeout_seconds+ ),+ "codex_ws_gated": False,+ },+ "websocket_sessions": {+ "active_sessions": ws_active_sessions,+ "active_relay_tasks": ws_active_relay_tasks,+ },+ }++ def _health_payload(*, include_config: bool) -> dict[str, Any]:+ checks = _health_checks()+ ready = all(check["ready"] for check in checks.values())+ payload: dict[str, Any] = {+ "service": "headroom-proxy",+ "status": "healthy" if ready else "unhealthy",+ "ready": ready,+ "version": __version__,+ "timestamp": _iso_utc_now(),+ "uptime_seconds": _uptime_seconds(),+ "checks": checks,+ "runtime": _runtime_payload(),+ }+ deployment_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE")+ if deployment_profile:+ payload["deployment"] = {+ "profile": deployment_profile,+ "preset": os.environ.get("HEADROOM_DEPLOYMENT_PRESET"),+ "runtime": os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME"),+ "supervisor": os.environ.get("HEADROOM_DEPLOYMENT_SUPERVISOR"),+ "scope": os.environ.get("HEADROOM_DEPLOYMENT_SCOPE"),+ }+ if include_config:+ payload["config"] = {+ "backend": config.backend,+ "optimize": config.optimize,+ "cache": config.cache_enabled,+ "rate_limit": config.rate_limit_enabled,+ "memory": config.memory_enabled,+ "learn": config.traffic_learning_enabled,+ "code_graph": config.code_graph_watcher,+ "pid": os.getpid(),+ }+ return payload++ # CORS+ app.add_middleware(+ CORSMiddleware,+ allow_origins=["*"],+ allow_credentials=True,+ allow_methods=["*"],+ allow_headers=["*"],+ )++ # X-Headroom-Stack: SDK adapters (TS openai/anthropic/etc.) tag their+ # requests so telemetry can segment by integration surface. Registered+ # before extension middleware so any extension-level auth/guards run+ # outermost and we don't count requests they reject.+ @app.middleware("http")+ async def _record_headroom_stack(request, call_next):+ if request.url.path.startswith("/v1/"):+ stack = request.headers.get("x-headroom-stack")+ if stack:+ try:+ proxy.metrics.record_stack(stack)+ except Exception:+ logger.debug("record_stack failed", exc_info=True)+ return await call_next(request)++ # Third-party proxy extensions (Enterprise, custom plugins). Discovered via+ # the `headroom.proxy_extension` entry-point group. An extension that raises+ # from its install() is a deliberate fail-closed signal and aborts startup.+ from headroom.proxy.extensions import install_all as _install_extensions++ _install_extensions(app, config)++ # Health & Metrics+ @app.get("/livez")+ async def livez():+ return JSONResponse(+ status_code=200,+ content={+ "service": "headroom-proxy",+ "status": "healthy",+ "alive": True,+ "version": __version__,+ "timestamp": _iso_utc_now(),+ "uptime_seconds": _uptime_seconds(),+ },+ )++ @app.get("/readyz")+ async def readyz():+ payload = _health_payload(include_config=False)+ return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)++ @app.get("/health")+ async def health():+ payload = _health_payload(include_config=True)+ return JSONResponse(status_code=200, content=payload)++ # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —+ # debug endpoints are invisible to external scanners.+ from headroom.proxy.debug_introspection import (+ collect_tasks as _collect_tasks,+ )+ from headroom.proxy.loopback_guard import require_loopback as _require_loopback++ @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])+ async def debug_tasks(stack: bool = False):+ """Enumerate running asyncio tasks.++ Default is cheap — ``stack_depth`` is ``null`` in every entry so+ a storm snapshot does not walk 50+ coroutine frames synchronously.+ Pass ``?stack=true`` to compute ``stack_depth`` for each task+ (useful for single-shot human debugging).+ """+ ws_registry = getattr(proxy, "ws_sessions", None)+ return JSONResponse(+ status_code=200,+ content=_collect_tasks(ws_registry, with_stack_depth=stack),+ )++ @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])+ async def debug_ws_sessions():+ ws_registry = getattr(proxy, "ws_sessions", None)+ snapshot = ws_registry.snapshot() if ws_registry is not None else []+ return JSONResponse(status_code=200, content=snapshot)++ @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])+ async def debug_warmup():+ warmup_registry = getattr(proxy, "warmup", None)+ payload = warmup_registry.to_dict() if warmup_registry is not None else {}+ payload["runtime"] = _runtime_payload()+ return JSONResponse(status_code=200, content=payload)++ @app.get("/dashboard", response_class=HTMLResponse)+ async def dashboard():+ """Serve the Headroom dashboard UI."""+ return get_dashboard_html()++ @app.get("/stats")+ async def stats():+ """Get comprehensive proxy statistics.++ This is the main stats endpoint - it aggregates data from all subsystems:+ - Request metrics (total, cached, failed, by model/provider)+ - Token usage and savings+ - Cost tracking+ - Canonical persisted display_session metrics for downstream dashboards+ - Compression (CCR) statistics+ - Telemetry/TOIN (data flywheel) statistics+ - Cache and rate limiter stats+ """+ m = proxy.metrics++ # Calculate average latency+ avg_latency_ms = round(m.latency_sum_ms / m.latency_count, 2) if m.latency_count > 0 else 0+ min_latency_ms = (+ round(m.latency_min_ms, 2)+ if m.latency_count > 0 and m.latency_min_ms != float("inf")+ else 0+ )+ max_latency_ms = round(m.latency_max_ms, 2) if m.latency_count > 0 else 0++ # Calculate Headroom overhead (optimization time only, excludes pass-through requests)+ avg_overhead_ms = (+ round(m.overhead_sum_ms / m.overhead_count, 2) if m.overhead_count > 0 else 0+ )+ min_overhead_ms = (+ round(m.overhead_min_ms, 2)+ if m.overhead_count > 0 and m.overhead_min_ms != float("inf")+ else 0+ )+ max_overhead_ms = round(m.overhead_max_ms, 2) if m.overhead_count > 0 else 0++ # Calculate TTFB (time to first byte)+ avg_ttfb_ms = round(m.ttfb_sum_ms / m.ttfb_count, 2) if m.ttfb_count > 0 else 0+ min_ttfb_ms = (+ round(m.ttfb_min_ms, 2) if m.ttfb_count > 0 and m.ttfb_min_ms != float("inf") else 0+ )+ max_ttfb_ms = round(m.ttfb_max_ms, 2) if m.ttfb_count > 0 else 0++ # Get compression store stats+ store = get_compression_store()+ compression_stats = store.get_stats()++ # Get telemetry/TOIN stats+ telemetry = get_telemetry_collector()+ telemetry_stats = telemetry.get_stats()++ # Get feedback loop stats+ feedback = get_compression_feedback()+ feedback_stats = feedback.get_stats()++ # Build prefix cache stats once (used in both prefix_cache and cost)+ prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)++ # Fetch CLI filtering savings (rtk — tokens avoided before reaching context)+ cli_filtering_stats = _get_rtk_stats()+ cli_tokens_avoided = (+ cli_filtering_stats.get("tokens_saved", 0) if cli_filtering_stats else 0+ )++ # Calculate total tokens before compression+ total_tokens_before = m.tokens_input_total + m.tokens_saved_total++ # Build human-readable summary+ summary = _build_session_summary(+ proxy, m, prefix_cache_stats, cli_tokens_avoided, total_tokens_before+ )+ # DEBUG: log the summary payload for external upsert consumers+ try:+ logger.debug("/stats summary data: %r", summary)+ except Exception:+ logger.warning("Failed to log /stats summary payload")++ # Compression cache stats (token mode)+ compression_cache_stats: dict = {}+ if proxy.config.mode == PROXY_MODE_TOKEN and proxy._compression_caches:+ total_entries = 0+ total_hits = 0+ total_misses = 0+ total_tokens_saved = 0+ for cache in proxy._compression_caches.values():+ s = cache.get_stats()+ total_entries += s.get("entries", 0)+ total_hits += s.get("hits", 0)+ total_misses += s.get("misses", 0)+ total_tokens_saved += s.get("total_tokens_saved", 0)+ compression_cache_stats = {+ "mode": PROXY_MODE_TOKEN,+ "active_sessions": len(proxy._compression_caches),+ "total_entries": total_entries,+ "total_hits": total_hits,+ "total_misses": total_misses,+ "hit_rate": round(total_hits / max(1, total_hits + total_misses) * 100, 1),+ "total_tokens_saved": total_tokens_saved,+ }+ else:+ compression_cache_stats = {"mode": proxy.config.mode}++ # Build unified savings summary (all layers)+ compression_tokens = m.tokens_saved_total+ cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)+ total_tokens_all_layers = compression_tokens + cli_tokens_avoided+ persistent_savings = m.savings_tracker.stats_preview()+ display_session = persistent_savings.get("display_session", {})++ return {+ "summary": summary,+ "savings": {+ "total_tokens": total_tokens_all_layers,+ "by_layer": {+ "cli_filtering": {+ "tokens": cli_tokens_avoided,+ "description": "Tokens avoided by CLI output filtering (rtk) before reaching context",+ },+ "compression": {+ "tokens": compression_tokens,+ "description": "Tokens removed by proxy compression (SmartCrusher, ContentRouter, etc.)",+ },+ "prefix_cache": {+ "discount_usd": round(cache_net_usd, 4),+ "description": (+ "Cost discount from provider prefix caching. "+ "Headroom's CacheAligner improves hit rates; "+ "baseline caching is provider-native."+ ),+ },+ },+ },+ "requests": {+ "total": m.requests_total,+ "cached": m.requests_cached,+ "rate_limited": m.requests_rate_limited,+ "failed": m.requests_failed,+ "by_provider": dict(m.requests_by_provider),+ "by_model": dict(m.requests_by_model),+ "by_stack": dict(m.requests_by_stack),+ },+ "tokens": {+ "input": m.tokens_input_total,+ "output": m.tokens_output_total,+ "saved": m.tokens_saved_total,+ "cli_tokens_avoided": cli_tokens_avoided,+ "total_before_compression": total_tokens_before,+ "savings_percent": round(+ (m.tokens_saved_total / total_tokens_before * 100)+ if total_tokens_before > 0+ else 0,+ 2,+ ),+ },+ "latency": {+ "average_ms": avg_latency_ms,+ "min_ms": min_latency_ms,+ "max_ms": max_latency_ms,+ "total_requests": m.latency_count,+ },+ "overhead": {+ "average_ms": avg_overhead_ms,+ "min_ms": min_overhead_ms,+ "max_ms": max_overhead_ms,+ },+ "ttfb": {+ "average_ms": avg_ttfb_ms,+ "min_ms": min_ttfb_ms,+ "max_ms": max_ttfb_ms,+ },+ "pipeline_timing": {+ name: {+ "average_ms": round(+ m.transform_timing_sum[name] / m.transform_timing_count[name], 2+ ),+ "max_ms": round(m.transform_timing_max[name], 2),+ "count": m.transform_timing_count[name],+ }+ for name in sorted(m.transform_timing_sum.keys())+ }+ if m.transform_timing_sum+ else {},+ "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},+ "savings_history": m.savings_history[-100:], # Last 100 data points+ "display_session": display_session,+ "persistent_savings": persistent_savings,+ "prefix_cache": prefix_cache_stats,+ "cost": _merge_cost_stats(+ proxy.cost_tracker.stats() if proxy.cost_tracker else None,+ prefix_cache_stats,+ cli_tokens_avoided=cli_tokens_avoided,+ ),+ "compression": {+ "ccr_entries": compression_stats.get("entry_count", 0),+ "ccr_max_entries": compression_stats.get("max_entries", 0),+ "original_tokens_cached": compression_stats.get("total_original_tokens", 0),+ "compressed_tokens_cached": compression_stats.get("total_compressed_tokens", 0),+ "ccr_retrievals": compression_stats.get("total_retrievals", 0),+ },+ "compression_cache": compression_cache_stats,+ "anon_telemetry_shipping": is_telemetry_enabled(),+ "telemetry": {+ "enabled": telemetry_stats.get("enabled", False),+ "total_compressions": telemetry_stats.get("total_compressions", 0),+ "total_retrievals": telemetry_stats.get("total_retrievals", 0),+ "global_retrieval_rate": round(telemetry_stats.get("global_retrieval_rate", 0), 4),+ "tool_signatures_tracked": telemetry_stats.get("tool_signatures_tracked", 0),+ "avg_compression_ratio": round(telemetry_stats.get("avg_compression_ratio", 0), 4),+ "avg_token_reduction": round(telemetry_stats.get("avg_token_reduction", 0), 4),+ },+ "otel": get_otel_metrics_status(),+ "langfuse": get_langfuse_tracing_status(),+ "feedback_loop": {+ "tools_tracked": feedback_stats.get("tools_tracked", 0),+ "total_compressions": feedback_stats.get("total_compressions", 0),+ "total_retrievals": feedback_stats.get("total_retrievals", 0),+ "global_retrieval_rate": round(feedback_stats.get("global_retrieval_rate", 0), 4),+ "tools_with_high_retrieval": sum(+ 1+ for p in feedback_stats.get("tool_patterns", {}).values()+ if p.get("retrieval_rate", 0) > 0.3+ ),+ },+ "toin": get_toin().get_stats(),+ "cli_filtering": cli_filtering_stats,+ "cache": await proxy.cache.stats() if proxy.cache else None,+ "rate_limiter": await proxy.rate_limiter.stats() if proxy.rate_limiter else None,+ "recent_requests": proxy.logger.get_recent(10) if proxy.logger else [],+ "log_full_messages": proxy.config.log_full_messages if proxy else False,+ **get_quota_registry().get_all_stats(),+ }++ @app.get("/stats-history")+ async def stats_history(+ format: Literal["json", "csv"] = "json",+ series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",+ history_mode: Literal["compact", "full", "none"] = "compact",+ ):+ """Get durable proxy compression history plus display-session state."""+ if format == "csv":+ filename = f"headroom-stats-history-{series}.csv"+ return Response(+ content=proxy.metrics.savings_tracker.export_csv(series=series),+ media_type="text/csv; charset=utf-8",+ headers={"Content-Disposition": f'attachment; filename="{filename}"'},+ )++ return proxy.metrics.savings_tracker.history_response(history_mode=history_mode)++ @app.get("/transformations/feed")+ async def transformations_feed(limit: int = 20):+ """Get recent message transformations for the live feed.++ Returns empty list if log_full_messages is disabled (messages are not stored).+ """+ if limit > 100:+ limit = 100++ transformations = []+ log_full_messages = proxy.config.log_full_messages if proxy else False++ if proxy and proxy.logger:+ logs = proxy.logger.get_recent_with_messages(limit)+ for log in logs:+ transformations.append(+ {+ "request_id": log.get("request_id"),+ "timestamp": log.get("timestamp"),+ "provider": log.get("provider"),+ "model": log.get("model"),+ "input_tokens_original": log.get("input_tokens_original"),+ "input_tokens_optimized": log.get("input_tokens_optimized"),+ "tokens_saved": log.get("tokens_saved"),+ "savings_percent": log.get("savings_percent"),+ "transforms_applied": log.get("transforms_applied", []),+ "request_messages": log.get("request_messages"),+ "response_content": log.get("response_content"),+ }+ )++ return {"transformations": transformations, "log_full_messages": log_full_messages}++ @app.get("/subscription-window")+ async def subscription_window():+ """Current Anthropic subscription window utilisation and Headroom contribution."""+ tracker = get_subscription_tracker()+ if tracker is None:+ return JSONResponse(+ status_code=503,+ content={"error": "Subscription tracking is not enabled"},+ )+ return JSONResponse(content=tracker.state)++ @app.get("/quota")+ async def quota():+ """Unified quota/rate-limit stats for all registered providers (Anthropic, Codex, Copilot)."""+ return JSONResponse(content=get_quota_registry().get_all_stats())++ @app.get("/metrics")+ async def metrics():+ """Prometheus metrics endpoint."""+ return PlainTextResponse(+ await proxy.metrics.export(),+ media_type="text/plain; version=0.0.4",+ )++ # Debug endpoints+ @app.get("/debug/memory")+ async def debug_memory():+ """Get detailed memory usage statistics.++ Returns memory usage for all tracked components including:+ - Process-level memory (RSS, VMS, percent)+ - Per-component memory usage and budgets+ - Cache hit/miss statistics+ - Total tracked vs target budget++ This endpoint is useful for debugging memory issues and+ monitoring memory budgets.+ """+ from ..memory.tracker import MemoryTracker++ tracker = MemoryTracker.get()++ # Register components if not already registered+ _register_memory_components(proxy, tracker)++ report = tracker.get_report()+ return report.to_dict()++ @app.post("/cache/clear")+ async def clear_cache():+ """Clear the response cache."""+ if proxy.cache:+ await proxy.cache.clear()+ return {"status": "cleared"}+ return {"status": "cache disabled"}++ # CCR (Compress-Cache-Retrieve) endpoints+ @app.post("/v1/retrieve")+ async def ccr_retrieve(request: Request):+ """Retrieve original content from CCR compression cache.++ This is the "Retrieve" part of CCR (Compress-Cache-Retrieve).+ When SmartCrusher compresses tool outputs, the original data is cached.+ LLMs can call this endpoint to get more data if needed.++ Request body:+ hash (str): Hash key from compression marker (required)+ query (str): Optional search query to filter results++ Response:+ Full retrieval: {"hash": "...", "original_content": "...", ...}+ Search: {"hash": "...", "query": "...", "results": [...], "count": N}+ """+ data = await request.json()+ hash_key = data.get("hash")+ query = data.get("query")++ if not hash_key:+ raise HTTPException(status_code=400, detail="hash required")++ store = get_compression_store()++ if query:+ # Search within cached content+ results = store.search(hash_key, query)+ return {+ "hash": hash_key,+ "query": query,+ "results": results,+ "count": len(results),+ }+ else:+ # Return full original content+ entry = store.retrieve(hash_key)+ if entry:+ return {+ "hash": hash_key,+ "original_content": entry.original_content,+ "original_tokens": entry.original_tokens,+ "original_item_count": entry.original_item_count,+ "compressed_item_count": entry.compressed_item_count,+ "tool_name": entry.tool_name,+ "retrieval_count": entry.retrieval_count,+ }+ raise HTTPException(+ status_code=404, detail="Entry not found or expired (TTL: 5 minutes)"+ )++ @app.get("/v1/retrieve/stats")+ async def ccr_stats():+ """Get CCR compression store statistics."""+ store = get_compression_store()+ stats = store.get_stats()+ events = store.get_retrieval_events(limit=20)+ return {+ "store": stats,+ "recent_retrievals": [+ {+ "hash": e.hash,+ "query": e.query,+ "items_retrieved": e.items_retrieved,+ "total_items": e.total_items,+ "tool_name": e.tool_name,+ "retrieval_type": e.retrieval_type,+ }+ for e in events+ ],+ }++ @app.get("/v1/feedback")+ async def ccr_feedback():+ """Get CCR feedback loop statistics and learned patterns.++ This endpoint exposes the feedback loop's learned patterns for monitoring+ and debugging. It shows:+ - Per-tool retrieval rates (high = compress less aggressively)+ - Common search queries per tool+ - Queried fields (suggest what to preserve)++ Use this to understand how well compression is working and whether+ the feedback loop is adjusting appropriately.+ """+ feedback = get_compression_feedback()+ stats = feedback.get_stats()+ return {+ "feedback": stats,+ "hints_example": {+ tool_name: {+ "hints": {+ "max_items": hints.max_items+ if (hints := feedback.get_compression_hints(tool_name))+ else 15,+ "suggested_items": hints.suggested_items if hints else None,+ "skip_compression": hints.skip_compression if hints else False,+ "preserve_fields": hints.preserve_fields if hints else [],+ "reason": hints.reason if hints else "",+ }+ }+ for tool_name in list(stats.get("tool_patterns", {}).keys())[:5]+ },+ }++ @app.get("/v1/feedback/{tool_name}")+ async def ccr_feedback_for_tool(tool_name: str):+ """Get compression hints for a specific tool.++ Returns feedback-based hints that would be used for compressing+ this tool's output.+ """+ feedback = get_compression_feedback()+ hints = feedback.get_compression_hints(tool_name)+ patterns = feedback.get_all_patterns().get(tool_name)++ return {+ "tool_name": tool_name,+ "hints": {+ "max_items": hints.max_items,+ "min_items": hints.min_items,+ "suggested_items": hints.suggested_items,+ "aggressiveness": hints.aggressiveness,+ "skip_compression": hints.skip_compression,+ "preserve_fields": hints.preserve_fields,+ "reason": hints.reason,+ },+ "pattern": {+ "total_compressions": patterns.total_compressions if patterns else 0,+ "total_retrievals": patterns.total_retrievals if patterns else 0,+ "retrieval_rate": patterns.retrieval_rate if patterns else 0.0,+ "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0,+ "search_rate": patterns.search_rate if patterns else 0.0,+ "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [],+ "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [],+ }+ if patterns+ else None,+ }++ # Telemetry endpoints (Data Flywheel)+ @app.get("/v1/telemetry")+ async def telemetry_stats():+ """Get telemetry statistics for the data flywheel.++ This endpoint exposes privacy-preserving telemetry data that powers+ the data flywheel - learning optimal compression strategies across+ tool types based on usage patterns.++ What's collected (anonymized):+ - Tool output structure patterns (field types, not values)+ - Compression decisions and ratios+ - Retrieval patterns (rate, type, not content)+ - Strategy effectiveness++ What's NOT collected:+ - Actual data values+ - User identifiers+ - Queries or search terms+ - File paths or tool names (hashed by default)+ """+ telemetry = get_telemetry_collector()+ return telemetry.get_stats()++ @app.get("/v1/telemetry/export")+ async def telemetry_export():+ """Export full telemetry data for aggregation.++ This endpoint exports all telemetry data in a format suitable for+ cross-user aggregation. The data is privacy-preserving - no actual+ values are included, only structural patterns and statistics.++ Use this for:+ - Building a central learning service+ - Sharing learned patterns across instances+ - Analysis and debugging+ """+ telemetry = get_telemetry_collector()+ return telemetry.export_stats()++ @app.post("/v1/telemetry/import")+ async def telemetry_import(request: Request):+ """Import telemetry data from another source.++ This allows merging telemetry from multiple sources for cross-user+ learning. The imported data is merged with existing statistics.++ Request body: Telemetry export data from /v1/telemetry/export+ """+ telemetry = get_telemetry_collector()+ data = await request.json()+ telemetry.import_stats(data)+ return {"status": "imported", "current_stats": telemetry.get_stats()}++ @app.get("/v1/telemetry/tools")+ async def telemetry_tools():+ """Get telemetry statistics for all tracked tool signatures.++ Returns statistics per tool signature (anonymized), including:+ - Compression ratios and strategy usage+ - Retrieval rates (high = compression too aggressive)+ - Learned recommendations+ """+ telemetry = get_telemetry_collector()+ all_stats = telemetry.get_all_tool_stats()+ return {+ "tool_count": len(all_stats),+ "tools": {sig_hash: stats.to_dict() for sig_hash, stats in all_stats.items()},+ }++ @app.get("/v1/telemetry/tools/{signature_hash}")+ async def telemetry_tool_detail(signature_hash: str):+ """Get detailed telemetry for a specific tool signature.++ Includes learned recommendations if enough data has been collected.+ """+ telemetry = get_telemetry_collector()+ stats = telemetry.get_tool_stats(signature_hash)+ recommendations = telemetry.get_recommendations(signature_hash)++ if stats is None:+ raise HTTPException(+ status_code=404, detail=f"No telemetry found for signature: {signature_hash}"+ )++ return {+ "signature_hash": signature_hash,+ "stats": stats.to_dict(),+ "recommendations": recommendations,+ }++ # TOIN (Tool Output Intelligence Network) endpoints+ @app.get("/v1/toin/stats")+ async def toin_stats():+ """Get overall TOIN statistics.++ Returns aggregated statistics from the Tool Output Intelligence Network,+ which learns optimal compression strategies across all tool types.++ Response includes:+ - enabled: Whether TOIN is enabled+ - patterns_tracked: Number of unique tool patterns being tracked+ - total_compressions: Total compression events recorded+ - total_retrievals: Total retrieval events recorded+ - global_retrieval_rate: Overall retrieval rate (high = compression too aggressive)+ - patterns_with_recommendations: Patterns with enough data for recommendations+ """+ toin = get_toin()+ return toin.get_stats()++ @app.get("/v1/toin/patterns")+ async def toin_patterns(limit: int = 20):+ """List TOIN patterns with most samples.++ Returns patterns sorted by sample_size descending. Use this to see+ which tool types have the most data and their learned behaviors.++ Query params:+ limit: Maximum number of patterns to return (default 20)++ Response includes for each pattern:+ - hash: Truncated tool signature hash (12 chars)+ - compressions: Total compression events+ - retrievals: Total retrieval events+ - retrieval_rate: Percentage of compressions that triggered retrieval+ - confidence: Confidence level in recommendations (0.0-1.0)+ - skip_recommended: Whether TOIN recommends skipping compression+ - optimal_max_items: Learned optimal max_items setting+ """+ toin = get_toin()+ exported = toin.export_patterns()+ patterns_data = exported.get("patterns", {})++ # Convert to list and sort by sample_size+ patterns_list = []+ for sig_hash, pattern_dict in patterns_data.items():+ sample_size = pattern_dict.get("sample_size", 0)+ total_compressions = pattern_dict.get("total_compressions", 0)+ total_retrievals = pattern_dict.get("total_retrievals", 0)+ retrieval_rate = (+ total_retrievals / total_compressions if total_compressions > 0 else 0.0+ )++ patterns_list.append(+ {+ "hash": sig_hash[:12],+ "compressions": total_compressions,+ "retrievals": total_retrievals,+ "retrieval_rate": f"{retrieval_rate:.1%}",+ "confidence": round(pattern_dict.get("confidence", 0.0), 3),+ "skip_recommended": pattern_dict.get("skip_compression_recommended", False),+ "optimal_max_items": pattern_dict.get("optimal_max_items", 20),+ "sample_size": sample_size,+ }+ )++ # Sort by sample_size descending+ patterns_list.sort(key=lambda p: p["sample_size"], reverse=True)++ # Remove sample_size from output (used only for sorting)+ for p in patterns_list:+ del p["sample_size"]++ return patterns_list[:limit]++ @app.get("/v1/toin/pattern/{hash_prefix}")+ async def toin_pattern_detail(hash_prefix: str):+ """Get detailed TOIN pattern info by hash prefix.++ Searches for a pattern where the tool signature hash starts with+ the provided prefix. Returns full pattern details if found.++ Path params:+ hash_prefix: Beginning of the tool signature hash (min 4 chars recommended)++ Response: Full pattern.to_dict() with all learned statistics and recommendations.+ """+ toin = get_toin()+ exported = toin.export_patterns()+ patterns_data = exported.get("patterns", {})++ # Search for pattern with matching hash prefix+ for sig_hash, pattern_dict in patterns_data.items():+ if sig_hash.startswith(hash_prefix):+ return pattern_dict++ raise HTTPException(+ status_code=404, detail=f"No TOIN pattern found with hash starting with: {hash_prefix}"+ )++ @app.get("/v1/retrieve/{hash_key}")+ async def ccr_retrieve_get(hash_key: str, query: str | None = None):+ """GET version of CCR retrieve for easier testing."""+ store = get_compression_store()++ if query:+ results = store.search(hash_key, query)+ return {+ "hash": hash_key,+ "query": query,+ "results": results,+ "count": len(results),+ }+ else:+ entry = store.retrieve(hash_key)+ if entry:+ return {+ "hash": hash_key,+ "original_content": entry.original_content,+ "original_tokens": entry.original_tokens,+ "original_item_count": entry.original_item_count,+ "compressed_item_count": entry.compressed_item_count,+ "tool_name": entry.tool_name,+ "retrieval_count": entry.retrieval_count,+ }+ raise HTTPException(status_code=404, detail="Entry not found or expired")++ # CCR Tool Call Handler - for agent frameworks to call when LLM uses headroom_retrieve+ @app.post("/v1/retrieve/tool_call")+ async def ccr_handle_tool_call(request: Request):+ """Handle a CCR tool call from an LLM response.++ This endpoint accepts tool call formats from various providers and returns+ a properly formatted tool result. Agent frameworks can use this to handle+ CCR tool calls without implementing the retrieval logic themselves.++ Request body (Anthropic format):+ {+ "tool_call": {+ "id": "toolu_123",+ "name": "headroom_retrieve",+ "input": {"hash": "abc123", "query": "optional search"}+ },+ "provider": "anthropic"+ }++ Request body (OpenAI format):+ {+ "tool_call": {+ "id": "call_123",+ "function": {+ "name": "headroom_retrieve",+ "arguments": "{\"hash\": \"abc123\"}"+ }+ },+ "provider": "openai"+ }++ Response:+ {+ "tool_result": {...}, # Formatted for the provider+ "success": true,+ "data": {...} # Raw retrieval data+ }+ """+ data = await request.json()+ tool_call = data.get("tool_call", {})+ provider = data.get("provider", "anthropic")++ # Parse the tool call+ hash_key, query = parse_tool_call(tool_call, provider)++ if hash_key is None:+ raise HTTPException(+ status_code=400, detail=f"Invalid tool call or not a {CCR_TOOL_NAME} call"+ )++ # Perform retrieval+ store = get_compression_store()++ if query:+ results = store.search(hash_key, query)+ retrieval_data = {+ "hash": hash_key,+ "query": query,+ "results": results,+ "count": len(results),+ }+ else:+ entry = store.retrieve(hash_key)+ if entry:+ retrieval_data = {+ "hash": hash_key,+ "original_content": entry.original_content,+ "original_item_count": entry.original_item_count,+ "compressed_item_count": entry.compressed_item_count,+ }+ else:+ retrieval_data = {+ "error": "Entry not found or expired (TTL: 5 minutes)",+ "hash": hash_key,+ }++ # Format tool result for provider+ tool_call_id = tool_call.get("id", "")+ result_content = json.dumps(retrieval_data, indent=2)++ if provider == "anthropic":+ tool_result = {+ "type": "tool_result",+ "tool_use_id": tool_call_id,+ "content": result_content,+ }+ elif provider == "openai":+ tool_result = {+ "role": "tool",+ "tool_call_id": tool_call_id,+ "content": result_content,+ }+ else:+ tool_result = {+ "tool_call_id": tool_call_id,+ "content": result_content,+ }++ return {+ "tool_result": tool_result,+ "success": "error" not in retrieval_data,+ "data": retrieval_data,+ }++ # Compression-only endpoint (for TypeScript SDK and other HTTP clients)+ @app.post("/v1/compress")+ async def compress_messages(request: Request):+ return await proxy.handle_compress(request)++ register_provider_routes(app, proxy)++ return app+++def _get_code_aware_banner_status(config: ProxyConfig) -> str:+ """Get code-aware compression status line for banner."""+ if config.code_aware_enabled:+ if is_tree_sitter_available():+ return "ENABLED (AST-based)"+ else:+ return "NOT INSTALLED (pip install headroom-ai[code])"+ else:+ if is_tree_sitter_available():+ return "DISABLED (remove --no-code-aware to enable)"+ return "DISABLED"+++def run_server(+ config: ProxyConfig | None = None,+ workers: int = 1,+ limit_concurrency: int = 1000,+):+ """Run the proxy server.++ Args:+ config: Proxy configuration+ workers: Number of worker processes (use N for multi-core scaling)+ limit_concurrency: Max concurrent connections before 503 response+ """+ if not FASTAPI_AVAILABLE:+ print("ERROR: FastAPI required. Install: pip install fastapi uvicorn httpx")+ sys.exit(1)++ config = config or ProxyConfig()+ app = create_app(config)++ code_aware_status = _get_code_aware_banner_status(config)++ # Format connection pool info+ pool_info = f"max={config.max_connections}, keepalive={config.max_keepalive_connections}"+ http2_status = "ENABLED" if config.http2 else "DISABLED"++ backend_status = format_backend_status(+ backend=config.backend,+ anyllm_provider=config.anyllm_provider,+ bedrock_region=config.bedrock_region,+ )++ print(f"""+╔══════════════════════════════════════════════════════════════════════╗+║ HEADROOM PROXY SERVER ║+╠══════════════════════════════════════════════════════════════════════╣+║ Version: 1.0.0 ║+║ Listening: http://{config.host}:{config.port:<5} ║+║ Workers: {workers:<3} Concurrency Limit: {limit_concurrency:<5} ║+║ Backend: {backend_status:<59}║+╠══════════════════════════════════════════════════════════════════════╣+║ FEATURES: ║+║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║+║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║+║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║+║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║+║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║+║ Code-Aware: {code_aware_status:<52}║+║ HTTP/2: {http2_status:<52}║+║ Conn Pool: {pool_info:<52}║+╠══════════════════════════════════════════════════════════════════════╣+║ USAGE: ║+║ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude ║+║ Cursor: Set base URL in settings ║+╠══════════════════════════════════════════════════════════════════════╣+║ ENDPOINTS: ║+║ /livez Process liveness ║+║ /readyz Traffic readiness ║+║ /health Aggregate health ║+║ /stats Detailed statistics ║+║ /metrics Prometheus metrics ║+║ /cache/clear Clear response cache ║+║ /v1/retrieve CCR: Retrieve compressed content ║+║ /v1/retrieve/stats CCR: Compression store stats ║+║ /v1/retrieve/tool_call CCR: Handle LLM tool calls ║+║ /v1/feedback CCR: Feedback loop stats & patterns ║+║ /v1/feedback/{{tool}} CCR: Compression hints for a tool ║+║ /v1/telemetry Data flywheel: Telemetry stats ║+║ /v1/telemetry/export Data flywheel: Export for aggregation ║+║ /v1/telemetry/tools Data flywheel: Per-tool stats ║+║ /v1/toin/stats TOIN: Overall intelligence stats ║+║ /v1/toin/patterns TOIN: List learned patterns ║+║ /v1/toin/pattern/{{hash}} TOIN: Pattern details by hash ║+╚══════════════════════════════════════════════════════════════════════╝+""")++ uvicorn.run(+ app,+ host=config.host,+ port=config.port,+ log_level="warning",+ workers=workers if workers > 1 else None, # None = single process (default)+ limit_concurrency=limit_concurrency,+ # Defense-in-depth: the loopback guard for /debug/* endpoints trusts+ # request.client.host. uvicorn's ProxyHeadersMiddleware rewrites that+ # from X-Forwarded-For when FORWARDED_ALLOW_IPS is broader than the+ # default. Disabling proxy_headers here guarantees the guard sees the+ # real peer address regardless of env.+ proxy_headers=False,+ )+++def _get_env_bool(name: str, default: bool) -> bool:+ """Get boolean from environment variable."""+ val = os.environ.get(name)+ if val is None:+ return default+ return val.lower() in ("true", "1", "yes", "on")+++def _get_env_int(name: str, default: int) -> int:+ """Get integer from environment variable."""+ val = os.environ.get(name)+ if val is None:+ return default+ try:+ return int(val)+ except ValueError:+ return default+++def _get_env_float(name: str, default: float) -> float:+ """Get float from environment variable."""+ val = os.environ.get(name)+ if val is None:+ return default+ try:+ return float(val)+ except ValueError:+ return default+++def _get_env_str(name: str, default: str) -> str:+ """Get string from environment variable."""+ return os.environ.get(name, default)+++def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:+ """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.++ Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)+ Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")++ Returns:+ Dict mapping tool names to CompressionProfile instances.+ """+ from headroom.config import PROFILE_PRESETS, CompressionProfile++ profiles: dict[str, CompressionProfile] = {}+ raw_entries: list[str] = list(cli_profiles)++ # Also check env var+ env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")+ if env_val:+ raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())++ for entry in raw_entries:+ if ":" not in entry:+ logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)+ continue+ tool_name, level = entry.split(":", 1)+ tool_name = tool_name.strip()+ level = level.strip().lower()++ if level in PROFILE_PRESETS:+ profiles[tool_name] = PROFILE_PRESETS[level]+ else:+ logger.warning(+ "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",+ level,+ tool_name,+ )++ return profiles+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Headroom Proxy Server")++ # Server+ parser.add_argument("--host", default="127.0.0.1")+ parser.add_argument("--port", type=int, default=8787)+ parser.add_argument(+ "--openai-api-url", help=f"Custom OpenAI API URL (default: {DEFAULT_OPENAI_API_URL})"+ )+ parser.add_argument(+ "--anthropic-api-url",+ help=f"Custom Anthropic API URL (default: {DEFAULT_ANTHROPIC_API_URL})",+ )++ # Backend (anthropic direct, bedrock, openrouter, anyllm, or litellm-<provider>)+ parser.add_argument(+ "--backend",+ default="anthropic",+ help=(+ "Backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', "+ "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-hosted_vllm, litellm-vertex)"+ ),+ )+ parser.add_argument(+ "--bedrock-region",+ default="us-west-2",+ help="AWS region for Bedrock backend (default: us-west-2)",+ )+ parser.add_argument(+ "--bedrock-profile",+ help="AWS profile for Bedrock backend (default: use default credentials)",+ )+ parser.add_argument(+ "--openrouter-api-key",+ help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",+ )+ parser.add_argument(+ "--anyllm-provider",+ default="openai",+ help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",+ )++ # Connection pool (scalability)+ parser.add_argument(+ "--max-connections",+ type=int,+ default=500,+ help="Max connections to upstream APIs (default: 500)",+ )+ parser.add_argument(+ "--max-keepalive", type=int, default=100, help="Max keepalive connections (default: 100)"+ )+ parser.add_argument(+ "--no-http2",+ action="store_true",+ help="Disable HTTP/2 (enabled by default for better throughput)",+ )+ parser.add_argument(+ "--workers",+ type=int,+ default=1,+ help="Number of worker processes (default: 1, use N for multi-core)",+ )+ parser.add_argument(+ "--limit-concurrency",+ type=int,+ default=1000,+ help="Max concurrent connections before 503 (default: 1000)",+ )++ # Optimization+ parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")+ parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")+ parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")+ parser.add_argument(+ "--tool-profile",+ action="append",+ default=[],+ help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "+ "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",+ )++ # Caching+ parser.add_argument("--no-cache", action="store_true", help="Disable caching")+ parser.add_argument("--cache-ttl", type=int, default=3600, help="Cache TTL seconds")++ # Rate limiting+ parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")+ parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")+ parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")++ # Cost+ parser.add_argument("--budget", type=float, help="Budget limit in USD")+ parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")++ # Logging+ parser.add_argument("--log-file", help="Log file path")+ parser.add_argument("--log-messages", action="store_true", help="Log full messages")++ # Smart routing (content-aware compression)+ parser.add_argument(+ "--no-smart-routing",+ action="store_true",+ help="Disable smart routing (use legacy sequential pipeline)",+ )++ # Code-aware compression+ parser.add_argument(+ "--code-aware",+ action="store_true",+ help="Enable AST-based code compression (requires: pip install headroom-ai[code])",+ )+ parser.add_argument(+ "--no-code-aware",+ action="store_true",+ help="Disable code-aware compression",+ )++ args = parser.parse_args()++ # Environment variable defaults (HEADROOM_* prefix)+ # CLI args override env vars, env vars override ProxyConfig defaults+ env_smart_routing = _get_env_bool("HEADROOM_SMART_ROUTING", True)+ env_code_aware = _get_env_bool("HEADROOM_CODE_AWARE_ENABLED", True)+ env_optimize = _get_env_bool("HEADROOM_OPTIMIZE", True)+ env_cache = _get_env_bool("HEADROOM_CACHE_ENABLED", True)+ env_rate_limit = _get_env_bool("HEADROOM_RATE_LIMIT_ENABLED", True)++ # Determine settings: CLI flags override env vars+ # --no-X explicitly disables, --X explicitly enables, neither uses env var+ smart_routing = env_smart_routing if not args.no_smart_routing else False+ code_aware_enabled = (+ env_code_aware+ if not (args.code_aware or args.no_code_aware)+ else (args.code_aware or not args.no_code_aware)+ )+ optimize = env_optimize if not args.no_optimize else False+ cache_enabled = env_cache if not args.no_cache else False+ rate_limit_enabled = env_rate_limit if not args.no_rate_limit else False++ # Set OpenRouter API key from CLI if provided+ if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:+ os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key++ # Parse per-tool compression profiles from CLI and env var+ tool_profiles = _parse_tool_profiles(args.tool_profile)++ config = ProxyConfig(+ host=_get_env_str("HEADROOM_HOST", args.host),+ port=_get_env_int("HEADROOM_PORT", args.port),+ openai_api_url=_get_env_str("OPENAI_TARGET_API_URL", args.openai_api_url),+ anthropic_api_url=_get_env_str("ANTHROPIC_TARGET_API_URL", args.anthropic_api_url),+ # Backend settings+ backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]+ bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),+ bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),+ anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),+ optimize=optimize,+ min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),+ max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),+ cache_enabled=cache_enabled,+ cache_ttl_seconds=_get_env_int("HEADROOM_CACHE_TTL", args.cache_ttl),+ rate_limit_enabled=rate_limit_enabled,+ rate_limit_requests_per_minute=_get_env_int("HEADROOM_RPM", args.rpm),+ rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),+ budget_limit_usd=args.budget,+ budget_period=args.budget_period,+ log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)+ if args.log_file+ else os.environ.get("HEADROOM_LOG_FILE"),+ log_full_messages=args.log_messages or _get_env_bool("HEADROOM_LOG_MESSAGES", False),+ smart_routing=smart_routing,+ code_aware_enabled=code_aware_enabled,+ # Connection pool settings+ max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),+ max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),+ http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),+ tool_profiles=tool_profiles if tool_profiles else None,+ mode=normalize_proxy_mode(_get_env_str("HEADROOM_MODE", PROXY_MODE_TOKEN)),+ )++ # Get worker and concurrency settings+ workers = _get_env_int("HEADROOM_WORKERS", args.workers)+ limit_concurrency = _get_env_int("HEADROOM_LIMIT_CONCURRENCY", args.limit_concurrency)++ run_server(config, workers=workers, limit_concurrency=limit_concurrency)diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.jsonindex 0a6e74107..4b2fd52e0 100644--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json@@ -1,17 +1,17 @@-{- "name": "headroom",- "version": "0.9.2",- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "author": {- "name": "Headroom Contributors",- "url": "https://github.com/chopratejas/headroom"- },- "homepage": "https://github.com/chopratejas/headroom",- "repository": "https://github.com/chopratejas/headroom",- "keywords": [- "headroom",- "hooks",- "claude-code",- "copilot-cli"- ]-}+{+ "name": "headroom",+ "version": "0.10.0",+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",+ "author": {+ "name": "Headroom Contributors",+ "url": "https://github.com/chopratejas/headroom"+ },+ "homepage": "https://github.com/chopratejas/headroom",+ "repository": "https://github.com/chopratejas/headroom",+ "keywords": [+ "headroom",+ "hooks",+ "claude-code",+ "copilot-cli"+ ]+}diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.jsonindex bdb07cc0e..f8fe831d1 100644--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json@@ -1,18 +1,18 @@-{- "name": "headroom",- "version": "0.9.2",- "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "author": {- "name": "Headroom Contributors",- "url": "https://github.com/chopratejas/headroom"- },- "homepage": "https://github.com/chopratejas/headroom",- "repository": "https://github.com/chopratejas/headroom",- "keywords": [- "headroom",- "hooks",- "claude-code",- "copilot-cli"- ],- "hooks": "./hooks"-}+{+ "name": "headroom",+ "version": "0.10.0",+ "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",+ "author": {+ "name": "Headroom Contributors",+ "url": "https://github.com/chopratejas/headroom"+ },+ "homepage": "https://github.com/chopratejas/headroom",+ "repository": "https://github.com/chopratejas/headroom",+ "keywords": [+ "headroom",+ "hooks",+ "claude-code",+ "copilot-cli"+ ],+ "hooks": "./hooks"+}diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.pyindex c7387136e..f261c95dc 100644--- a/tests/test_proxy_savings_history.py+++ b/tests/test_proxy_savings_history.py@@ -1,710 +1,711 @@-"""Tests for durable proxy savings history."""--from __future__ import annotations--import asyncio-import json-from datetime import datetime, timedelta, timezone-from types import SimpleNamespace--import pytest--pytest.importorskip("fastapi")--from fastapi.testclient import TestClient--import headroom.proxy.savings_tracker as savings_tracker_module-from headroom.proxy.savings_tracker import HEADROOM_SAVINGS_PATH_ENV_VAR, SavingsTracker-from headroom.proxy.server import ProxyConfig, create_app---def _record_request(- client: TestClient,- *,- model: str,- tokens_saved: int,- input_tokens: int = 120,-) -> None:- proxy = client.app.state.proxy- if proxy.cost_tracker:- proxy.cost_tracker.record_tokens(model, tokens_saved, input_tokens)- asyncio.run(- proxy.metrics.record_request(- provider="openai",- model=model,- input_tokens=input_tokens,- output_tokens=24,- tokens_saved=tokens_saved,- latency_ms=15.0,- )- )---def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatch):- override_path = tmp_path / "custom-savings.json"- monkeypatch.setenv(HEADROOM_SAVINGS_PATH_ENV_VAR, str(override_path))- assert savings_tracker_module.get_default_savings_storage_path() == str(override_path)-- monkeypatch.delenv(HEADROOM_SAVINGS_PATH_ENV_VAR, raising=False)- default_path = savings_tracker_module.get_default_savings_storage_path()- assert default_path.endswith(".headroom/proxy_savings.json")-- assert savings_tracker_module._parse_timestamp("") is None- assert savings_tracker_module._parse_timestamp("not-a-timestamp") is None- assert savings_tracker_module._parse_timestamp("2026-03-27T09:00:00") == datetime(- 2026, 3, 27, 9, 0, tzinfo=timezone.utc- )-- assert savings_tracker_module._coerce_int("7") == 7- assert savings_tracker_module._coerce_int(-5) == 0- assert savings_tracker_module._coerce_float("0.25") == pytest.approx(0.25)- assert savings_tracker_module._coerce_float(-0.25) == 0.0-- assert savings_tracker_module._normalize_history_entry(- ["2026-03-27T09:00:00Z", "12", "0.5"]- ) == {- "timestamp": "2026-03-27T09:00:00Z",- "total_tokens_saved": 12,- "compression_savings_usd": 0.5,- "total_input_tokens": 0,- "total_input_cost_usd": 0.0,- }- assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None- assert savings_tracker_module._normalize_history_entry(object()) is None---def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):- path = tmp_path / "proxy_savings.json"- path.write_text(- json.dumps(- {- "schema_version": 0,- "lifetime": {- "tokens_saved": 1,- "compression_savings_usd": 0.001,- },- "history": [- ["2026-03-24T08:00:00Z", 10, 0.01],- {- "timestamp": "2026-03-26T12:00:00Z",- "total_tokens_saved": 20,- "compression_savings_usd": 0.02,- },- {- "timestamp": "2026-03-27T09:00:00Z",- "total_tokens_saved": 30,- "compression_savings_usd": 0.03,- },- {"timestamp": "bad", "total_tokens_saved": 999},- ],- }- ),- encoding="utf-8",- )-- tracker = SavingsTracker(- path=str(path),- max_history_points=1,- max_history_age_days=2,- )- snapshot = tracker.snapshot()-- assert snapshot["schema_version"] == 2- assert snapshot["lifetime"] == {- "requests": 0,- "tokens_saved": 30,- "compression_savings_usd": pytest.approx(0.03),- "total_input_tokens": 0,- "total_input_cost_usd": 0.0,- }- assert snapshot["display_session"] == savings_tracker_module._empty_display_session()- assert snapshot["history"] == [- {- "timestamp": "2026-03-27T09:00:00Z",- "total_tokens_saved": 30,- "compression_savings_usd": 0.03,- "total_input_tokens": 0,- "total_input_cost_usd": 0.0,- }- ]- assert snapshot["retention"] == {- "max_history_points": 1,- "max_history_age_days": 2,- "max_response_history_points": 500,- }---def test_non_dict_savings_state_resets_to_default(tmp_path):- path = tmp_path / "proxy_savings.json"- path.write_text("[]", encoding="utf-8")-- tracker = SavingsTracker(path=str(path))- snapshot = tracker.snapshot()-- assert snapshot["lifetime"] == {- "requests": 0,- "tokens_saved": 0,- "compression_savings_usd": 0.0,- "total_input_tokens": 0,- "total_input_cost_usd": 0.0,- }- assert snapshot["display_session"] == savings_tracker_module._empty_display_session()- assert snapshot["history"] == []---def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamps(- tmp_path, monkeypatch-):- path = tmp_path / "proxy_savings.json"- tracker = SavingsTracker(path=str(path))- monkeypatch.setattr(- savings_tracker_module,- "_estimate_compression_savings_usd",- lambda model, tokens_saved: tokens_saved / 1000.0,- )-- assert tracker.record_compression_savings(model="gpt-4o", tokens_saved=0) is False- assert not path.exists()-- local_time = datetime(2026, 3, 27, 10, 0, tzinfo=timezone(timedelta(hours=2)))- assert tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=10,- total_input_tokens=120,- total_input_cost_usd=0.24,- timestamp=local_time,- )-- fallback_time = datetime(2026, 3, 27, 12, 34, tzinfo=timezone.utc)- monkeypatch.setattr(savings_tracker_module, "_utc_now", lambda: fallback_time)- assert tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=5,- total_input_tokens=180,- total_input_cost_usd=0.36,- timestamp="not-a-timestamp",- )-- snapshot = tracker.snapshot()- assert snapshot["history"] == [- {- "timestamp": "2026-03-27T08:00:00Z",- "total_tokens_saved": 10,- "compression_savings_usd": 0.01,- "total_input_tokens": 120,- "total_input_cost_usd": 0.24,- },- {- "timestamp": "2026-03-27T12:34:00Z",- "total_tokens_saved": 15,- "compression_savings_usd": 0.015,- "total_input_tokens": 180,- "total_input_cost_usd": 0.36,- },- ]-- persisted = json.loads(path.read_text(encoding="utf-8"))- assert persisted["lifetime"]["tokens_saved"] == 15- assert persisted["lifetime"]["total_input_tokens"] == 180- assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.36)- assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"---def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):- def fake_cost_per_token(*, model, prompt_tokens, completion_tokens):- if model in {"gpt-4o", "anthropic/claude-sonnet-4-6"}:- return {- "model": model,- "prompt_tokens": prompt_tokens,- "completion_tokens": completion_tokens,- }- raise RuntimeError("unknown model")-- fake_litellm = SimpleNamespace(- cost_per_token=fake_cost_per_token,- model_cost={- "anthropic/claude-sonnet-4-6": {"input_cost_per_token": 0.002},- "gpt-4o": {"input_cost_per_token": 0.001},- },- )- monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True)- monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm)-- assert savings_tracker_module._resolve_litellm_model("gpt-4o") == "gpt-4o"- assert (- savings_tracker_module._resolve_litellm_model("claude-sonnet-4-6")- == "anthropic/claude-sonnet-4-6"- )- assert savings_tracker_module._estimate_compression_savings_usd(- "claude-sonnet-4-6", 100- ) == pytest.approx(0.2)- assert savings_tracker_module._estimate_input_cost_usd(- "claude-sonnet-4-6",- 100,- cache_read_tokens=10,- cache_write_tokens=5,- uncached_input_tokens=85,- ) == pytest.approx(0.2)-- fake_litellm.model_cost = {}- assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0- assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0-- monkeypatch.setattr(- fake_litellm,- "cost_per_token",- lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom")),- )- assert savings_tracker_module._resolve_litellm_model("mystery-model") == "mystery-model"- assert savings_tracker_module._estimate_compression_savings_usd("mystery-model", 100) == 0.0-- monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)- assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0- assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0---def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests(- tmp_path, monkeypatch-):- path = tmp_path / "proxy_savings.json"- tracker = SavingsTracker(path=str(path), display_session_inactivity_minutes=30)- monkeypatch.setattr(- savings_tracker_module,- "_estimate_compression_savings_usd",- lambda model, tokens_saved: tokens_saved / 1000.0,- )- monkeypatch.setattr(- savings_tracker_module,- "_estimate_input_cost_usd",- lambda model, input_tokens, **kwargs: input_tokens / 1000.0,- )-- tracker.record_request(- model="gpt-4o",- input_tokens=120,- tokens_saved=0,- timestamp="2026-03-27T09:00:00Z",- )- tracker.record_request(- model="gpt-4o",- input_tokens=80,- tokens_saved=20,- timestamp="2026-03-27T09:10:00Z",- )-- monkeypatch.setattr(- savings_tracker_module,- "_utc_now",- lambda: datetime(2026, 3, 27, 9, 15, tzinfo=timezone.utc),- )- active_session = tracker.snapshot()["display_session"]- assert active_session == {- "requests": 2,- "tokens_saved": 20,- "compression_savings_usd": pytest.approx(0.02),- "total_input_tokens": 200,- "total_input_cost_usd": pytest.approx(0.2),- "savings_percent": pytest.approx(9.09),- "started_at": "2026-03-27T09:00:00Z",- "last_activity_at": "2026-03-27T09:10:00Z",- }-- monkeypatch.setattr(- savings_tracker_module,- "_utc_now",- lambda: datetime(2026, 3, 27, 9, 45, tzinfo=timezone.utc),- )- assert tracker.snapshot()["display_session"] == savings_tracker_module._empty_display_session()-- tracker.record_request(- model="gpt-4o",- input_tokens=50,- tokens_saved=5,- timestamp="2026-03-27T10:05:00Z",- )-- monkeypatch.setattr(- savings_tracker_module,- "_utc_now",- lambda: datetime(2026, 3, 27, 10, 10, tzinfo=timezone.utc),- )- rolled = tracker.snapshot()- assert rolled["lifetime"]["requests"] == 3- assert rolled["display_session"] == {- "requests": 1,- "tokens_saved": 5,- "compression_savings_usd": pytest.approx(0.005),- "total_input_tokens": 50,- "total_input_cost_usd": pytest.approx(0.05),- "savings_percent": pytest.approx(9.09),- "started_at": "2026-03-27T10:05:00Z",- "last_activity_at": "2026-03-27T10:05:00Z",- }---def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):- path = tmp_path / "proxy_savings.json"- tracker = SavingsTracker(- path=str(path),- max_history_points=100,- max_history_age_days=30,- )- monkeypatch.setattr(- "headroom.proxy.savings_tracker._estimate_compression_savings_usd",- lambda model, tokens_saved: tokens_saved / 1000.0,- )-- tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=100,- total_input_tokens=120,- total_input_cost_usd=0.24,- timestamp="2026-03-27T09:10:00Z",- )- tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=50,- total_input_tokens=210,- total_input_cost_usd=0.42,- timestamp="2026-03-27T09:40:00Z",- )- tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=25,- total_input_tokens=300,- total_input_cost_usd=0.63,- timestamp="2026-03-27T10:05:00Z",- )- tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=10,- total_input_tokens=360,- total_input_cost_usd=0.75,- timestamp="2026-03-28T08:00:00Z",- )- tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=20,- total_input_tokens=450,- total_input_cost_usd=0.93,- timestamp="2026-04-02T14:00:00Z",- )-- response = tracker.history_response()-- assert response["lifetime"]["tokens_saved"] == 205- assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)- assert response["lifetime"]["total_input_tokens"] == 450- assert response["lifetime"]["total_input_cost_usd"] == pytest.approx(0.93)- assert len(response["history"]) == 5-- hourly = response["series"]["hourly"]- assert [point["timestamp"] for point in hourly] == [- "2026-03-27T09:00:00Z",- "2026-03-27T10:00:00Z",- "2026-03-28T08:00:00Z",- "2026-04-02T14:00:00Z",- ]- assert hourly[0]["tokens_saved"] == 150- assert hourly[0]["total_tokens_saved"] == 150- assert hourly[0]["total_input_tokens_delta"] == 210- assert hourly[0]["total_input_tokens"] == 210- assert hourly[0]["total_input_cost_usd_delta"] == pytest.approx(0.42)- assert hourly[0]["total_input_cost_usd"] == pytest.approx(0.42)- assert hourly[1]["tokens_saved"] == 25- assert hourly[1]["total_tokens_saved"] == 175- assert hourly[1]["total_input_tokens_delta"] == 90- assert hourly[1]["total_input_tokens"] == 300- assert hourly[1]["total_input_cost_usd_delta"] == pytest.approx(0.21)- assert hourly[1]["total_input_cost_usd"] == pytest.approx(0.63)- assert hourly[2]["tokens_saved"] == 10- assert hourly[2]["total_tokens_saved"] == 185- assert hourly[2]["total_input_tokens_delta"] == 60- assert hourly[2]["total_input_tokens"] == 360- assert hourly[2]["total_input_cost_usd_delta"] == pytest.approx(0.12)- assert hourly[2]["total_input_cost_usd"] == pytest.approx(0.75)- assert hourly[3]["tokens_saved"] == 20- assert hourly[3]["total_tokens_saved"] == 205- assert hourly[3]["total_input_tokens_delta"] == 90- assert hourly[3]["total_input_tokens"] == 450- assert hourly[3]["total_input_cost_usd_delta"] == pytest.approx(0.18)- assert hourly[3]["total_input_cost_usd"] == pytest.approx(0.93)-- daily = response["series"]["daily"]- assert [point["timestamp"] for point in daily] == [- "2026-03-27T00:00:00Z",- "2026-03-28T00:00:00Z",- "2026-04-02T00:00:00Z",- ]- assert daily[0]["tokens_saved"] == 175- assert daily[0]["total_tokens_saved"] == 175- assert daily[0]["total_input_tokens_delta"] == 300- assert daily[0]["total_input_tokens"] == 300- assert daily[0]["total_input_cost_usd_delta"] == pytest.approx(0.63)- assert daily[0]["total_input_cost_usd"] == pytest.approx(0.63)- assert daily[1]["tokens_saved"] == 10- assert daily[1]["total_tokens_saved"] == 185- assert daily[1]["total_input_tokens_delta"] == 60- assert daily[1]["total_input_tokens"] == 360- assert daily[1]["total_input_cost_usd_delta"] == pytest.approx(0.12)- assert daily[1]["total_input_cost_usd"] == pytest.approx(0.75)- assert daily[2]["tokens_saved"] == 20- assert daily[2]["total_tokens_saved"] == 205- assert daily[2]["total_input_tokens_delta"] == 90- assert daily[2]["total_input_tokens"] == 450- assert daily[2]["total_input_cost_usd_delta"] == pytest.approx(0.18)- assert daily[2]["total_input_cost_usd"] == pytest.approx(0.93)-- weekly = response["series"]["weekly"]- assert [point["timestamp"] for point in weekly] == [- "2026-03-23T00:00:00Z",- "2026-03-30T00:00:00Z",- ]- assert weekly[0]["tokens_saved"] == 185- assert weekly[0]["total_tokens_saved"] == 185- assert weekly[1]["tokens_saved"] == 20- assert weekly[1]["total_tokens_saved"] == 205-- monthly = response["series"]["monthly"]- assert [point["timestamp"] for point in monthly] == [- "2026-03-01T00:00:00Z",- "2026-04-01T00:00:00Z",- ]- assert monthly[0]["tokens_saved"] == 185- assert monthly[0]["total_tokens_saved"] == 185- assert monthly[1]["tokens_saved"] == 20- assert monthly[1]["total_tokens_saved"] == 205-- assert response["exports"]["available_formats"] == ["json", "csv"]- assert response["exports"]["available_series"] == [- "history",- "hourly",- "daily",- "weekly",- "monthly",- ]---def test_stats_history_defaults_to_compact_history_but_can_return_full_history(- tmp_path, monkeypatch-):- path = tmp_path / "proxy_savings.json"- tracker = SavingsTracker(- path=str(path),- max_history_points=100,- max_history_age_days=30,- max_response_history_points=5,- )- monkeypatch.setattr(- "headroom.proxy.savings_tracker._estimate_compression_savings_usd",- lambda model, tokens_saved: tokens_saved / 1000.0,- )-- for i in range(8):- tracker.record_compression_savings(- model="gpt-4o",- tokens_saved=10,- total_input_tokens=(i + 1) * 100,- total_input_cost_usd=(i + 1) * 0.1,- timestamp=f"2026-03-27T09:{i:02d}:00Z",- )-- compact = tracker.history_response()- assert compact["history_summary"] == {- "mode": "compact",- "stored_points": 8,- "returned_points": 5,- "compacted": True,- }- assert len(compact["history"]) == 5- assert compact["history"][0]["timestamp"] == "2026-03-27T09:00:00Z"- assert compact["history"][-1]["timestamp"] == "2026-03-27T09:07:00Z"-- full = tracker.history_response(history_mode="full")- assert full["history_summary"] == {- "mode": "full",- "stored_points": 8,- "returned_points": 8,- "compacted": False,- }- assert len(full["history"]) == 8-- none = tracker.history_response(history_mode="none")- assert none["history"] == []- assert none["history_summary"] == {- "mode": "none",- "stored_points": 8,- "returned_points": 0,- "compacted": True,- }---def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):- savings_path = tmp_path / "proxy_savings.json"- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))- monkeypatch.setattr(- "headroom.proxy.server.CostTracker._get_cache_prices",- lambda self, model: (0.001, 0.0015, 0.002),- )-- config = ProxyConfig(- cache_enabled=False,- rate_limit_enabled=False,- log_requests=False,- )-- with TestClient(create_app(config)) as client:- _record_request(client, model="gpt-4o", tokens_saved=40)-- stats = client.get("/stats")- assert stats.status_code == 200- stats_data = stats.json()- assert "savings_history" in stats_data- assert "persistent_savings" in stats_data- assert all(len(point) == 2 for point in stats_data["savings_history"])- assert stats_data["persistent_savings"]["lifetime"]["tokens_saved"] == 40- assert stats_data["persistent_savings"]["storage_path"] == str(savings_path)-- history = client.get("/stats-history")- assert history.status_code == 200- history_data = history.json()- assert history_data["schema_version"] == 2- assert history_data["storage_path"] == str(savings_path)- assert history_data["lifetime"]["tokens_saved"] == 40- assert history_data["lifetime"]["total_input_tokens"] == 120- assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)- assert history_data["display_session"]["requests"] == 1- assert history_data["display_session"]["tokens_saved"] == 40- assert history_data["display_session"]["total_input_tokens"] == 120- assert history_data["display_session"]["savings_percent"] == pytest.approx(25.0)- assert list(history_data["series"].keys()) == [- "hourly",- "daily",- "weekly",- "monthly",- ]- assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]- assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120- assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(- 0.24- )- assert history_data["history_summary"] == {- "mode": "compact",- "stored_points": 1,- "returned_points": 1,- "compacted": False,- }-- assert stats_data["display_session"] == history_data["display_session"]- assert (- stats_data["persistent_savings"]["display_session"] == history_data["display_session"]- )-- with TestClient(create_app(config)) as client:- history = client.get("/stats-history")- assert history.status_code == 200- assert history.json()["lifetime"]["tokens_saved"] == 40- assert history.json()["display_session"]["requests"] == 1-- _record_request(client, model="gpt-4o", tokens_saved=15)-- updated = client.get("/stats-history").json()- assert updated["lifetime"]["tokens_saved"] == 55- assert updated["lifetime"]["total_input_tokens"] == 240- assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)- assert updated["lifetime"]["requests"] == 2- assert len(updated["history"]) == 2- assert updated["display_session"]["requests"] == 2- assert updated["display_session"]["tokens_saved"] == 55- assert updated["display_session"]["total_input_tokens"] == 240- assert updated["display_session"]["savings_percent"] == pytest.approx(18.64)- assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240- assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)-- full = client.get("/stats-history?history_mode=full").json()- assert full["history_summary"]["mode"] == "full"- assert full["history_summary"]["stored_points"] == 2- assert full["history_summary"]["returned_points"] == 2-- persisted = json.loads(savings_path.read_text())- assert persisted["lifetime"]["tokens_saved"] == 55- assert persisted["lifetime"]["total_input_tokens"] == 240- assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)- assert persisted["display_session"]["requests"] == 2---def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):- savings_path = tmp_path / "proxy_savings.json"- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))- monkeypatch.setattr(- "headroom.proxy.server.CostTracker._get_cache_prices",- lambda self, model: (0.001, 0.0015, 0.002),- )-- config = ProxyConfig(- cache_enabled=False,- rate_limit_enabled=False,- log_requests=False,- )-- with TestClient(create_app(config)) as client:- _record_request(client, model="gpt-4o", tokens_saved=40)- _record_request(client, model="gpt-4o", tokens_saved=10)-- response = client.get("/stats-history?format=csv&series=daily")- assert response.status_code == 200- assert response.headers["content-type"].startswith("text/csv")- assert (- 'attachment; filename="headroom-stats-history-daily.csv"'- == response.headers["content-disposition"]- )- lines = response.text.strip().splitlines()- assert lines[0] == (- "timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"- "compression_savings_usd,total_input_tokens_delta,total_input_tokens,"- "total_input_cost_usd_delta,total_input_cost_usd"- )- assert len(lines) >= 2- assert "total_tokens_saved" in lines[0]- assert "total_input_cost_usd" in lines[0]---def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):- savings_path = tmp_path / "proxy_savings.json"- savings_path.write_text("{not valid json", encoding="utf-8")- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))-- config = ProxyConfig(- cache_enabled=False,- rate_limit_enabled=False,- log_requests=False,- )-- with TestClient(create_app(config)) as client:- response = client.get("/stats-history")- assert response.status_code == 200- data = response.json()- assert data["lifetime"]["tokens_saved"] == 0- assert data["history"] == []---def test_dashboard_includes_history_toggle_and_endpoint(tmp_path, monkeypatch):- savings_path = tmp_path / "proxy_savings.json"- monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))-- config = ProxyConfig(- cache_enabled=False,- rate_limit_enabled=False,- log_requests=False,- )-- with TestClient(create_app(config)) as client:- response = client.get("/dashboard")- assert response.status_code == 200- html = response.text- assert "Session" in html- assert "Historical" in html- assert "fetch('/stats-history')" in html- assert "Export CSV" in html- assert "Weekly Savings" in html- assert "Monthly Savings" in html+"""Tests for durable proxy savings history."""++from __future__ import annotations++import asyncio+import json+from datetime import datetime, timedelta, timezone+from pathlib import Path+from types import SimpleNamespace++import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient++import headroom.proxy.savings_tracker as savings_tracker_module+from headroom.proxy.savings_tracker import HEADROOM_SAVINGS_PATH_ENV_VAR, SavingsTracker+from headroom.proxy.server import ProxyConfig, create_app+++def _record_request(+ client: TestClient,+ *,+ model: str,+ tokens_saved: int,+ input_tokens: int = 120,+) -> None:+ proxy = client.app.state.proxy+ if proxy.cost_tracker:+ proxy.cost_tracker.record_tokens(model, tokens_saved, input_tokens)+ asyncio.run(+ proxy.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=input_tokens,+ output_tokens=24,+ tokens_saved=tokens_saved,+ latency_ms=15.0,+ )+ )+++def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatch):+ override_path = tmp_path / "custom-savings.json"+ monkeypatch.setenv(HEADROOM_SAVINGS_PATH_ENV_VAR, str(override_path))+ assert savings_tracker_module.get_default_savings_storage_path() == str(override_path)++ monkeypatch.delenv(HEADROOM_SAVINGS_PATH_ENV_VAR, raising=False)+ default_path = savings_tracker_module.get_default_savings_storage_path()+ assert Path(default_path).as_posix().endswith(".headroom/proxy_savings.json")++ assert savings_tracker_module._parse_timestamp("") is None+ assert savings_tracker_module._parse_timestamp("not-a-timestamp") is None+ assert savings_tracker_module._parse_timestamp("2026-03-27T09:00:00") == datetime(+ 2026, 3, 27, 9, 0, tzinfo=timezone.utc+ )++ assert savings_tracker_module._coerce_int("7") == 7+ assert savings_tracker_module._coerce_int(-5) == 0+ assert savings_tracker_module._coerce_float("0.25") == pytest.approx(0.25)+ assert savings_tracker_module._coerce_float(-0.25) == 0.0++ assert savings_tracker_module._normalize_history_entry(+ ["2026-03-27T09:00:00Z", "12", "0.5"]+ ) == {+ "timestamp": "2026-03-27T09:00:00Z",+ "total_tokens_saved": 12,+ "compression_savings_usd": 0.5,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,+ }+ assert savings_tracker_module._normalize_history_entry({"timestamp": "bad"}) is None+ assert savings_tracker_module._normalize_history_entry(object()) is None+++def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):+ path = tmp_path / "proxy_savings.json"+ path.write_text(+ json.dumps(+ {+ "schema_version": 0,+ "lifetime": {+ "tokens_saved": 1,+ "compression_savings_usd": 0.001,+ },+ "history": [+ ["2026-03-24T08:00:00Z", 10, 0.01],+ {+ "timestamp": "2026-03-26T12:00:00Z",+ "total_tokens_saved": 20,+ "compression_savings_usd": 0.02,+ },+ {+ "timestamp": "2026-03-27T09:00:00Z",+ "total_tokens_saved": 30,+ "compression_savings_usd": 0.03,+ },+ {"timestamp": "bad", "total_tokens_saved": 999},+ ],+ }+ ),+ encoding="utf-8",+ )++ tracker = SavingsTracker(+ path=str(path),+ max_history_points=1,+ max_history_age_days=2,+ )+ snapshot = tracker.snapshot()++ assert snapshot["schema_version"] == 2+ assert snapshot["lifetime"] == {+ "requests": 0,+ "tokens_saved": 30,+ "compression_savings_usd": pytest.approx(0.03),+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,+ }+ assert snapshot["display_session"] == savings_tracker_module._empty_display_session()+ assert snapshot["history"] == [+ {+ "timestamp": "2026-03-27T09:00:00Z",+ "total_tokens_saved": 30,+ "compression_savings_usd": 0.03,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,+ }+ ]+ assert snapshot["retention"] == {+ "max_history_points": 1,+ "max_history_age_days": 2,+ "max_response_history_points": 500,+ }+++def test_non_dict_savings_state_resets_to_default(tmp_path):+ path = tmp_path / "proxy_savings.json"+ path.write_text("[]", encoding="utf-8")++ tracker = SavingsTracker(path=str(path))+ snapshot = tracker.snapshot()++ assert snapshot["lifetime"] == {+ "requests": 0,+ "tokens_saved": 0,+ "compression_savings_usd": 0.0,+ "total_input_tokens": 0,+ "total_input_cost_usd": 0.0,+ }+ assert snapshot["display_session"] == savings_tracker_module._empty_display_session()+ assert snapshot["history"] == []+++def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamps(+ tmp_path, monkeypatch+):+ path = tmp_path / "proxy_savings.json"+ tracker = SavingsTracker(path=str(path))+ monkeypatch.setattr(+ savings_tracker_module,+ "_estimate_compression_savings_usd",+ lambda model, tokens_saved: tokens_saved / 1000.0,+ )++ assert tracker.record_compression_savings(model="gpt-4o", tokens_saved=0) is False+ assert not path.exists()++ local_time = datetime(2026, 3, 27, 10, 0, tzinfo=timezone(timedelta(hours=2)))+ assert tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=10,+ total_input_tokens=120,+ total_input_cost_usd=0.24,+ timestamp=local_time,+ )++ fallback_time = datetime(2026, 3, 27, 12, 34, tzinfo=timezone.utc)+ monkeypatch.setattr(savings_tracker_module, "_utc_now", lambda: fallback_time)+ assert tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=5,+ total_input_tokens=180,+ total_input_cost_usd=0.36,+ timestamp="not-a-timestamp",+ )++ snapshot = tracker.snapshot()+ assert snapshot["history"] == [+ {+ "timestamp": "2026-03-27T08:00:00Z",+ "total_tokens_saved": 10,+ "compression_savings_usd": 0.01,+ "total_input_tokens": 120,+ "total_input_cost_usd": 0.24,+ },+ {+ "timestamp": "2026-03-27T12:34:00Z",+ "total_tokens_saved": 15,+ "compression_savings_usd": 0.015,+ "total_input_tokens": 180,+ "total_input_cost_usd": 0.36,+ },+ ]++ persisted = json.loads(path.read_text(encoding="utf-8"))+ assert persisted["lifetime"]["tokens_saved"] == 15+ assert persisted["lifetime"]["total_input_tokens"] == 180+ assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.36)+ assert persisted["history"][-1]["timestamp"] == "2026-03-27T12:34:00Z"+++def test_litellm_resolution_and_savings_estimation_fallbacks(monkeypatch):+ def fake_cost_per_token(*, model, prompt_tokens, completion_tokens):+ if model in {"gpt-4o", "anthropic/claude-sonnet-4-6"}:+ return {+ "model": model,+ "prompt_tokens": prompt_tokens,+ "completion_tokens": completion_tokens,+ }+ raise RuntimeError("unknown model")++ fake_litellm = SimpleNamespace(+ cost_per_token=fake_cost_per_token,+ model_cost={+ "anthropic/claude-sonnet-4-6": {"input_cost_per_token": 0.002},+ "gpt-4o": {"input_cost_per_token": 0.001},+ },+ )+ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", True)+ monkeypatch.setattr(savings_tracker_module, "litellm", fake_litellm)++ assert savings_tracker_module._resolve_litellm_model("gpt-4o") == "gpt-4o"+ assert (+ savings_tracker_module._resolve_litellm_model("claude-sonnet-4-6")+ == "anthropic/claude-sonnet-4-6"+ )+ assert savings_tracker_module._estimate_compression_savings_usd(+ "claude-sonnet-4-6", 100+ ) == pytest.approx(0.2)+ assert savings_tracker_module._estimate_input_cost_usd(+ "claude-sonnet-4-6",+ 100,+ cache_read_tokens=10,+ cache_write_tokens=5,+ uncached_input_tokens=85,+ ) == pytest.approx(0.2)++ fake_litellm.model_cost = {}+ assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0+ assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0++ monkeypatch.setattr(+ fake_litellm,+ "cost_per_token",+ lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom")),+ )+ assert savings_tracker_module._resolve_litellm_model("mystery-model") == "mystery-model"+ assert savings_tracker_module._estimate_compression_savings_usd("mystery-model", 100) == 0.0++ monkeypatch.setattr(savings_tracker_module, "LITELLM_AVAILABLE", False)+ assert savings_tracker_module._estimate_compression_savings_usd("gpt-4o", 100) == 0.0+ assert savings_tracker_module._estimate_input_cost_usd("gpt-4o", 100) == 0.0+++def test_display_session_rolls_after_inactivity_and_counts_zero_savings_requests(+ tmp_path, monkeypatch+):+ path = tmp_path / "proxy_savings.json"+ tracker = SavingsTracker(path=str(path), display_session_inactivity_minutes=30)+ monkeypatch.setattr(+ savings_tracker_module,+ "_estimate_compression_savings_usd",+ lambda model, tokens_saved: tokens_saved / 1000.0,+ )+ monkeypatch.setattr(+ savings_tracker_module,+ "_estimate_input_cost_usd",+ lambda model, input_tokens, **kwargs: input_tokens / 1000.0,+ )++ tracker.record_request(+ model="gpt-4o",+ input_tokens=120,+ tokens_saved=0,+ timestamp="2026-03-27T09:00:00Z",+ )+ tracker.record_request(+ model="gpt-4o",+ input_tokens=80,+ tokens_saved=20,+ timestamp="2026-03-27T09:10:00Z",+ )++ monkeypatch.setattr(+ savings_tracker_module,+ "_utc_now",+ lambda: datetime(2026, 3, 27, 9, 15, tzinfo=timezone.utc),+ )+ active_session = tracker.snapshot()["display_session"]+ assert active_session == {+ "requests": 2,+ "tokens_saved": 20,+ "compression_savings_usd": pytest.approx(0.02),+ "total_input_tokens": 200,+ "total_input_cost_usd": pytest.approx(0.2),+ "savings_percent": pytest.approx(9.09),+ "started_at": "2026-03-27T09:00:00Z",+ "last_activity_at": "2026-03-27T09:10:00Z",+ }++ monkeypatch.setattr(+ savings_tracker_module,+ "_utc_now",+ lambda: datetime(2026, 3, 27, 9, 45, tzinfo=timezone.utc),+ )+ assert tracker.snapshot()["display_session"] == savings_tracker_module._empty_display_session()++ tracker.record_request(+ model="gpt-4o",+ input_tokens=50,+ tokens_saved=5,+ timestamp="2026-03-27T10:05:00Z",+ )++ monkeypatch.setattr(+ savings_tracker_module,+ "_utc_now",+ lambda: datetime(2026, 3, 27, 10, 10, tzinfo=timezone.utc),+ )+ rolled = tracker.snapshot()+ assert rolled["lifetime"]["requests"] == 3+ assert rolled["display_session"] == {+ "requests": 1,+ "tokens_saved": 5,+ "compression_savings_usd": pytest.approx(0.005),+ "total_input_tokens": 50,+ "total_input_cost_usd": pytest.approx(0.05),+ "savings_percent": pytest.approx(9.09),+ "started_at": "2026-03-27T10:05:00Z",+ "last_activity_at": "2026-03-27T10:05:00Z",+ }+++def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monkeypatch):+ path = tmp_path / "proxy_savings.json"+ tracker = SavingsTracker(+ path=str(path),+ max_history_points=100,+ max_history_age_days=30,+ )+ monkeypatch.setattr(+ "headroom.proxy.savings_tracker._estimate_compression_savings_usd",+ lambda model, tokens_saved: tokens_saved / 1000.0,+ )++ tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=100,+ total_input_tokens=120,+ total_input_cost_usd=0.24,+ timestamp="2026-03-27T09:10:00Z",+ )+ tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=50,+ total_input_tokens=210,+ total_input_cost_usd=0.42,+ timestamp="2026-03-27T09:40:00Z",+ )+ tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=25,+ total_input_tokens=300,+ total_input_cost_usd=0.63,+ timestamp="2026-03-27T10:05:00Z",+ )+ tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=10,+ total_input_tokens=360,+ total_input_cost_usd=0.75,+ timestamp="2026-03-28T08:00:00Z",+ )+ tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=20,+ total_input_tokens=450,+ total_input_cost_usd=0.93,+ timestamp="2026-04-02T14:00:00Z",+ )++ response = tracker.history_response()++ assert response["lifetime"]["tokens_saved"] == 205+ assert response["lifetime"]["compression_savings_usd"] == pytest.approx(0.205)+ assert response["lifetime"]["total_input_tokens"] == 450+ assert response["lifetime"]["total_input_cost_usd"] == pytest.approx(0.93)+ assert len(response["history"]) == 5++ hourly = response["series"]["hourly"]+ assert [point["timestamp"] for point in hourly] == [+ "2026-03-27T09:00:00Z",+ "2026-03-27T10:00:00Z",+ "2026-03-28T08:00:00Z",+ "2026-04-02T14:00:00Z",+ ]+ assert hourly[0]["tokens_saved"] == 150+ assert hourly[0]["total_tokens_saved"] == 150+ assert hourly[0]["total_input_tokens_delta"] == 210+ assert hourly[0]["total_input_tokens"] == 210+ assert hourly[0]["total_input_cost_usd_delta"] == pytest.approx(0.42)+ assert hourly[0]["total_input_cost_usd"] == pytest.approx(0.42)+ assert hourly[1]["tokens_saved"] == 25+ assert hourly[1]["total_tokens_saved"] == 175+ assert hourly[1]["total_input_tokens_delta"] == 90+ assert hourly[1]["total_input_tokens"] == 300+ assert hourly[1]["total_input_cost_usd_delta"] == pytest.approx(0.21)+ assert hourly[1]["total_input_cost_usd"] == pytest.approx(0.63)+ assert hourly[2]["tokens_saved"] == 10+ assert hourly[2]["total_tokens_saved"] == 185+ assert hourly[2]["total_input_tokens_delta"] == 60+ assert hourly[2]["total_input_tokens"] == 360+ assert hourly[2]["total_input_cost_usd_delta"] == pytest.approx(0.12)+ assert hourly[2]["total_input_cost_usd"] == pytest.approx(0.75)+ assert hourly[3]["tokens_saved"] == 20+ assert hourly[3]["total_tokens_saved"] == 205+ assert hourly[3]["total_input_tokens_delta"] == 90+ assert hourly[3]["total_input_tokens"] == 450+ assert hourly[3]["total_input_cost_usd_delta"] == pytest.approx(0.18)+ assert hourly[3]["total_input_cost_usd"] == pytest.approx(0.93)++ daily = response["series"]["daily"]+ assert [point["timestamp"] for point in daily] == [+ "2026-03-27T00:00:00Z",+ "2026-03-28T00:00:00Z",+ "2026-04-02T00:00:00Z",+ ]+ assert daily[0]["tokens_saved"] == 175+ assert daily[0]["total_tokens_saved"] == 175+ assert daily[0]["total_input_tokens_delta"] == 300+ assert daily[0]["total_input_tokens"] == 300+ assert daily[0]["total_input_cost_usd_delta"] == pytest.approx(0.63)+ assert daily[0]["total_input_cost_usd"] == pytest.approx(0.63)+ assert daily[1]["tokens_saved"] == 10+ assert daily[1]["total_tokens_saved"] == 185+ assert daily[1]["total_input_tokens_delta"] == 60+ assert daily[1]["total_input_tokens"] == 360+ assert daily[1]["total_input_cost_usd_delta"] == pytest.approx(0.12)+ assert daily[1]["total_input_cost_usd"] == pytest.approx(0.75)+ assert daily[2]["tokens_saved"] == 20+ assert daily[2]["total_tokens_saved"] == 205+ assert daily[2]["total_input_tokens_delta"] == 90+ assert daily[2]["total_input_tokens"] == 450+ assert daily[2]["total_input_cost_usd_delta"] == pytest.approx(0.18)+ assert daily[2]["total_input_cost_usd"] == pytest.approx(0.93)++ weekly = response["series"]["weekly"]+ assert [point["timestamp"] for point in weekly] == [+ "2026-03-23T00:00:00Z",+ "2026-03-30T00:00:00Z",+ ]+ assert weekly[0]["tokens_saved"] == 185+ assert weekly[0]["total_tokens_saved"] == 185+ assert weekly[1]["tokens_saved"] == 20+ assert weekly[1]["total_tokens_saved"] == 205++ monthly = response["series"]["monthly"]+ assert [point["timestamp"] for point in monthly] == [+ "2026-03-01T00:00:00Z",+ "2026-04-01T00:00:00Z",+ ]+ assert monthly[0]["tokens_saved"] == 185+ assert monthly[0]["total_tokens_saved"] == 185+ assert monthly[1]["tokens_saved"] == 20+ assert monthly[1]["total_tokens_saved"] == 205++ assert response["exports"]["available_formats"] == ["json", "csv"]+ assert response["exports"]["available_series"] == [+ "history",+ "hourly",+ "daily",+ "weekly",+ "monthly",+ ]+++def test_stats_history_defaults_to_compact_history_but_can_return_full_history(+ tmp_path, monkeypatch+):+ path = tmp_path / "proxy_savings.json"+ tracker = SavingsTracker(+ path=str(path),+ max_history_points=100,+ max_history_age_days=30,+ max_response_history_points=5,+ )+ monkeypatch.setattr(+ "headroom.proxy.savings_tracker._estimate_compression_savings_usd",+ lambda model, tokens_saved: tokens_saved / 1000.0,+ )++ for i in range(8):+ tracker.record_compression_savings(+ model="gpt-4o",+ tokens_saved=10,+ total_input_tokens=(i + 1) * 100,+ total_input_cost_usd=(i + 1) * 0.1,+ timestamp=f"2026-03-27T09:{i:02d}:00Z",+ )++ compact = tracker.history_response()+ assert compact["history_summary"] == {+ "mode": "compact",+ "stored_points": 8,+ "returned_points": 5,+ "compacted": True,+ }+ assert len(compact["history"]) == 5+ assert compact["history"][0]["timestamp"] == "2026-03-27T09:00:00Z"+ assert compact["history"][-1]["timestamp"] == "2026-03-27T09:07:00Z"++ full = tracker.history_response(history_mode="full")+ assert full["history_summary"] == {+ "mode": "full",+ "stored_points": 8,+ "returned_points": 8,+ "compacted": False,+ }+ assert len(full["history"]) == 8++ none = tracker.history_response(history_mode="none")+ assert none["history"] == []+ assert none["history_summary"] == {+ "mode": "none",+ "stored_points": 8,+ "returned_points": 0,+ "compacted": True,+ }+++def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):+ savings_path = tmp_path / "proxy_savings.json"+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))+ monkeypatch.setattr(+ "headroom.proxy.server.CostTracker._get_cache_prices",+ lambda self, model: (0.001, 0.0015, 0.002),+ )++ config = ProxyConfig(+ cache_enabled=False,+ rate_limit_enabled=False,+ log_requests=False,+ )++ with TestClient(create_app(config)) as client:+ _record_request(client, model="gpt-4o", tokens_saved=40)++ stats = client.get("/stats")+ assert stats.status_code == 200+ stats_data = stats.json()+ assert "savings_history" in stats_data+ assert "persistent_savings" in stats_data+ assert all(len(point) == 2 for point in stats_data["savings_history"])+ assert stats_data["persistent_savings"]["lifetime"]["tokens_saved"] == 40+ assert stats_data["persistent_savings"]["storage_path"] == str(savings_path)++ history = client.get("/stats-history")+ assert history.status_code == 200+ history_data = history.json()+ assert history_data["schema_version"] == 2+ assert history_data["storage_path"] == str(savings_path)+ assert history_data["lifetime"]["tokens_saved"] == 40+ assert history_data["lifetime"]["total_input_tokens"] == 120+ assert history_data["lifetime"]["total_input_cost_usd"] == pytest.approx(0.24)+ assert history_data["display_session"]["requests"] == 1+ assert history_data["display_session"]["tokens_saved"] == 40+ assert history_data["display_session"]["total_input_tokens"] == 120+ assert history_data["display_session"]["savings_percent"] == pytest.approx(25.0)+ assert list(history_data["series"].keys()) == [+ "hourly",+ "daily",+ "weekly",+ "monthly",+ ]+ assert history_data["exports"]["available_series"][-2:] == ["weekly", "monthly"]+ assert history_data["series"]["hourly"][0]["total_input_tokens_delta"] == 120+ assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(+ 0.24+ )+ assert history_data["history_summary"] == {+ "mode": "compact",+ "stored_points": 1,+ "returned_points": 1,+ "compacted": False,+ }++ assert stats_data["display_session"] == history_data["display_session"]+ assert (+ stats_data["persistent_savings"]["display_session"] == history_data["display_session"]+ )++ with TestClient(create_app(config)) as client:+ history = client.get("/stats-history")+ assert history.status_code == 200+ assert history.json()["lifetime"]["tokens_saved"] == 40+ assert history.json()["display_session"]["requests"] == 1++ _record_request(client, model="gpt-4o", tokens_saved=15)++ updated = client.get("/stats-history").json()+ assert updated["lifetime"]["tokens_saved"] == 55+ assert updated["lifetime"]["total_input_tokens"] == 240+ assert updated["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)+ assert updated["lifetime"]["requests"] == 2+ assert len(updated["history"]) == 2+ assert updated["display_session"]["requests"] == 2+ assert updated["display_session"]["tokens_saved"] == 55+ assert updated["display_session"]["total_input_tokens"] == 240+ assert updated["display_session"]["savings_percent"] == pytest.approx(18.64)+ assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240+ assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)++ full = client.get("/stats-history?history_mode=full").json()+ assert full["history_summary"]["mode"] == "full"+ assert full["history_summary"]["stored_points"] == 2+ assert full["history_summary"]["returned_points"] == 2++ persisted = json.loads(savings_path.read_text())+ assert persisted["lifetime"]["tokens_saved"] == 55+ assert persisted["lifetime"]["total_input_tokens"] == 240+ assert persisted["lifetime"]["total_input_cost_usd"] == pytest.approx(0.48)+ assert persisted["display_session"]["requests"] == 2+++def test_stats_history_csv_export_is_frontend_friendly(tmp_path, monkeypatch):+ savings_path = tmp_path / "proxy_savings.json"+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))+ monkeypatch.setattr(+ "headroom.proxy.server.CostTracker._get_cache_prices",+ lambda self, model: (0.001, 0.0015, 0.002),+ )++ config = ProxyConfig(+ cache_enabled=False,+ rate_limit_enabled=False,+ log_requests=False,+ )++ with TestClient(create_app(config)) as client:+ _record_request(client, model="gpt-4o", tokens_saved=40)+ _record_request(client, model="gpt-4o", tokens_saved=10)++ response = client.get("/stats-history?format=csv&series=daily")+ assert response.status_code == 200+ assert response.headers["content-type"].startswith("text/csv")+ assert (+ 'attachment; filename="headroom-stats-history-daily.csv"'+ == response.headers["content-disposition"]+ )+ lines = response.text.strip().splitlines()+ assert lines[0] == (+ "timestamp,tokens_saved,compression_savings_usd_delta,total_tokens_saved,"+ "compression_savings_usd,total_input_tokens_delta,total_input_tokens,"+ "total_input_cost_usd_delta,total_input_cost_usd"+ )+ assert len(lines) >= 2+ assert "total_tokens_saved" in lines[0]+ assert "total_input_cost_usd" in lines[0]+++def test_malformed_savings_state_is_ignored_safely(tmp_path, monkeypatch):+ savings_path = tmp_path / "proxy_savings.json"+ savings_path.write_text("{not valid json", encoding="utf-8")+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))++ config = ProxyConfig(+ cache_enabled=False,+ rate_limit_enabled=False,+ log_requests=False,+ )++ with TestClient(create_app(config)) as client:+ response = client.get("/stats-history")+ assert response.status_code == 200+ data = response.json()+ assert data["lifetime"]["tokens_saved"] == 0+ assert data["history"] == []+++def test_dashboard_includes_history_toggle_and_endpoint(tmp_path, monkeypatch):+ savings_path = tmp_path / "proxy_savings.json"+ monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))++ config = ProxyConfig(+ cache_enabled=False,+ rate_limit_enabled=False,+ log_requests=False,+ )++ with TestClient(create_app(config)) as client:+ response = client.get("/dashboard")+ assert response.status_code == 200+ html = response.text+ assert "Session" in html+ assert "Historical" in html+ assert "fetch('/stats-history')" in html+ assert "Export CSV" in html+ assert "Weekly Savings" in html+ assert "Monthly Savings" in htmldiff --git a/tests/test_release_version.py b/tests/test_release_version.pyindex 52bb0fcac..5485eeb20 100644--- a/tests/test_release_version.py+++ b/tests/test_release_version.py@@ -14,6 +14,7 @@ from headroom.release_version import (compute_release_version,determine_bump_level,find_latest_release_tag,+ get_canonical_version,list_release_commits,normalize_release_tag,parse_release_tag,@@ -174,10 +175,11 @@ def test_release_version_script_runs_directly_without_importing_headroom_package)assert result.returncode == 0, result.stderr+ canonical_version = get_canonical_version(ROOT)assert output_path.read_text(encoding="utf-8").splitlines() == ["version=0.6.0","npm_version=0.6.0",- "canonical=0.5.25",+ f"canonical={canonical_version}","height=0","bump=manual","previous_tag=",