Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Merge remote-tracking branch 'origin/main' into rust-rewrite

chopratejas4429a1116671
diff --cc headroom/proxy/server.py
index e8ae00f02,256776c70..051c82181
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@@ -1,2622 -1,2620 +1,2623 @@@
- """Headroom Proxy Server - Production Ready.
-
- A full-featured LLM proxy with optimization, caching, rate limiting,
- and observability.
-
- Features:
- - Context optimization (SmartCrusher, CacheAligner, RollingWindow)
- - Semantic caching (save costs on repeated queries)
- - Rate limiting (token bucket)
- - Retry with exponential backoff
- - Cost tracking and budgets
- - Request tagging and metadata
- - Provider fallback
- - Prometheus metrics
- - Full request/response logging
-
- Usage:
- python -m headroom.proxy.server --port 8787
-
- # With Claude Code:
- ANTHROPIC_BASE_URL=http://localhost:8787 claude
- """
-
- from __future__ import annotations
-
- import argparse
- import asyncio
- import json
- import logging
- import os
- import sys
- import time
- from datetime import datetime, timezone
- from pathlib import Path
- from typing import TYPE_CHECKING, Any, Literal, cast
-
- if TYPE_CHECKING:
- from ..backends.base import Backend
- from ..cache.compression_cache import CompressionCache
- from ..memory.tracker import MemoryTracker
-
-
- import httpx
-
- try:
- import uvicorn
- from fastapi import Depends, FastAPI, HTTPException, Request, Response
- from fastapi.middleware.cors import CORSMiddleware
- from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
-
- FASTAPI_AVAILABLE = True
- except ImportError:
- FASTAPI_AVAILABLE = False
-
- # Add parent to path for imports
- sys.path.insert(0, str(Path(__file__).parent.parent.parent))
-
- from headroom._version import __version__
- from headroom.cache.compression_feedback import get_compression_feedback
- from headroom.cache.compression_store import get_compression_store
- from headroom.ccr import (
- CCR_TOOL_NAME,
- # Batch processing
- CCRResponseHandler,
- CCRToolInjector,
- ContextTracker,
- ContextTrackerConfig,
- ResponseHandlerConfig,
- parse_tool_call,
- )
- from headroom.config import (
- CacheAlignerConfig,
- CCRConfig,
- IntelligentContextConfig,
- ReadLifecycleConfig,
- RollingWindowConfig,
- SmartCrusherConfig,
- )
- from headroom.dashboard import get_dashboard_html
- from headroom.observability import (
- LangfuseTracingConfig,
- OTelMetricsConfig,
- configure_langfuse_tracing,
- configure_otel_metrics,
- get_langfuse_tracing_status,
- get_otel_metrics_status,
- shutdown_headroom_tracing,
- shutdown_otel_metrics,
- )
- from headroom.pipeline import PipelineExtensionManager, PipelineStage
- from headroom.providers.proxy_routes import register_provider_routes
- from headroom.providers.registry import (
- DEFAULT_ANTHROPIC_API_URL,
- DEFAULT_CLOUDCODE_API_URL,
- DEFAULT_GEMINI_API_URL,
- DEFAULT_OPENAI_API_URL,
- build_proxy_provider_runtime,
- create_proxy_backend,
- format_backend_status,
- )
-
- # =============================================================================
- # Extracted modules (re-exported for backward compatibility)
- # =============================================================================
- from headroom.proxy.cost import (
- _CACHE_ECONOMICS, # noqa: F401
- CostTracker, # noqa: F401
- _summarize_transforms, # noqa: F401
- build_prefix_cache_stats, # noqa: F401
- build_session_summary, # noqa: F401
- merge_cost_stats, # noqa: F401
- )
- from headroom.proxy.helpers import (
- COMPRESSION_TIMEOUT_SECONDS, # noqa: F401
- MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401
- MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401
- MAX_REQUEST_BODY_SIZE, # noqa: F401
- MAX_SSE_BUFFER_SIZE, # noqa: F401
- _get_image_compressor, # noqa: F401
- _get_rtk_stats, # noqa: F401
- _read_request_json, # noqa: F401
- _setup_file_logging, # noqa: F401
- is_anthropic_auth, # noqa: F401
- jitter_delay_ms,
- )
- from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
-
- # Data models (extracted to headroom/proxy/models.py for maintainability)
- from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
- from headroom.proxy.modes import (
- PROXY_MODE_CACHE,
- PROXY_MODE_TOKEN,
- is_token_mode,
- normalize_proxy_mode,
- )
- from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
- from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
- from headroom.proxy.request_logger import RequestLogger # noqa: F401
- from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
- from headroom.proxy.warmup import WarmupRegistry
- from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
- from headroom.subscription.base import get_quota_registry, reset_quota_registry
- from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state
- from headroom.subscription.copilot_quota import get_copilot_quota_tracker
- from headroom.subscription.tracker import (
- configure_subscription_tracker,
- get_subscription_tracker,
- )
- from headroom.telemetry import get_telemetry_collector
- from headroom.telemetry.beacon import is_telemetry_enabled
- from headroom.telemetry.toin import get_toin
- from headroom.transforms import (
- CacheAligner,
- CodeAwareCompressor,
- CodeCompressorConfig,
- ContentRouter,
- ContentRouterConfig,
- IntelligentContextManager,
- RollingWindow,
- SmartCrusher,
- Transform,
- TransformPipeline,
- is_tree_sitter_available,
- )
-
- AnyLLMBackend: Any = None
- LiteLLMBackend: Any = None
-
- fcntl: Any = None
- try:
- import fcntl as _fcntl
-
- fcntl = _fcntl
- HAS_FCNTL = True
- except ImportError:
- HAS_FCNTL = False
-
- _build_prefix_cache_stats = build_prefix_cache_stats
- _build_session_summary = build_session_summary
- _merge_cost_stats = merge_cost_stats
-
-
- logging.basicConfig(
- level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
- )
- logger = logging.getLogger("headroom.proxy")
-
- # Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis
- _setup_file_logging()
-
-
- # Compression pipeline timeout in seconds
-
-
- from headroom.proxy.handlers import ( # noqa: E402
- AnthropicHandlerMixin,
- BatchHandlerMixin,
- GeminiHandlerMixin,
- OpenAIHandlerMixin,
- StreamingMixin,
- )
-
-
- class HeadroomProxy(
- StreamingMixin,
- AnthropicHandlerMixin,
- OpenAIHandlerMixin,
- GeminiHandlerMixin,
- BatchHandlerMixin,
- ):
- """Production-ready Headroom optimization proxy."""
-
- ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL
- OPENAI_API_URL = DEFAULT_OPENAI_API_URL
- GEMINI_API_URL = DEFAULT_GEMINI_API_URL
- CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL
-
- def __init__(self, config: ProxyConfig):
- self.config = config
- self.config.mode = normalize_proxy_mode(self.config.mode)
- self.pipeline_extensions = PipelineExtensionManager(
- hooks=config.hooks,
- extensions=config.pipeline_extensions,
- discover=config.discover_pipeline_extensions,
- )
-
- self.provider_runtime = build_proxy_provider_runtime(config)
- api_targets = self.provider_runtime.api_targets
-
- # Preserve the long-standing proxy compatibility surface while keeping
- # provider_runtime as the source of truth for resolved upstream targets.
- HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic
- HeadroomProxy.OPENAI_API_URL = api_targets.openai
- HeadroomProxy.GEMINI_API_URL = api_targets.gemini
- HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode
- self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")
- self.openai_provider = self.provider_runtime.pipeline_provider("openai")
-
- # Initialize transforms based on routing mode
- # Choose context manager: IntelligentContextManager (smart) or RollingWindow (legacy)
- context_manager: Transform # Can be either IntelligentContextManager or RollingWindow
- if config.intelligent_context:
- # Get TOIN instance for learned pattern integration
- toin = get_toin() if config.intelligent_context_scoring else None
- context_manager = IntelligentContextManager(
- config=IntelligentContextConfig(
- enabled=True,
- keep_system=True,
- keep_last_turns=config.keep_last_turns,
- use_importance_scoring=config.intelligent_context_scoring,
- toin_integration=config.intelligent_context_scoring,
- compress_threshold=0.10 if config.intelligent_context_compress_first else 0.0,
- ),
- toin=toin,
- )
- self._context_manager_status = "intelligent"
- else:
- context_manager = RollingWindow(
- RollingWindowConfig(
- enabled=True,
- keep_system=True,
- keep_last_turns=config.keep_last_turns,
- )
- )
- self._context_manager_status = "rolling_window"
-
- if config.smart_routing:
- # Smart routing: ContentRouter handles all content types intelligently
- # It lazy-loads compressors only when needed
- router_config = ContentRouterConfig(
- enable_code_aware=config.code_aware_enabled,
- tool_profiles=config.tool_profiles,
- read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),
- )
- # Token mode: allow compression of older excluded-tool results
- if is_token_mode(config.mode):
- router_config.protect_recent_reads_fraction = 0.3
- transforms = [
- CacheAligner(CacheAlignerConfig(enabled=False)),
- ContentRouter(router_config),
- context_manager,
- ]
- self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"
- else:
- # Legacy mode: sequential pipeline
- transforms = [
- CacheAligner(CacheAlignerConfig(enabled=False)),
- SmartCrusher(
- SmartCrusherConfig( # type: ignore[arg-type]
- enabled=True,
- min_tokens_to_crush=config.min_tokens_to_crush,
- max_items_after_crush=config.max_items_after_crush,
- ),
- ccr_config=CCRConfig(
- enabled=config.ccr_inject_tool,
- inject_retrieval_marker=config.ccr_inject_tool, # Add CCR markers
- ),
- ),
- context_manager,
- ]
- # Add CodeAware if enabled and available
- self._code_aware_status = self._setup_code_aware(config, transforms)
-
- self.anthropic_pipeline = TransformPipeline(
- transforms=transforms,
- provider=self.anthropic_provider,
- )
- self.openai_pipeline = TransformPipeline(
- transforms=transforms,
- provider=self.openai_provider,
- )
-
- # Initialize components
- self.cache = (
- SemanticCache(
- max_entries=config.cache_max_entries,
- ttl_seconds=config.cache_ttl_seconds,
- )
- if config.cache_enabled
- else None
- )
-
- self.rate_limiter = (
- TokenBucketRateLimiter(
- requests_per_minute=config.rate_limit_requests_per_minute,
- tokens_per_minute=config.rate_limit_tokens_per_minute,
- )
- if config.rate_limit_enabled
- else None
- )
-
- self.cost_tracker = (
- CostTracker(
- budget_limit_usd=config.budget_limit_usd,
- budget_period=config.budget_period,
- )
- if config.cost_tracking_enabled
- else None
- )
-
- self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)
-
- # Prefix cache tracking: freeze already-cached messages to avoid
- # invalidating the provider's prefix cache with our transforms
- from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore
-
- self.session_tracker_store = SessionTrackerStore(
- default_config=PrefixFreezeConfig(
- enabled=config.prefix_freeze_enabled,
- session_ttl_seconds=config.prefix_freeze_session_ttl,
- )
- )
-
- # Compression cache store for token mode (session-scoped)
- self._compression_caches: dict[str, CompressionCache] = {}
-
- self.logger = (
- RequestLogger(
- log_file=config.log_file,
- log_full_messages=config.log_full_messages,
- )
- if config.log_requests
- else None
- )
-
- # Enterprise security plugin (loaded dynamically if available + licensed)
- self.security = None
-
- # HTTP client
- self.http_client: httpx.AsyncClient | None = None
-
- # Shared cold-start warmup registry (populated by startup()).
- # Holds typed slots with loaded / loading / null / error status for
- # each preloaded heavy asset. Exposed as ``proxy.warmup`` and
- # serialized by the /debug/warmup route (Unit 5).
- self.warmup: WarmupRegistry = WarmupRegistry()
- # Unit 3: live registry of Codex WS sessions. Populated by
- # ``handle_openai_responses_ws`` on accept; drained in its
- # outermost ``finally``. Consumed by ``/debug/ws-sessions``.
- self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()
-
- # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP
- # path. Caps how many ``handle_anthropic_messages`` calls may be
- # running deep-copy / first-stage compression / memory-context
- # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,
- # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are
- # intentionally NOT gated by this semaphore.
- #
- # A value of ``0`` or negative disables the semaphore (unbounded
- # mode); this is useful for the Unit 6 counter-factual where we
- # deliberately reproduce the original starvation. The default is
- # ``max(2, min(8, os.cpu_count() or 4))``.
- _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency
- if _pre_upstream_cfg is None:
- _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))
- else:
- _pre_upstream_resolved = _pre_upstream_cfg
- self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved
- self.anthropic_pre_upstream_acquire_timeout_seconds = float(
- config.anthropic_pre_upstream_acquire_timeout_seconds
- )
- self.anthropic_pre_upstream_memory_context_timeout_seconds = float(
- config.anthropic_pre_upstream_memory_context_timeout_seconds
- )
- if _pre_upstream_resolved > 0:
- self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = asyncio.Semaphore(
- _pre_upstream_resolved
- )
- else:
- self.anthropic_pre_upstream_sem = None
-
- # Backend for Anthropic API (direct, LiteLLM, or any-llm)
- # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
- self.anthropic_backend: Backend | None = create_proxy_backend(
- backend=config.backend,
- anyllm_provider=config.anyllm_provider,
- bedrock_region=config.bedrock_region,
- logger=logger,
- anyllm_backend_cls=AnyLLMBackend,
- litellm_backend_cls=LiteLLMBackend,
- )
-
- # Request counter for IDs
- self._request_counter = 0
- self._request_counter_lock = asyncio.Lock()
-
- # CCR tool injectors (one per provider)
- self.anthropic_tool_injector = CCRToolInjector(
- provider="anthropic",
- inject_tool=config.ccr_inject_tool,
- inject_system_instructions=config.ccr_inject_system_instructions,
- )
- self.openai_tool_injector = CCRToolInjector(
- provider="openai",
- inject_tool=config.ccr_inject_tool,
- inject_system_instructions=config.ccr_inject_system_instructions,
- )
-
- # CCR Response Handler (handles CCR tool calls automatically)
- self.ccr_response_handler = (
- CCRResponseHandler(
- ResponseHandlerConfig(
- enabled=True,
- max_retrieval_rounds=config.ccr_max_retrieval_rounds,
- )
- )
- if config.ccr_handle_responses
- else None
- )
-
- # CCR Context Tracker (tracks compressed content across turns)
- self.ccr_context_tracker = (
- ContextTracker(
- ContextTrackerConfig(
- enabled=True,
- proactive_expansion=config.ccr_proactive_expansion,
- max_proactive_expansions=config.ccr_max_proactive_expansions,
- )
- )
- if config.ccr_context_tracking
- else None
- )
-
- # Turn counter for context tracking
- self._turn_counter = 0
-
- # Memory Handler (persistent user memory)
- self.memory_handler: MemoryHandler | None = None
- if config.memory_enabled:
- # Resolve memory DB path: empty → project-scoped default
- _mem_db_path = config.memory_db_path
- if not _mem_db_path:
- _mem_dir = Path.cwd() / ".headroom"
- _mem_dir.mkdir(parents=True, exist_ok=True)
- _mem_db_path = str(_mem_dir / "memory.db")
- logger.info(f"Memory: Project-scoped DB at {_mem_db_path}")
-
- memory_config = MemoryConfig(
- enabled=True,
- backend=config.memory_backend,
- db_path=_mem_db_path,
- inject_tools=config.memory_inject_tools,
- use_native_tool=config.memory_use_native_tool,
- inject_context=config.memory_inject_context,
- top_k=config.memory_top_k,
- min_similarity=config.memory_min_similarity,
- qdrant_host=config.memory_qdrant_host,
- qdrant_port=config.memory_qdrant_port,
- neo4j_uri=config.memory_neo4j_uri,
- neo4j_user=config.memory_neo4j_user,
- neo4j_password=config.memory_neo4j_password,
- bridge_enabled=config.memory_bridge_enabled,
- bridge_md_paths=config.memory_bridge_md_paths,
- bridge_md_format=config.memory_bridge_md_format,
- bridge_auto_import=config.memory_bridge_auto_import,
- bridge_export_path=config.memory_bridge_export_path,
- )
- self.memory_handler = MemoryHandler(
- memory_config,
- agent_type=config.traffic_learning_agent_type,
- )
-
- # Usage Reporter (license validation + phone-home for managed/enterprise)
- self.usage_reporter: UsageReporter | None = None
- if config.license_key:
- from headroom.telemetry.reporter import UsageReporter
-
- self.usage_reporter = UsageReporter(
- license_key=config.license_key,
- cloud_url=config.license_cloud_url,
- report_interval=config.license_report_interval,
- )
-
- # Traffic Learner (live pattern extraction from proxy traffic)
- # Only activates with --learn flag; requires --memory for backend
- self.traffic_learner: TrafficLearner | None = None
- self.traffic_learning_agent_type: str = config.traffic_learning_agent_type
- if config.traffic_learning_enabled:
- from headroom.memory.traffic_learner import TrafficLearner
-
- self.traffic_learner = TrafficLearner(
- user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
- agent_type=config.traffic_learning_agent_type,
- )
-
- # Code graph file watcher (live reindex on file changes)
- self.code_graph_watcher: CodeGraphWatcher | None = None # type: ignore[annotation-unchecked]
- if config.code_graph_watcher:
- from headroom.graph.watcher import CodeGraphWatcher
-
- self.code_graph_watcher = CodeGraphWatcher(project_dir=Path.cwd())
- if self.code_graph_watcher.start():
- logger.info("Code graph: file watcher started")
- else:
- self.code_graph_watcher = None
-
- self.pipeline_extensions.emit(
- PipelineStage.SETUP,
- operation="proxy.setup",
- metadata={
- "mode": self.config.mode,
- "optimize": self.config.optimize,
- "backend": self.config.backend,
- "memory_enabled": self.config.memory_enabled,
- },
- )
-
- def _get_compression_cache(self, session_id: str) -> CompressionCache:
- """Get or create a CompressionCache for a session."""
- if session_id not in self._compression_caches:
- from headroom.cache.compression_cache import CompressionCache
-
- # Evict oldest caches if at capacity
- if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:
- # Remove oldest quarter to amortize cleanup cost
- oldest_keys = list(self._compression_caches.keys())[
- : MAX_COMPRESSION_CACHE_SESSIONS // 4
- ]
- for key in oldest_keys:
- del self._compression_caches[key]
- logger.info(
- "Evicted %d compression caches (exceeded %d max sessions)",
- len(oldest_keys),
- MAX_COMPRESSION_CACHE_SESSIONS,
- )
-
- self._compression_caches[session_id] = CompressionCache()
- return self._compression_caches[session_id]
-
- def _setup_code_aware(self, config: ProxyConfig, transforms: list) -> str:
- """Set up code-aware compression if enabled.
-
- Args:
- config: Proxy configuration
- transforms: Transform list to append to
-
- Returns:
- Status string for logging: 'enabled', 'disabled', 'available', 'unavailable'
- """
- if config.code_aware_enabled:
- if is_tree_sitter_available():
- code_config = CodeCompressorConfig(
- preserve_imports=True,
- preserve_signatures=True,
- preserve_type_annotations=True,
- )
- # Insert before RollingWindow (which should be last)
- transforms.insert(-1, CodeAwareCompressor(code_config))
- return "enabled"
- else:
- logger.warning(
- "Code-aware compression requested but tree-sitter not installed. "
- "Install with: pip install headroom-ai[code]"
- )
- return "unavailable"
- else:
- if is_tree_sitter_available():
- return "available" # Available but not enabled
- return "disabled"
-
- async def startup(self):
- """Initialize async resources."""
- self.pipeline_extensions.emit(
- PipelineStage.PRE_START,
- operation="proxy.startup",
- metadata={"port": self.config.port, "host": self.config.host},
- )
- self.http_client = httpx.AsyncClient(
- timeout=httpx.Timeout(
- connect=self.config.connect_timeout_seconds,
- read=self.config.request_timeout_seconds,
- write=self.config.request_timeout_seconds,
- pool=self.config.connect_timeout_seconds,
- ),
- limits=httpx.Limits(
- max_connections=self.config.max_connections,
- max_keepalive_connections=self.config.max_keepalive_connections,
- ),
- http2=self.config.http2,
- )
- logger.info("Headroom Proxy started")
- logger.info(f"Optimization: {'ENABLED' if self.config.optimize else 'DISABLED'}")
- self.config.mode = normalize_proxy_mode(self.config.mode)
- logger.info(f"Mode: {self.config.mode}")
- if self.config.mode == PROXY_MODE_TOKEN:
- logger.info(" Prefix freeze: re-freeze after compression")
- logger.info(" Read protection window: 30%% of excluded-tool messages")
- logger.info(" CCR TTL: extended for session lifetime")
- logger.info(" Compression cache: active")
- if self.config.mode == PROXY_MODE_CACHE:
- logger.info(" Prefix freeze: strict (all prior turns immutable)")
- logger.info(" Mutations: latest turn only")
- logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}")
- logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}")
- logger.info(
- f"Connection Pool: max_connections={self.config.max_connections}, "
- f"max_keepalive={self.config.max_keepalive_connections}, "
- f"http2={'ENABLED' if self.config.http2 else 'DISABLED'}"
- )
-
- # Unit 4 pre-upstream concurrency announcement. Report the resolved
- # value (auto-detected vs. explicit) so operators can correlate
- # ``pre_upstream_wait_ms`` log lines with the configured cap.
- if self.anthropic_pre_upstream_sem is None:
- logger.info("Anthropic pre-upstream concurrency: unbounded (explicitly disabled)")
- else:
- _explicit = self.config.anthropic_pre_upstream_concurrency
- _origin = "auto-detected" if _explicit is None else "explicit"
- logger.info(
- "Anthropic pre-upstream concurrency: %d (%s)",
- self.anthropic_pre_upstream_concurrency,
- _origin,
- )
- logger.info(
- "Anthropic pre-upstream timeouts: acquire=%.1fs compression=%.1fs memory_context=%.1fs",
- self.anthropic_pre_upstream_acquire_timeout_seconds,
- float(COMPRESSION_TIMEOUT_SECONDS),
- self.anthropic_pre_upstream_memory_context_timeout_seconds,
- )
-
- # Smart routing status
- if self.config.smart_routing:
- logger.info("Smart Routing: ENABLED (intelligent content detection)")
- else:
- logger.info("Smart Routing: DISABLED (legacy sequential mode)")
-
- # Eagerly load ALL compressors, parsers, and detectors at startup
- # This eliminates cold-start latency spikes on first requests.
- # Iterate BOTH pipelines (Anthropic + OpenAI) and dedupe transforms
- # by id() so shared-transform instances never load twice. The
- # resulting status dict is merged into ``self.warmup`` so /debug/warmup
- # (Unit 5) and /readyz have a single source of truth.
- self._kompress_status = "not installed"
- eager_status: dict[str, str] = {}
-
- if self.config.optimize:
- logger.info("Pre-loading compressors and parsers...")
- seen_transform_ids: set[int] = set()
- pipelines = (self.anthropic_pipeline, self.openai_pipeline)
- for pipeline in pipelines:
- for transform in pipeline.transforms:
- if id(transform) in seen_transform_ids:
- continue
- seen_transform_ids.add(id(transform))
- if not hasattr(transform, "eager_load_compressors"):
- continue
- try:
- transform_status = transform.eager_load_compressors()
- except Exception as exc:
- logger.warning(
- "Eager preload failed for %s: %s",
- type(transform).__name__,
- exc,
- )
- continue
- if not isinstance(transform_status, dict):
- continue
- # Merge: later writers win only if the key wasn't set.
- # Preload a transform ONCE — if another pipeline also has
- # ``eager_load_compressors`` it contributes only new keys.
- for key, value in transform_status.items():
- eager_status.setdefault(key, value)
- self.warmup.merge_transform_status(transform_status)
-
- # Update internal status from eager loading results
- if eager_status.get("kompress") == "enabled":
- self._kompress_status = "enabled"
- if eager_status.get("code_aware") == "enabled":
- self._code_aware_status = "enabled"
-
- # Log component status
- if self._kompress_status == "enabled":
- logger.info("Kompress: ENABLED (ModernBERT token compressor)")
- elif self.config.optimize:
- logger.info("Kompress: not installed (pip install headroom-ai[ml] for ML compression)")
-
- if self._code_aware_status == "enabled":
- logger.info("Code-Aware: ENABLED (AST-based compression)")
- if "tree_sitter" in eager_status:
- logger.info(f"Tree-Sitter: {eager_status['tree_sitter']}")
- elif self._code_aware_status == "lazy":
- logger.info("Code-Aware: LAZY (will load when code content detected)")
- elif self._code_aware_status == "available":
- logger.info("Code-Aware: available but disabled (use --code-aware)")
- elif self._code_aware_status == "unavailable":
- logger.info("Code-Aware: not installed (pip install headroom-ai[code])")
- elif self._code_aware_status == "disabled":
- logger.info("Code-Aware: DISABLED")
-
- if eager_status.get("magika") == "enabled":
- logger.info("Magika: ENABLED (ML content detection)")
-
- if self.memory_handler:
- self.warmup.memory_backend.mark_loading()
- try:
- await self.memory_handler.ensure_initialized()
- except Exception as exc: # pragma: no cover - defensive
- self.warmup.memory_backend.mark_error(str(exc))
- logger.warning("Memory: backend initialization failed (startup continues): %s", exc)
- memory_status = self.memory_handler.health_status()
- if memory_status.get("initialized"):
- self.warmup.memory_backend.mark_loaded(
- handle=self.memory_handler,
- backend=memory_status.get("backend"),
- )
- # Force one embed call so the ONNX graph is compiled now,
- # not lazily during the first request. Best-effort — any
- # failure is swallowed inside warmup_embedder.
- self.warmup.memory_embedder.mark_loading()
- warmed = await self.memory_handler.warmup_embedder()
- if warmed:
- self.warmup.memory_embedder.mark_loaded()
- else:
- # Not an error — e.g. qdrant-neo4j has no embedder slot
- # we can reach, or the backend simply exposes no handle.
- self.warmup.memory_embedder.mark_null()
- else:
- if self.warmup.memory_backend.status != "error":
- self.warmup.memory_backend.mark_null()
- self.warmup.memory_embedder.mark_null()
- logger.info(
- "Memory: ENABLED "
- f"(backend={memory_status['backend']}, initialized={memory_status['initialized']})"
- )
- else:
- logger.info("Memory: DISABLED")
-
- # CCR status
- ccr_features = []
- if self.config.ccr_inject_tool:
- ccr_features.append("tool_injection")
- if self.config.ccr_handle_responses:
- ccr_features.append("response_handling")
- if self.config.ccr_context_tracking:
- ccr_features.append("context_tracking")
- if self.config.ccr_proactive_expansion:
- ccr_features.append("proactive_expansion")
- if ccr_features:
- logger.info(f"CCR (Compress-Cache-Retrieve): ENABLED ({', '.join(ccr_features)})")
- else:
- logger.info("CCR: DISABLED")
- logger.info(f"Savings history: {self.metrics.savings_tracker.storage_path}")
-
- # Reset and rebuild the quota tracker registry for this server instance.
- # reset_quota_registry() ensures a clean slate when the proxy is restarted
- # (e.g. in tests that spin up multiple app instances in the same process).
- reset_quota_registry()
- registry = get_quota_registry()
- tracker = configure_subscription_tracker(
- poll_interval_s=self.config.subscription_poll_interval_s,
- active_window_s=self.config.subscription_active_window_s,
- enabled=self.config.subscription_tracking_enabled,
- )
- registry.register(tracker)
- registry.register(get_codex_rate_limit_state())
- registry.register(get_copilot_quota_tracker())
- await registry.start_all()
-
- if self.config.subscription_tracking_enabled:
- logger.info(
- "Subscription tracking: ENABLED "
- f"(poll_interval={self.config.subscription_poll_interval_s}s, "
- f"active_window={self.config.subscription_active_window_s}s)"
- )
- else:
- logger.info("Subscription tracking: DISABLED")
-
- copilot_tracker = get_copilot_quota_tracker()
- if copilot_tracker.is_available():
- logger.info("GitHub Copilot quota tracking: ENABLED")
- else:
- logger.info(
- "GitHub Copilot quota tracking: DISABLED "
- "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)"
- )
-
- # Log anonymous telemetry status so operators can see it in the log stream
- if is_telemetry_enabled():
- logger.info(
- "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). "
- "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry"
- )
- else:
- logger.info("Anonymous telemetry: DISABLED")
-
- self.pipeline_extensions.emit(
- PipelineStage.POST_START,
- operation="proxy.startup",
- metadata={
- "port": self.config.port,
- "host": self.config.host,
- "warmup": self.warmup.to_dict(),
- },
- )
-
- async def shutdown(self):
- """Cleanup async resources."""
- if self.http_client:
- await self.http_client.aclose()
- self.http_client = None
-
- if self.memory_handler and hasattr(self.memory_handler, "close"):
- await self.memory_handler.close()
-
- # Stop all quota trackers via the registry
- await get_quota_registry().stop_all()
-
- # Print final stats
- self._print_summary()
-
- def _print_summary(self):
- """Print session summary."""
- m = self.metrics
- logger.info("=" * 70)
- logger.info("HEADROOM PROXY SESSION SUMMARY")
- logger.info("=" * 70)
- logger.info(f"Total requests: {m.requests_total}")
- logger.info(f"Cached responses: {m.requests_cached}")
- logger.info(f"Rate limited: {m.requests_rate_limited}")
- logger.info(f"Failed: {m.requests_failed}")
- logger.info(f"Input tokens: {m.tokens_input_total:,}")
- logger.info(f"Output tokens: {m.tokens_output_total:,}")
- logger.info(f"Tokens saved: {m.tokens_saved_total:,}")
- if m.tokens_input_total > 0:
- savings_pct = (
- m.tokens_saved_total / (m.tokens_input_total + m.tokens_saved_total)
- ) * 100
- logger.info(f"Token savings: {savings_pct:.1f}%")
- if m.latency_count > 0:
- avg_latency = m.latency_sum_ms / m.latency_count
- logger.info(f"Avg latency: {avg_latency:.0f}ms")
- logger.info("=" * 70)
-
- async def _next_request_id(self) -> str:
- """Generate unique request ID."""
- async with self._request_counter_lock:
- self._request_counter += 1
- return f"hr_{int(time.time())}_{self._request_counter:06d}"
-
- def _extract_tags(self, headers: dict) -> dict[str, str]:
- """Extract Headroom tags from headers."""
- tags = {}
- for key, value in headers.items():
- if key.lower().startswith("x-headroom-"):
- tag_name = key.lower().replace("x-headroom-", "")
- tags[tag_name] = value
- return tags
-
- def _inject_system_context(
- self,
- messages: list[dict[str, Any]],
- context: str,
- body: dict[str, Any] | None = None,
- ) -> list[dict[str, Any]]:
- """Inject context into the system message/parameter.
-
- For Anthropic API: Uses top-level 'system' parameter (not messages array).
- For OpenAI API: Uses system role in messages array.
-
- Args:
- messages: The messages list.
- context: Context to inject.
- body: Optional request body to update system parameter (for Anthropic).
-
- Returns:
- Updated messages list.
- """
- messages = list(messages) # Copy to avoid mutation
-
- # For Anthropic API: use top-level 'system' parameter
- if body is not None:
- existing_system = body.get("system", "")
- if isinstance(existing_system, str):
- body["system"] = (existing_system + "\n\n" + context).strip()
- elif isinstance(existing_system, list):
- # system is a list of content blocks (e.g., with cache_control).
- # Append memory context as a new text block — never overwrite.
- body["system"] = existing_system + [{"type": "text", "text": context}]
- else:
- # No existing system prompt — set as string
- body["system"] = context
- return messages
-
- # For OpenAI API: use system role in messages
- for i, msg in enumerate(messages):
- if msg.get("role") == "system":
- content = msg.get("content", "")
- if isinstance(content, str):
- messages[i] = {**msg, "content": content + "\n\n" + context}
- return messages
-
- # No system message found - prepend one
- messages.insert(0, {"role": "system", "content": context})
- return messages
-
- async def _retry_request(
- self,
- method: str,
- url: str,
- headers: dict,
- body: dict,
- stream: bool = False,
- ) -> httpx.Response:
- """Make request with retry and exponential backoff."""
- last_error = None
-
- for attempt in range(self.config.retry_max_attempts):
- try:
- if stream:
- # For streaming, we return early - retry happens at higher level
- return await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
- else:
- response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
-
- # Don't retry client errors (4xx)
- if 400 <= response.status_code < 500:
- return response
-
- # Retry server errors (5xx)
- if response.status_code >= 500:
- raise httpx.HTTPStatusError(
- f"Server error: {response.status_code}",
- request=response.request,
- response=response,
- )
-
- return response
-
- except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
- last_error = e
-
- if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:
- raise
-
- # Exponential backoff with jitter
- delay_with_jitter = jitter_delay_ms(
- self.config.retry_base_delay_ms,
- self.config.retry_max_delay_ms,
- attempt,
- )
-
- logger.warning(
- f"Request failed (attempt {attempt + 1}), retrying in {delay_with_jitter:.0f}ms: {e}"
- )
- await asyncio.sleep(delay_with_jitter / 1000)
-
- raise last_error # type: ignore[misc]
-
-
- async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:
- """Background task that logs TOIN stats periodically.
-
- Args:
- interval_seconds: How often to log stats (default: 5 minutes).
- """
- while True:
- await asyncio.sleep(interval_seconds)
- try:
- toin = get_toin()
- stats = toin.get_stats()
- total_compressions = stats.get("total_compressions", 0)
- if total_compressions > 0:
- patterns = stats.get("patterns_tracked", 0)
- retrievals = stats.get("total_retrievals", 0)
- retrieval_rate = stats.get("global_retrieval_rate", 0.0)
- logger.info(
- "TOIN: %d patterns, %d compressions, %d retrievals, %.1f%% retrieval rate",
- patterns,
- total_compressions,
- retrievals,
- retrieval_rate * 100,
- )
- except Exception as e:
- logger.debug("Failed to log TOIN stats: %s", e)
-
-
- def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:
- """Register all memory-tracked components with the tracker.
-
- This function is idempotent - it checks if components are already registered.
-
- Args:
- proxy: The HeadroomProxy instance.
- tracker: The MemoryTracker instance.
- """
- # Register compression store (global singleton)
- if "compression_store" not in tracker.registered_components:
- store = get_compression_store()
- tracker.register("compression_store", store.get_memory_stats)
-
- # Register semantic cache (instance on proxy)
- if proxy.cache and "semantic_cache" not in tracker.registered_components:
- tracker.register("semantic_cache", proxy.cache.get_memory_stats)
-
- # Register request logger (instance on proxy)
- if proxy.logger and "request_logger" not in tracker.registered_components:
- tracker.register("request_logger", proxy.logger.get_memory_stats)
-
- # Register batch context store (global singleton)
- if "batch_context_store" not in tracker.registered_components:
- try:
- from ..ccr.batch_store import get_batch_context_store
-
- batch_store = get_batch_context_store()
- if hasattr(batch_store, "get_memory_stats"):
- tracker.register("batch_context_store", batch_store.get_memory_stats)
- except ImportError:
- pass
-
- # Note: graph_store and vector_index are created per-user within the
- # LocalMemoryBackend, not as global singletons. They would need to be
- # registered when the memory system is initialized with specific backends.
-
-
- def create_app(config: ProxyConfig | None = None) -> FastAPI:
- """Create FastAPI application."""
- if not FASTAPI_AVAILABLE:
- raise ImportError("FastAPI required. Install: pip install fastapi uvicorn httpx")
-
- from contextlib import asynccontextmanager
-
- config = config or ProxyConfig()
- proxy = HeadroomProxy(config)
-
- # Telemetry beacon (anonymous aggregate stats).
- # With uvicorn workers > 1, each worker runs the lifespan independently.
- # We must ensure only ONE beacon runs across all workers — otherwise each
- # worker creates its own beacon, spamming the telemetry table with N rows
- # per cycle instead of 1 (all reading the same /stats from the same port).
- #
- # Strategy: use a file lock to ensure only the first worker starts the
- # beacon. Other workers see the lock and skip.
- from headroom.telemetry.beacon import TelemetryBeacon
-
- _beacon = TelemetryBeacon(
- port=config.port if hasattr(config, "port") else 8787,
- sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
- backend=config.backend if hasattr(config, "backend") else "anthropic",
- )
- from headroom import paths as _hr_paths
-
- _beacon_lock_path = _hr_paths.beacon_lock_path(config.port)
- _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor
- _beacon_is_owner: list = [False]
-
- def _try_acquire_beacon_lock() -> bool:
- """Try to acquire the beacon file lock (non-blocking).
-
- Returns True if this process is the beacon owner.
- """
- if not HAS_FCNTL:
- return True
-
- fd = None
- try:
- _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)
- fd = open(_beacon_lock_path, "w") # noqa: SIM115
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- fd.write(str(os.getpid()))
- fd.flush()
- _beacon_lock_fd[0] = fd
- return True
- except OSError:
- if fd is not None:
- fd.close()
- return False
-
- def _release_beacon_lock() -> None:
- """Release the beacon file lock."""
- fd = _beacon_lock_fd[0]
- if fd:
- try:
- if HAS_FCNTL:
- fcntl.flock(fd, fcntl.LOCK_UN)
- fd.close()
- except Exception:
- pass
- _beacon_lock_fd[0] = None
- try:
- _beacon_lock_path.unlink(missing_ok=True)
- except Exception:
- pass
-
- @asynccontextmanager
- async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
- configure_otel_metrics(OTelMetricsConfig.from_env(default_service_name="headroom-proxy"))
- configure_langfuse_tracing(
- LangfuseTracingConfig.from_env(default_service_name="headroom-proxy")
- )
-
- app.state.started_at = time.time()
- app.state.ready = False
- app.state.startup_error = None
-
- try:
- try:
- # Startup
- await proxy.startup()
- asyncio.create_task(_log_toin_stats_periodically())
- if proxy.usage_reporter:
- await proxy.usage_reporter.start(proxy)
- if proxy.traffic_learner:
- await proxy.traffic_learner.start()
-
- # Only start beacon if we acquire the lock (first worker wins)
- _beacon_is_owner[0] = _try_acquire_beacon_lock()
- if _beacon_is_owner[0]:
- await _beacon.start()
- else:
- logger.debug("Beacon: skipping (another worker owns the lock)")
-
- app.state.ready = True
- yield
- except Exception as exc:
- app.state.startup_error = str(exc)
- raise
- finally:
- app.state.ready = False
- # Shutdown
- if _beacon_is_owner[0]:
- await _beacon.stop()
- _release_beacon_lock()
- if proxy.usage_reporter:
- await proxy.usage_reporter.stop()
- if proxy.traffic_learner:
- await proxy.traffic_learner.stop()
- if proxy.code_graph_watcher:
- proxy.code_graph_watcher.stop()
- await proxy.shutdown()
- shutdown_headroom_tracing()
- shutdown_otel_metrics()
-
- app = FastAPI(
- title="Headroom Proxy",
- description="Production-ready LLM optimization proxy",
- version=__version__,
- lifespan=lifespan,
- )
- app.state.proxy = proxy
- app.state.started_at = None
- app.state.ready = False
- app.state.startup_error = None
-
- def _iso_utc_now() -> str:
- return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
-
- def _uptime_seconds() -> float:
- started_at = getattr(app.state, "started_at", None)
- if not isinstance(started_at, int | float):
- return 0.0
- return round(max(0.0, time.time() - float(started_at)), 3)
-
- def _component_health(
- *,
- enabled: bool,
- ready: bool,
- **details: Any,
- ) -> dict[str, Any]:
- status = "disabled" if not enabled else ("healthy" if ready else "unhealthy")
- return {
- "enabled": enabled,
- "ready": (ready if enabled else True),
- "status": status,
- **details,
- }
-
- def _health_checks() -> dict[str, dict[str, Any]]:
- memory_status = (
- proxy.memory_handler.health_status()
- if proxy.memory_handler
- else {
- "enabled": False,
- "backend": None,
- "initialized": False,
- "native_tool": False,
- "bridge_enabled": False,
- }
- )
- memory_enabled = bool(memory_status.get("enabled", False))
- memory_initialized = bool(memory_status.get("initialized", False))
- return {
- "startup": _component_health(
- enabled=True,
- ready=bool(getattr(app.state, "ready", False)),
- error=getattr(app.state, "startup_error", None),
- ),
- "http_client": _component_health(
- enabled=True,
- ready=proxy.http_client is not None,
- ),
- "cache": _component_health(
- enabled=config.cache_enabled,
- ready=(proxy.cache is not None),
- ),
- "rate_limiter": _component_health(
- enabled=config.rate_limit_enabled,
- ready=(proxy.rate_limiter is not None),
- ),
- "memory": _component_health(
- enabled=memory_enabled,
- ready=memory_initialized,
- backend=memory_status["backend"],
- initialized=memory_initialized,
- native_tool=bool(memory_status.get("native_tool", False)),
- bridge_enabled=bool(memory_status.get("bridge_enabled", False)),
- ),
- }
-
- def _runtime_payload() -> dict[str, Any]:
- ws_registry = getattr(proxy, "ws_sessions", None)
- ws_active_sessions = ws_registry.active_count() if ws_registry is not None else 0
- ws_active_relay_tasks = (
- ws_registry.active_relay_task_count() if ws_registry is not None else 0
- )
- return {
- "anthropic_pre_upstream": {
- "enabled": proxy.anthropic_pre_upstream_sem is not None,
- "resolved_concurrency": proxy.anthropic_pre_upstream_concurrency,
- "source": (
- "auto" if config.anthropic_pre_upstream_concurrency is None else "explicit"
- ),
- "acquire_timeout_seconds": proxy.anthropic_pre_upstream_acquire_timeout_seconds,
- "compression_timeout_seconds": float(COMPRESSION_TIMEOUT_SECONDS),
- "memory_context_timeout_seconds": (
- proxy.anthropic_pre_upstream_memory_context_timeout_seconds
- ),
- "codex_ws_gated": False,
- },
- "websocket_sessions": {
- "active_sessions": ws_active_sessions,
- "active_relay_tasks": ws_active_relay_tasks,
- },
- }
-
- def _health_payload(*, include_config: bool) -> dict[str, Any]:
- checks = _health_checks()
- ready = all(check["ready"] for check in checks.values())
- payload: dict[str, Any] = {
- "service": "headroom-proxy",
- "status": "healthy" if ready else "unhealthy",
- "ready": ready,
- "version": __version__,
- "timestamp": _iso_utc_now(),
- "uptime_seconds": _uptime_seconds(),
- "checks": checks,
- "runtime": _runtime_payload(),
- }
- deployment_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE")
- if deployment_profile:
- payload["deployment"] = {
- "profile": deployment_profile,
- "preset": os.environ.get("HEADROOM_DEPLOYMENT_PRESET"),
- "runtime": os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME"),
- "supervisor": os.environ.get("HEADROOM_DEPLOYMENT_SUPERVISOR"),
- "scope": os.environ.get("HEADROOM_DEPLOYMENT_SCOPE"),
- }
- if include_config:
- payload["config"] = {
- "backend": config.backend,
- "optimize": config.optimize,
- "cache": config.cache_enabled,
- "rate_limit": config.rate_limit_enabled,
- "memory": config.memory_enabled,
- "learn": config.traffic_learning_enabled,
- "code_graph": config.code_graph_watcher,
- "pid": os.getpid(),
- }
- return payload
-
- # CORS
- app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"],
- allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
- )
-
- # X-Headroom-Stack: SDK adapters (TS openai/anthropic/etc.) tag their
- # requests so telemetry can segment by integration surface. Registered
- # before extension middleware so any extension-level auth/guards run
- # outermost and we don't count requests they reject.
- @app.middleware("http")
- async def _record_headroom_stack(request, call_next):
- if request.url.path.startswith("/v1/"):
- stack = request.headers.get("x-headroom-stack")
- if stack:
- try:
- proxy.metrics.record_stack(stack)
- except Exception:
- logger.debug("record_stack failed", exc_info=True)
- return await call_next(request)
-
- # Third-party proxy extensions (Enterprise, custom plugins). Discovered via
- # the `headroom.proxy_extension` entry-point group, but **opt-in only**:
- # only names listed in config.proxy_extensions (CLI: --proxy-extension,
- # env: HEADROOM_PROXY_EXTENSIONS) actually get installed. Discovery alone
- # never runs third-party code. An extension that raises from its install()
- # is a deliberate fail-closed signal and aborts startup.
- from headroom.proxy.extensions import install_all as _install_extensions
-
- _install_extensions(app, config, enabled=getattr(config, "proxy_extensions", None))
-
- # Health & Metrics
- @app.get("/livez")
- async def livez():
- return JSONResponse(
- status_code=200,
- content={
- "service": "headroom-proxy",
- "status": "healthy",
- "alive": True,
- "version": __version__,
- "timestamp": _iso_utc_now(),
- "uptime_seconds": _uptime_seconds(),
- },
- )
-
- @app.get("/readyz")
- async def readyz():
- payload = _health_payload(include_config=False)
- return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)
-
- @app.get("/health")
- async def health():
- payload = _health_payload(include_config=True)
- return JSONResponse(status_code=200, content=payload)
-
- # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —
- # debug endpoints are invisible to external scanners.
- from headroom.proxy.debug_introspection import (
- collect_tasks as _collect_tasks,
- )
- from headroom.proxy.loopback_guard import require_loopback as _require_loopback
-
- @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])
- async def debug_tasks(stack: bool = False):
- """Enumerate running asyncio tasks.
-
- Default is cheap — ``stack_depth`` is ``null`` in every entry so
- a storm snapshot does not walk 50+ coroutine frames synchronously.
- Pass ``?stack=true`` to compute ``stack_depth`` for each task
- (useful for single-shot human debugging).
- """
- ws_registry = getattr(proxy, "ws_sessions", None)
- return JSONResponse(
- status_code=200,
- content=_collect_tasks(ws_registry, with_stack_depth=stack),
- )
-
- @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])
- async def debug_ws_sessions():
- ws_registry = getattr(proxy, "ws_sessions", None)
- snapshot = ws_registry.snapshot() if ws_registry is not None else []
- return JSONResponse(status_code=200, content=snapshot)
-
- @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])
- async def debug_warmup():
- warmup_registry = getattr(proxy, "warmup", None)
- payload = warmup_registry.to_dict() if warmup_registry is not None else {}
- payload["runtime"] = _runtime_payload()
- return JSONResponse(status_code=200, content=payload)
-
- @app.get("/dashboard", response_class=HTMLResponse)
- async def dashboard():
- """Serve the Headroom dashboard UI."""
- return get_dashboard_html()
-
- DASHBOARD_STATS_CACHE_TTL_SECONDS = 5.0
- _stats_snapshot_lock = asyncio.Lock()
- _stats_snapshot: dict[str, Any] = {"expires_at": 0.0, "value": None}
-
- async def _build_stats_payload() -> dict[str, Any]:
- """Build the full `/stats` response payload.
-
- This is the main stats endpoint - it aggregates data from all subsystems:
- - Request metrics (total, cached, failed, by model/provider)
- - Token usage and savings
- - Cost tracking
- - Canonical persisted display_session metrics for downstream dashboards
- - Compression (CCR) statistics
- - Telemetry/TOIN (data flywheel) statistics
- - Cache and rate limiter stats
- """
- m = proxy.metrics
-
- # Calculate average latency
- avg_latency_ms = round(m.latency_sum_ms / m.latency_count, 2) if m.latency_count > 0 else 0
- min_latency_ms = (
- round(m.latency_min_ms, 2)
- if m.latency_count > 0 and m.latency_min_ms != float("inf")
- else 0
- )
- max_latency_ms = round(m.latency_max_ms, 2) if m.latency_count > 0 else 0
-
- # Calculate Headroom overhead (optimization time only, excludes pass-through requests)
- avg_overhead_ms = (
- round(m.overhead_sum_ms / m.overhead_count, 2) if m.overhead_count > 0 else 0
- )
- min_overhead_ms = (
- round(m.overhead_min_ms, 2)
- if m.overhead_count > 0 and m.overhead_min_ms != float("inf")
- else 0
- )
- max_overhead_ms = round(m.overhead_max_ms, 2) if m.overhead_count > 0 else 0
-
- # Calculate TTFB (time to first byte)
- avg_ttfb_ms = round(m.ttfb_sum_ms / m.ttfb_count, 2) if m.ttfb_count > 0 else 0
- min_ttfb_ms = (
- round(m.ttfb_min_ms, 2) if m.ttfb_count > 0 and m.ttfb_min_ms != float("inf") else 0
- )
- max_ttfb_ms = round(m.ttfb_max_ms, 2) if m.ttfb_count > 0 else 0
-
- # Get compression store stats
- store = get_compression_store()
- compression_stats = store.get_stats()
-
- # Get telemetry/TOIN stats
- telemetry = get_telemetry_collector()
- telemetry_stats = telemetry.get_stats()
-
- # Get feedback loop stats
- feedback = get_compression_feedback()
- feedback_stats = feedback.get_stats()
-
- # Build prefix cache stats once (used in both prefix_cache and cost)
- prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)
-
- # Fetch CLI filtering savings (rtk — tokens avoided before reaching context)
- cli_filtering_stats = _get_rtk_stats()
- cli_tokens_avoided = (
- cli_filtering_stats.get("tokens_saved", 0) if cli_filtering_stats else 0
- )
-
- # Calculate total tokens before compression
- total_tokens_before = m.tokens_input_total + m.tokens_saved_total
-
- # Build human-readable summary
- summary = _build_session_summary(
- proxy, m, prefix_cache_stats, cli_tokens_avoided, total_tokens_before
- )
- # DEBUG: log the summary payload for external upsert consumers
- try:
- logger.debug("/stats summary data: %r", summary)
- except Exception:
- logger.warning("Failed to log /stats summary payload")
-
- # Compression cache stats (token mode)
- compression_cache_stats: dict = {}
- if proxy.config.mode == PROXY_MODE_TOKEN and proxy._compression_caches:
- total_entries = 0
- total_hits = 0
- total_misses = 0
- total_tokens_saved = 0
- for cache in proxy._compression_caches.values():
- s = cache.get_stats()
- total_entries += s.get("entries", 0)
- total_hits += s.get("hits", 0)
- total_misses += s.get("misses", 0)
- total_tokens_saved += s.get("total_tokens_saved", 0)
- compression_cache_stats = {
- "mode": PROXY_MODE_TOKEN,
- "active_sessions": len(proxy._compression_caches),
- "total_entries": total_entries,
- "total_hits": total_hits,
- "total_misses": total_misses,
- "hit_rate": round(total_hits / max(1, total_hits + total_misses) * 100, 1),
- "total_tokens_saved": total_tokens_saved,
- }
- else:
- compression_cache_stats = {"mode": proxy.config.mode}
-
- # Build unified savings summary (all layers)
- compression_tokens = m.tokens_saved_total
- cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
- total_tokens_all_layers = compression_tokens + cli_tokens_avoided
- persistent_savings = m.savings_tracker.stats_preview()
- display_session = persistent_savings.get("display_session", {})
-
- return {
- "summary": summary,
- "savings": {
- "total_tokens": total_tokens_all_layers,
- "by_layer": {
- "cli_filtering": {
- "tokens": cli_tokens_avoided,
- "description": "Tokens avoided by CLI output filtering (rtk) before reaching context",
- },
- "compression": {
- "tokens": compression_tokens,
- "description": "Tokens removed by proxy compression (SmartCrusher, ContentRouter, etc.)",
- },
- "prefix_cache": {
- "discount_usd": round(cache_net_usd, 4),
- "description": (
- "Cost discount from provider prefix caching. "
- "Headroom's CacheAligner improves hit rates; "
- "baseline caching is provider-native."
- ),
- },
- },
- },
- "requests": {
- "total": m.requests_total,
- "cached": m.requests_cached,
- "rate_limited": m.requests_rate_limited,
- "failed": m.requests_failed,
- "by_provider": dict(m.requests_by_provider),
- "by_model": dict(m.requests_by_model),
- "by_stack": dict(m.requests_by_stack),
- },
- "tokens": {
- "input": m.tokens_input_total,
- "output": m.tokens_output_total,
- "saved": m.tokens_saved_total,
- "cli_tokens_avoided": cli_tokens_avoided,
- "total_before_compression": total_tokens_before,
- "savings_percent": round(
- (m.tokens_saved_total / total_tokens_before * 100)
- if total_tokens_before > 0
- else 0,
- 2,
- ),
- },
- "latency": {
- "average_ms": avg_latency_ms,
- "min_ms": min_latency_ms,
- "max_ms": max_latency_ms,
- "total_requests": m.latency_count,
- },
- "overhead": {
- "average_ms": avg_overhead_ms,
- "min_ms": min_overhead_ms,
- "max_ms": max_overhead_ms,
- },
- "ttfb": {
- "average_ms": avg_ttfb_ms,
- "min_ms": min_ttfb_ms,
- "max_ms": max_ttfb_ms,
- },
- "pipeline_timing": {
- name: {
- "average_ms": round(
- m.transform_timing_sum[name] / m.transform_timing_count[name], 2
- ),
- "max_ms": round(m.transform_timing_max[name], 2),
- "count": m.transform_timing_count[name],
- }
- for name in sorted(m.transform_timing_sum.keys())
- }
- if m.transform_timing_sum
- else {},
- "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
- "savings_history": m.savings_history[-100:], # Last 100 data points
- "display_session": display_session,
- "persistent_savings": persistent_savings,
- "prefix_cache": prefix_cache_stats,
- "cost": _merge_cost_stats(
- proxy.cost_tracker.stats() if proxy.cost_tracker else None,
- prefix_cache_stats,
- cli_tokens_avoided=cli_tokens_avoided,
- ),
- "compression": {
- "ccr_entries": compression_stats.get("entry_count", 0),
- "ccr_max_entries": compression_stats.get("max_entries", 0),
- "original_tokens_cached": compression_stats.get("total_original_tokens", 0),
- "compressed_tokens_cached": compression_stats.get("total_compressed_tokens", 0),
- "ccr_retrievals": compression_stats.get("total_retrievals", 0),
- },
- "compression_cache": compression_cache_stats,
- "anon_telemetry_shipping": is_telemetry_enabled(),
- "telemetry": {
- "enabled": telemetry_stats.get("enabled", False),
- "total_compressions": telemetry_stats.get("total_compressions", 0),
- "total_retrievals": telemetry_stats.get("total_retrievals", 0),
- "global_retrieval_rate": round(telemetry_stats.get("global_retrieval_rate", 0), 4),
- "tool_signatures_tracked": telemetry_stats.get("tool_signatures_tracked", 0),
- "avg_compression_ratio": round(telemetry_stats.get("avg_compression_ratio", 0), 4),
- "avg_token_reduction": round(telemetry_stats.get("avg_token_reduction", 0), 4),
- },
- "otel": get_otel_metrics_status(),
- "langfuse": get_langfuse_tracing_status(),
- "feedback_loop": {
- "tools_tracked": feedback_stats.get("tools_tracked", 0),
- "total_compressions": feedback_stats.get("total_compressions", 0),
- "total_retrievals": feedback_stats.get("total_retrievals", 0),
- "global_retrieval_rate": round(feedback_stats.get("global_retrieval_rate", 0), 4),
- "tools_with_high_retrieval": sum(
- 1
- for p in feedback_stats.get("tool_patterns", {}).values()
- if p.get("retrieval_rate", 0) > 0.3
- ),
- },
- "toin": get_toin().get_stats(),
- "cli_filtering": cli_filtering_stats,
- "cache": await proxy.cache.stats() if proxy.cache else None,
- "rate_limiter": await proxy.rate_limiter.stats() if proxy.rate_limiter else None,
- "recent_requests": proxy.logger.get_recent(10) if proxy.logger else [],
- "log_full_messages": proxy.config.log_full_messages if proxy else False,
- **get_quota_registry().get_all_stats(),
- }
-
- async def _get_cached_stats_payload() -> dict[str, Any]:
- """Return a short-TTL cached `/stats` snapshot for dashboard polling."""
- now = time.monotonic()
- cached_payload = cast(dict[str, Any] | None, _stats_snapshot.get("value"))
- if cached_payload is not None and now < float(_stats_snapshot["expires_at"]):
- return cached_payload
-
- async with _stats_snapshot_lock:
- now = time.monotonic()
- cached_payload = cast(dict[str, Any] | None, _stats_snapshot.get("value"))
- if cached_payload is not None and now < float(_stats_snapshot["expires_at"]):
- return cached_payload
-
- payload = await _build_stats_payload()
- _stats_snapshot["value"] = payload
- _stats_snapshot["expires_at"] = time.monotonic() + DASHBOARD_STATS_CACHE_TTL_SECONDS
- return payload
-
- @app.get("/stats")
- async def stats(cached: bool = False):
- """Get comprehensive proxy statistics.
-
- This is the main stats endpoint - it aggregates data from all subsystems:
- - Request metrics (total, cached, failed, by model/provider)
- - Token usage and savings
- - Cost tracking
- - Canonical persisted display_session metrics for downstream dashboards
- - Compression (CCR) statistics
- - Telemetry/TOIN (data flywheel) statistics
- - Cache and rate limiter stats
-
- Use ``?cached=1`` for the dashboard fast path. That returns a short-TTL
- snapshot to avoid rebuilding the full payload on every UI poll.
- """
- if cached:
- return await _get_cached_stats_payload()
- return await _build_stats_payload()
-
- @app.get("/stats-history")
- async def stats_history(
- format: Literal["json", "csv"] = "json",
- series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
- history_mode: Literal["compact", "full", "none"] = "compact",
- ):
- """Get durable proxy compression history plus display-session state."""
- if format == "csv":
- filename = f"headroom-stats-history-{series}.csv"
- return Response(
- content=proxy.metrics.savings_tracker.export_csv(series=series),
- media_type="text/csv; charset=utf-8",
- headers={"Content-Disposition": f'attachment; filename="{filename}"'},
- )
-
- return proxy.metrics.savings_tracker.history_response(history_mode=history_mode)
-
- @app.get("/transformations/feed")
- async def transformations_feed(limit: int = 20):
- """Get recent message transformations for the live feed.
-
- Returns empty list if log_full_messages is disabled (messages are not stored).
- """
- if limit > 100:
- limit = 100
-
- transformations = []
- log_full_messages = proxy.config.log_full_messages if proxy else False
-
- if proxy and proxy.logger:
- logs = proxy.logger.get_recent_with_messages(limit)
- for log in logs:
- transformations.append(
- {
- "request_id": log.get("request_id"),
- "timestamp": log.get("timestamp"),
- "provider": log.get("provider"),
- "model": log.get("model"),
- "input_tokens_original": log.get("input_tokens_original"),
- "input_tokens_optimized": log.get("input_tokens_optimized"),
- "tokens_saved": log.get("tokens_saved"),
- "savings_percent": log.get("savings_percent"),
- "transforms_applied": log.get("transforms_applied", []),
- "request_messages": log.get("request_messages"),
- "response_content": log.get("response_content"),
- }
- )
-
- return {"transformations": transformations, "log_full_messages": log_full_messages}
-
- @app.get("/subscription-window")
- async def subscription_window():
- """Current Anthropic subscription window utilisation and Headroom contribution."""
- tracker = get_subscription_tracker()
- if tracker is None:
- return JSONResponse(
- status_code=503,
- content={"error": "Subscription tracking is not enabled"},
- )
- return JSONResponse(content=tracker.state)
-
- @app.get("/quota")
- async def quota():
- """Unified quota/rate-limit stats for all registered providers (Anthropic, Codex, Copilot)."""
- return JSONResponse(content=get_quota_registry().get_all_stats())
-
- @app.get("/metrics")
- async def metrics():
- """Prometheus metrics endpoint."""
- return PlainTextResponse(
- await proxy.metrics.export(),
- media_type="text/plain; version=0.0.4",
- )
-
- # Debug endpoints
- @app.get("/debug/memory")
- async def debug_memory():
- """Get detailed memory usage statistics.
-
- Returns memory usage for all tracked components including:
- - Process-level memory (RSS, VMS, percent)
- - Per-component memory usage and budgets
- - Cache hit/miss statistics
- - Total tracked vs target budget
-
- This endpoint is useful for debugging memory issues and
- monitoring memory budgets.
- """
- from ..memory.tracker import MemoryTracker
-
- tracker = MemoryTracker.get()
-
- # Register components if not already registered
- _register_memory_components(proxy, tracker)
-
- report = tracker.get_report()
- return report.to_dict()
-
- @app.post("/cache/clear")
- async def clear_cache():
- """Clear the response cache."""
- if proxy.cache:
- await proxy.cache.clear()
- return {"status": "cleared"}
- return {"status": "cache disabled"}
-
- # CCR (Compress-Cache-Retrieve) endpoints
- @app.post("/v1/retrieve")
- async def ccr_retrieve(request: Request):
- """Retrieve original content from CCR compression cache.
-
- This is the "Retrieve" part of CCR (Compress-Cache-Retrieve).
- When SmartCrusher compresses tool outputs, the original data is cached.
- LLMs can call this endpoint to get more data if needed.
-
- Request body:
- hash (str): Hash key from compression marker (required)
- query (str): Optional search query to filter results
-
- Response:
- Full retrieval: {"hash": "...", "original_content": "...", ...}
- Search: {"hash": "...", "query": "...", "results": [...], "count": N}
- """
- data = await request.json()
- hash_key = data.get("hash")
- query = data.get("query")
-
- if not hash_key:
- raise HTTPException(status_code=400, detail="hash required")
-
- store = get_compression_store()
-
- if query:
- # Search within cached content
- results = store.search(hash_key, query)
- return {
- "hash": hash_key,
- "query": query,
- "results": results,
- "count": len(results),
- }
- else:
- # Return full original content
- entry = store.retrieve(hash_key)
- if entry:
- return {
- "hash": hash_key,
- "original_content": entry.original_content,
- "original_tokens": entry.original_tokens,
- "original_item_count": entry.original_item_count,
- "compressed_item_count": entry.compressed_item_count,
- "tool_name": entry.tool_name,
- "retrieval_count": entry.retrieval_count,
- }
- raise HTTPException(
- status_code=404, detail="Entry not found or expired (TTL: 5 minutes)"
- )
-
- @app.get("/v1/retrieve/stats")
- async def ccr_stats():
- """Get CCR compression store statistics."""
- store = get_compression_store()
- stats = store.get_stats()
- events = store.get_retrieval_events(limit=20)
- return {
- "store": stats,
- "recent_retrievals": [
- {
- "hash": e.hash,
- "query": e.query,
- "items_retrieved": e.items_retrieved,
- "total_items": e.total_items,
- "tool_name": e.tool_name,
- "retrieval_type": e.retrieval_type,
- }
- for e in events
- ],
- }
-
- @app.get("/v1/feedback")
- async def ccr_feedback():
- """Get CCR feedback loop statistics and learned patterns.
-
- This endpoint exposes the feedback loop's learned patterns for monitoring
- and debugging. It shows:
- - Per-tool retrieval rates (high = compress less aggressively)
- - Common search queries per tool
- - Queried fields (suggest what to preserve)
-
- Use this to understand how well compression is working and whether
- the feedback loop is adjusting appropriately.
- """
- feedback = get_compression_feedback()
- stats = feedback.get_stats()
- return {
- "feedback": stats,
- "hints_example": {
- tool_name: {
- "hints": {
- "max_items": hints.max_items
- if (hints := feedback.get_compression_hints(tool_name))
- else 15,
- "suggested_items": hints.suggested_items if hints else None,
- "skip_compression": hints.skip_compression if hints else False,
- "preserve_fields": hints.preserve_fields if hints else [],
- "reason": hints.reason if hints else "",
- }
- }
- for tool_name in list(stats.get("tool_patterns", {}).keys())[:5]
- },
- }
-
- @app.get("/v1/feedback/{tool_name}")
- async def ccr_feedback_for_tool(tool_name: str):
- """Get compression hints for a specific tool.
-
- Returns feedback-based hints that would be used for compressing
- this tool's output.
- """
- feedback = get_compression_feedback()
- hints = feedback.get_compression_hints(tool_name)
- patterns = feedback.get_all_patterns().get(tool_name)
-
- return {
- "tool_name": tool_name,
- "hints": {
- "max_items": hints.max_items,
- "min_items": hints.min_items,
- "suggested_items": hints.suggested_items,
- "aggressiveness": hints.aggressiveness,
- "skip_compression": hints.skip_compression,
- "preserve_fields": hints.preserve_fields,
- "reason": hints.reason,
- },
- "pattern": {
- "total_compressions": patterns.total_compressions if patterns else 0,
- "total_retrievals": patterns.total_retrievals if patterns else 0,
- "retrieval_rate": patterns.retrieval_rate if patterns else 0.0,
- "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0,
- "search_rate": patterns.search_rate if patterns else 0.0,
- "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [],
- "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [],
- }
- if patterns
- else None,
- }
-
- # Telemetry endpoints (Data Flywheel)
- @app.get("/v1/telemetry")
- async def telemetry_stats():
- """Get telemetry statistics for the data flywheel.
-
- This endpoint exposes privacy-preserving telemetry data that powers
- the data flywheel - learning optimal compression strategies across
- tool types based on usage patterns.
-
- What's collected (anonymized):
- - Tool output structure patterns (field types, not values)
- - Compression decisions and ratios
- - Retrieval patterns (rate, type, not content)
- - Strategy effectiveness
-
- What's NOT collected:
- - Actual data values
- - User identifiers
- - Queries or search terms
- - File paths or tool names (hashed by default)
- """
- telemetry = get_telemetry_collector()
- return telemetry.get_stats()
-
- @app.get("/v1/telemetry/export")
- async def telemetry_export():
- """Export full telemetry data for aggregation.
-
- This endpoint exports all telemetry data in a format suitable for
- cross-user aggregation. The data is privacy-preserving - no actual
- values are included, only structural patterns and statistics.
-
- Use this for:
- - Building a central learning service
- - Sharing learned patterns across instances
- - Analysis and debugging
- """
- telemetry = get_telemetry_collector()
- return telemetry.export_stats()
-
- @app.post("/v1/telemetry/import")
- async def telemetry_import(request: Request):
- """Import telemetry data from another source.
-
- This allows merging telemetry from multiple sources for cross-user
- learning. The imported data is merged with existing statistics.
-
- Request body: Telemetry export data from /v1/telemetry/export
- """
- telemetry = get_telemetry_collector()
- data = await request.json()
- telemetry.import_stats(data)
- return {"status": "imported", "current_stats": telemetry.get_stats()}
-
- @app.get("/v1/telemetry/tools")
- async def telemetry_tools():
- """Get telemetry statistics for all tracked tool signatures.
-
- Returns statistics per tool signature (anonymized), including:
- - Compression ratios and strategy usage
- - Retrieval rates (high = compression too aggressive)
- - Learned recommendations
- """
- telemetry = get_telemetry_collector()
- all_stats = telemetry.get_all_tool_stats()
- return {
- "tool_count": len(all_stats),
- "tools": {sig_hash: stats.to_dict() for sig_hash, stats in all_stats.items()},
- }
-
- @app.get("/v1/telemetry/tools/{signature_hash}")
- async def telemetry_tool_detail(signature_hash: str):
- """Get detailed telemetry for a specific tool signature.
-
- Includes learned recommendations if enough data has been collected.
- """
- telemetry = get_telemetry_collector()
- stats = telemetry.get_tool_stats(signature_hash)
- recommendations = telemetry.get_recommendations(signature_hash)
-
- if stats is None:
- raise HTTPException(
- status_code=404, detail=f"No telemetry found for signature: {signature_hash}"
- )
-
- return {
- "signature_hash": signature_hash,
- "stats": stats.to_dict(),
- "recommendations": recommendations,
- }
-
- # TOIN (Tool Output Intelligence Network) endpoints
- @app.get("/v1/toin/stats")
- async def toin_stats():
- """Get overall TOIN statistics.
-
- Returns aggregated statistics from the Tool Output Intelligence Network,
- which learns optimal compression strategies across all tool types.
-
- Response includes:
- - enabled: Whether TOIN is enabled
- - patterns_tracked: Number of unique tool patterns being tracked
- - total_compressions: Total compression events recorded
- - total_retrievals: Total retrieval events recorded
- - global_retrieval_rate: Overall retrieval rate (high = compression too aggressive)
- - patterns_with_recommendations: Patterns with enough data for recommendations
- """
- toin = get_toin()
- return toin.get_stats()
-
- @app.get("/v1/toin/patterns")
- async def toin_patterns(limit: int = 20):
- """List TOIN patterns with most samples.
-
- Returns patterns sorted by sample_size descending. Use this to see
- which tool types have the most data and their learned behaviors.
-
- Query params:
- limit: Maximum number of patterns to return (default 20)
-
- Response includes for each pattern:
- - hash: Truncated tool signature hash (12 chars)
- - compressions: Total compression events
- - retrievals: Total retrieval events
- - retrieval_rate: Percentage of compressions that triggered retrieval
- - confidence: Confidence level in recommendations (0.0-1.0)
- - skip_recommended: Whether TOIN recommends skipping compression
- - optimal_max_items: Learned optimal max_items setting
- """
- toin = get_toin()
- exported = toin.export_patterns()
- patterns_data = exported.get("patterns", {})
-
- # Convert to list and sort by sample_size
- patterns_list = []
- for sig_hash, pattern_dict in patterns_data.items():
- sample_size = pattern_dict.get("sample_size", 0)
- total_compressions = pattern_dict.get("total_compressions", 0)
- total_retrievals = pattern_dict.get("total_retrievals", 0)
- retrieval_rate = (
- total_retrievals / total_compressions if total_compressions > 0 else 0.0
- )
-
- patterns_list.append(
- {
- "hash": sig_hash[:12],
- "compressions": total_compressions,
- "retrievals": total_retrievals,
- "retrieval_rate": f"{retrieval_rate:.1%}",
- "confidence": round(pattern_dict.get("confidence", 0.0), 3),
- "skip_recommended": pattern_dict.get("skip_compression_recommended", False),
- "optimal_max_items": pattern_dict.get("optimal_max_items", 20),
- "sample_size": sample_size,
- }
- )
-
- # Sort by sample_size descending
- patterns_list.sort(key=lambda p: p["sample_size"], reverse=True)
-
- # Remove sample_size from output (used only for sorting)
- for p in patterns_list:
- del p["sample_size"]
-
- return patterns_list[:limit]
-
- @app.get("/v1/toin/pattern/{hash_prefix}")
- async def toin_pattern_detail(hash_prefix: str):
- """Get detailed TOIN pattern info by hash prefix.
-
- Searches for a pattern where the tool signature hash starts with
- the provided prefix. Returns full pattern details if found.
-
- Path params:
- hash_prefix: Beginning of the tool signature hash (min 4 chars recommended)
-
- Response: Full pattern.to_dict() with all learned statistics and recommendations.
- """
- toin = get_toin()
- exported = toin.export_patterns()
- patterns_data = exported.get("patterns", {})
-
- # Search for pattern with matching hash prefix
- for sig_hash, pattern_dict in patterns_data.items():
- if sig_hash.startswith(hash_prefix):
- return pattern_dict
-
- raise HTTPException(
- status_code=404, detail=f"No TOIN pattern found with hash starting with: {hash_prefix}"
- )
-
- @app.get("/v1/retrieve/{hash_key}")
- async def ccr_retrieve_get(hash_key: str, query: str | None = None):
- """GET version of CCR retrieve for easier testing."""
- store = get_compression_store()
-
- if query:
- results = store.search(hash_key, query)
- return {
- "hash": hash_key,
- "query": query,
- "results": results,
- "count": len(results),
- }
- else:
- entry = store.retrieve(hash_key)
- if entry:
- return {
- "hash": hash_key,
- "original_content": entry.original_content,
- "original_tokens": entry.original_tokens,
- "original_item_count": entry.original_item_count,
- "compressed_item_count": entry.compressed_item_count,
- "tool_name": entry.tool_name,
- "retrieval_count": entry.retrieval_count,
- }
- raise HTTPException(status_code=404, detail="Entry not found or expired")
-
- # CCR Tool Call Handler - for agent frameworks to call when LLM uses headroom_retrieve
- @app.post("/v1/retrieve/tool_call")
- async def ccr_handle_tool_call(request: Request):
- """Handle a CCR tool call from an LLM response.
-
- This endpoint accepts tool call formats from various providers and returns
- a properly formatted tool result. Agent frameworks can use this to handle
- CCR tool calls without implementing the retrieval logic themselves.
-
- Request body (Anthropic format):
- {
- "tool_call": {
- "id": "toolu_123",
- "name": "headroom_retrieve",
- "input": {"hash": "abc123", "query": "optional search"}
- },
- "provider": "anthropic"
- }
-
- Request body (OpenAI format):
- {
- "tool_call": {
- "id": "call_123",
- "function": {
- "name": "headroom_retrieve",
- "arguments": "{\"hash\": \"abc123\"}"
- }
- },
- "provider": "openai"
- }
-
- Response:
- {
- "tool_result": {...}, # Formatted for the provider
- "success": true,
- "data": {...} # Raw retrieval data
- }
- """
- data = await request.json()
- tool_call = data.get("tool_call", {})
- provider = data.get("provider", "anthropic")
-
- # Parse the tool call
- hash_key, query = parse_tool_call(tool_call, provider)
-
- if hash_key is None:
- raise HTTPException(
- status_code=400, detail=f"Invalid tool call or not a {CCR_TOOL_NAME} call"
- )
-
- # Perform retrieval
- store = get_compression_store()
-
- if query:
- results = store.search(hash_key, query)
- retrieval_data = {
- "hash": hash_key,
- "query": query,
- "results": results,
- "count": len(results),
- }
- else:
- entry = store.retrieve(hash_key)
- if entry:
- retrieval_data = {
- "hash": hash_key,
- "original_content": entry.original_content,
- "original_item_count": entry.original_item_count,
- "compressed_item_count": entry.compressed_item_count,
- }
- else:
- retrieval_data = {
- "error": "Entry not found or expired (TTL: 5 minutes)",
- "hash": hash_key,
- }
-
- # Format tool result for provider
- tool_call_id = tool_call.get("id", "")
- result_content = json.dumps(retrieval_data, indent=2)
-
- if provider == "anthropic":
- tool_result = {
- "type": "tool_result",
- "tool_use_id": tool_call_id,
- "content": result_content,
- }
- elif provider == "openai":
- tool_result = {
- "role": "tool",
- "tool_call_id": tool_call_id,
- "content": result_content,
- }
- else:
- tool_result = {
- "tool_call_id": tool_call_id,
- "content": result_content,
- }
-
- return {
- "tool_result": tool_result,
- "success": "error" not in retrieval_data,
- "data": retrieval_data,
- }
-
- # Compression-only endpoint (for TypeScript SDK and other HTTP clients)
- @app.post("/v1/compress")
- async def compress_messages(request: Request):
- return await proxy.handle_compress(request)
-
- register_provider_routes(app, proxy)
-
- return app
-
-
- def _get_code_aware_banner_status(config: ProxyConfig) -> str:
- """Get code-aware compression status line for banner."""
- if config.code_aware_enabled:
- if is_tree_sitter_available():
- return "ENABLED (AST-based)"
- else:
- return "NOT INSTALLED (pip install headroom-ai[code])"
- else:
- if is_tree_sitter_available():
- return "DISABLED (remove --no-code-aware to enable)"
- return "DISABLED"
-
-
- def run_server(
- config: ProxyConfig | None = None,
- workers: int = 1,
- limit_concurrency: int = 1000,
- ):
- """Run the proxy server.
-
- Args:
- config: Proxy configuration
- workers: Number of worker processes (use N for multi-core scaling)
- limit_concurrency: Max concurrent connections before 503 response
- """
- if not FASTAPI_AVAILABLE:
- print("ERROR: FastAPI required. Install: pip install fastapi uvicorn httpx")
- sys.exit(1)
-
- config = config or ProxyConfig()
- app = create_app(config)
-
- code_aware_status = _get_code_aware_banner_status(config)
-
- # Format connection pool info
- pool_info = f"max={config.max_connections}, keepalive={config.max_keepalive_connections}"
- http2_status = "ENABLED" if config.http2 else "DISABLED"
-
- backend_status = format_backend_status(
- backend=config.backend,
- anyllm_provider=config.anyllm_provider,
- bedrock_region=config.bedrock_region,
- )
-
- print(f"""
- ╔══════════════════════════════════════════════════════════════════════╗
- ║ HEADROOM PROXY SERVER ║
- ╠══════════════════════════════════════════════════════════════════════╣
- ║ Version: 1.0.0 ║
- ║ Listening: http://{config.host}:{config.port:<5} ║
- ║ Workers: {workers:<3} Concurrency Limit: {limit_concurrency:<5} ║
- ║ Backend: {backend_status:<59}║
- ╠══════════════════════════════════════════════════════════════════════╣
- ║ FEATURES: ║
- ║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║
- ║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║
- ║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║
- ║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║
- ║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║
- ║ Code-Aware: {code_aware_status:<52}║
- ║ HTTP/2: {http2_status:<52}║
- ║ Conn Pool: {pool_info:<52}║
- ╠══════════════════════════════════════════════════════════════════════╣
- ║ USAGE: ║
- ║ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude ║
- ║ Cursor: Set base URL in settings ║
- ╠══════════════════════════════════════════════════════════════════════╣
- ║ ENDPOINTS: ║
- ║ /livez Process liveness ║
- ║ /readyz Traffic readiness ║
- ║ /health Aggregate health ║
- ║ /stats Detailed statistics ║
- ║ /metrics Prometheus metrics ║
- ║ /cache/clear Clear response cache ║
- ║ /v1/retrieve CCR: Retrieve compressed content ║
- ║ /v1/retrieve/stats CCR: Compression store stats ║
- ║ /v1/retrieve/tool_call CCR: Handle LLM tool calls ║
- ║ /v1/feedback CCR: Feedback loop stats & patterns ║
- ║ /v1/feedback/{{tool}} CCR: Compression hints for a tool ║
- ║ /v1/telemetry Data flywheel: Telemetry stats ║
- ║ /v1/telemetry/export Data flywheel: Export for aggregation ║
- ║ /v1/telemetry/tools Data flywheel: Per-tool stats ║
- ║ /v1/toin/stats TOIN: Overall intelligence stats ║
- ║ /v1/toin/patterns TOIN: List learned patterns ║
- ║ /v1/toin/pattern/{{hash}} TOIN: Pattern details by hash ║
- ╚══════════════════════════════════════════════════════════════════════╝
- """)
-
- uvicorn.run(
- app,
- host=config.host,
- port=config.port,
- log_level="warning",
- workers=workers if workers > 1 else None, # None = single process (default)
- limit_concurrency=limit_concurrency,
- # Defense-in-depth: the loopback guard for /debug/* endpoints trusts
- # request.client.host. uvicorn's ProxyHeadersMiddleware rewrites that
- # from X-Forwarded-For when FORWARDED_ALLOW_IPS is broader than the
- # default. Disabling proxy_headers here guarantees the guard sees the
- # real peer address regardless of env.
- proxy_headers=False,
- )
-
-
- def _get_env_bool(name: str, default: bool) -> bool:
- """Get boolean from environment variable."""
- val = os.environ.get(name)
- if val is None:
- return default
- return val.lower() in ("true", "1", "yes", "on")
-
-
- def _get_env_int(name: str, default: int) -> int:
- """Get integer from environment variable."""
- val = os.environ.get(name)
- if val is None:
- return default
- try:
- return int(val)
- except ValueError:
- return default
-
-
- def _get_env_float(name: str, default: float) -> float:
- """Get float from environment variable."""
- val = os.environ.get(name)
- if val is None:
- return default
- try:
- return float(val)
- except ValueError:
- return default
-
-
- def _get_env_str(name: str, default: str) -> str:
- """Get string from environment variable."""
- return os.environ.get(name, default)
-
-
- def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:
- """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.
-
- Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)
- Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")
-
- Returns:
- Dict mapping tool names to CompressionProfile instances.
- """
- from headroom.config import PROFILE_PRESETS, CompressionProfile
-
- profiles: dict[str, CompressionProfile] = {}
- raw_entries: list[str] = list(cli_profiles)
-
- # Also check env var
- env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")
- if env_val:
- raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())
-
- for entry in raw_entries:
- if ":" not in entry:
- logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)
- continue
- tool_name, level = entry.split(":", 1)
- tool_name = tool_name.strip()
- level = level.strip().lower()
-
- if level in PROFILE_PRESETS:
- profiles[tool_name] = PROFILE_PRESETS[level]
- else:
- logger.warning(
- "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",
- level,
- tool_name,
- )
-
- return profiles
-
-
- if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Headroom Proxy Server")
-
- # Server
- parser.add_argument("--host", default="127.0.0.1")
- parser.add_argument("--port", type=int, default=8787)
- parser.add_argument(
- "--openai-api-url", help=f"Custom OpenAI API URL (default: {DEFAULT_OPENAI_API_URL})"
- )
- parser.add_argument(
- "--anthropic-api-url",
- help=f"Custom Anthropic API URL (default: {DEFAULT_ANTHROPIC_API_URL})",
- )
-
- # Backend (anthropic direct, bedrock, openrouter, anyllm, or litellm-<provider>)
- parser.add_argument(
- "--backend",
- default="anthropic",
- help=(
- "Backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', "
- "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-hosted_vllm, litellm-vertex)"
- ),
- )
- parser.add_argument(
- "--bedrock-region",
- default="us-west-2",
- help="AWS region for Bedrock backend (default: us-west-2)",
- )
- parser.add_argument(
- "--bedrock-profile",
- help="AWS profile for Bedrock backend (default: use default credentials)",
- )
- parser.add_argument(
- "--openrouter-api-key",
- help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",
- )
- parser.add_argument(
- "--anyllm-provider",
- default="openai",
- help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",
- )
-
- # Connection pool (scalability)
- parser.add_argument(
- "--max-connections",
- type=int,
- default=500,
- help="Max connections to upstream APIs (default: 500)",
- )
- parser.add_argument(
- "--max-keepalive", type=int, default=100, help="Max keepalive connections (default: 100)"
- )
- parser.add_argument(
- "--no-http2",
- action="store_true",
- help="Disable HTTP/2 (enabled by default for better throughput)",
- )
- parser.add_argument(
- "--workers",
- type=int,
- default=1,
- help="Number of worker processes (default: 1, use N for multi-core)",
- )
- parser.add_argument(
- "--limit-concurrency",
- type=int,
- default=1000,
- help="Max concurrent connections before 503 (default: 1000)",
- )
-
- # Optimization
- parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")
- parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")
- parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")
- parser.add_argument(
- "--tool-profile",
- action="append",
- default=[],
- help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "
- "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",
- )
-
- # Caching
- parser.add_argument("--no-cache", action="store_true", help="Disable caching")
- parser.add_argument("--cache-ttl", type=int, default=3600, help="Cache TTL seconds")
-
- # Rate limiting
- parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")
- parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")
- parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")
-
- # Cost
- parser.add_argument("--budget", type=float, help="Budget limit in USD")
- parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")
-
- # Logging
- parser.add_argument("--log-file", help="Log file path")
- parser.add_argument("--log-messages", action="store_true", help="Log full messages")
-
- # Smart routing (content-aware compression)
- parser.add_argument(
- "--no-smart-routing",
- action="store_true",
- help="Disable smart routing (use legacy sequential pipeline)",
- )
-
- # Code-aware compression
- parser.add_argument(
- "--code-aware",
- action="store_true",
- help="Enable AST-based code compression (requires: pip install headroom-ai[code])",
- )
- parser.add_argument(
- "--no-code-aware",
- action="store_true",
- help="Disable code-aware compression",
- )
-
- args = parser.parse_args()
-
- # Environment variable defaults (HEADROOM_* prefix)
- # CLI args override env vars, env vars override ProxyConfig defaults
- env_smart_routing = _get_env_bool("HEADROOM_SMART_ROUTING", True)
- env_code_aware = _get_env_bool("HEADROOM_CODE_AWARE_ENABLED", True)
- env_optimize = _get_env_bool("HEADROOM_OPTIMIZE", True)
- env_cache = _get_env_bool("HEADROOM_CACHE_ENABLED", True)
- env_rate_limit = _get_env_bool("HEADROOM_RATE_LIMIT_ENABLED", True)
-
- # Determine settings: CLI flags override env vars
- # --no-X explicitly disables, --X explicitly enables, neither uses env var
- smart_routing = env_smart_routing if not args.no_smart_routing else False
- code_aware_enabled = (
- env_code_aware
- if not (args.code_aware or args.no_code_aware)
- else (args.code_aware or not args.no_code_aware)
- )
- optimize = env_optimize if not args.no_optimize else False
- cache_enabled = env_cache if not args.no_cache else False
- rate_limit_enabled = env_rate_limit if not args.no_rate_limit else False
-
- # Set OpenRouter API key from CLI if provided
- if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:
- os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key
-
- # Parse per-tool compression profiles from CLI and env var
- tool_profiles = _parse_tool_profiles(args.tool_profile)
-
- config = ProxyConfig(
- host=_get_env_str("HEADROOM_HOST", args.host),
- port=_get_env_int("HEADROOM_PORT", args.port),
- openai_api_url=_get_env_str("OPENAI_TARGET_API_URL", args.openai_api_url),
- anthropic_api_url=_get_env_str("ANTHROPIC_TARGET_API_URL", args.anthropic_api_url),
- # Backend settings
- backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]
- bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),
- bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),
- anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),
- optimize=optimize,
- min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),
- max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),
- cache_enabled=cache_enabled,
- cache_ttl_seconds=_get_env_int("HEADROOM_CACHE_TTL", args.cache_ttl),
- rate_limit_enabled=rate_limit_enabled,
- rate_limit_requests_per_minute=_get_env_int("HEADROOM_RPM", args.rpm),
- rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),
- budget_limit_usd=args.budget,
- budget_period=args.budget_period,
- log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)
- if args.log_file
- else os.environ.get("HEADROOM_LOG_FILE"),
- log_full_messages=args.log_messages or _get_env_bool("HEADROOM_LOG_MESSAGES", False),
- smart_routing=smart_routing,
- code_aware_enabled=code_aware_enabled,
- # Connection pool settings
- max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),
- max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),
- http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),
- tool_profiles=tool_profiles if tool_profiles else None,
- mode=normalize_proxy_mode(_get_env_str("HEADROOM_MODE", PROXY_MODE_TOKEN)),
- )
-
- # Get worker and concurrency settings
- workers = _get_env_int("HEADROOM_WORKERS", args.workers)
- limit_concurrency = _get_env_int("HEADROOM_LIMIT_CONCURRENCY", args.limit_concurrency)
-
- run_server(config, workers=workers, limit_concurrency=limit_concurrency)
+ """Headroom Proxy Server - Production Ready.
+
+ A full-featured LLM proxy with optimization, caching, rate limiting,
+ and observability.
+
+ Features:
+ - Context optimization (SmartCrusher, CacheAligner, RollingWindow)
+ - Semantic caching (save costs on repeated queries)
+ - Rate limiting (token bucket)
+ - Retry with exponential backoff
+ - Cost tracking and budgets
+ - Request tagging and metadata
+ - Provider fallback
+ - Prometheus metrics
+ - Full request/response logging
+
+ Usage:
+ python -m headroom.proxy.server --port 8787
+
+ # With Claude Code:
+ ANTHROPIC_BASE_URL=http://localhost:8787 claude
+ """
+
+ from __future__ import annotations
+
+ import argparse
+ import asyncio
+ import json
+ import logging
+ import os
+ import sys
+ import time
+ from datetime import datetime, timezone
+ from pathlib import Path
+ from typing import TYPE_CHECKING, Any, Literal, cast
+
+ if TYPE_CHECKING:
+ from ..backends.base import Backend
+ from ..cache.compression_cache import CompressionCache
+ from ..memory.tracker import MemoryTracker
+
+
+ import httpx
+
+ try:
+ import uvicorn
+ from fastapi import Depends, FastAPI, HTTPException, Request, Response
+ from fastapi.middleware.cors import CORSMiddleware
+ from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
+
+ FASTAPI_AVAILABLE = True
+ except ImportError:
+ FASTAPI_AVAILABLE = False
+
+ # Add parent to path for imports
+ sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+ from headroom._version import __version__
+ from headroom.cache.compression_feedback import get_compression_feedback
+ from headroom.cache.compression_store import get_compression_store
+ from headroom.ccr import (
+ CCR_TOOL_NAME,
+ # Batch processing
+ CCRResponseHandler,
+ CCRToolInjector,
+ ContextTracker,
+ ContextTrackerConfig,
+ ResponseHandlerConfig,
+ parse_tool_call,
+ )
+ from headroom.config import (
+ CacheAlignerConfig,
+ CCRConfig,
+ IntelligentContextConfig,
+ ReadLifecycleConfig,
+ RollingWindowConfig,
+ SmartCrusherConfig,
+ )
+ from headroom.dashboard import get_dashboard_html
+ from headroom.observability import (
+ LangfuseTracingConfig,
+ OTelMetricsConfig,
+ configure_langfuse_tracing,
+ configure_otel_metrics,
+ get_langfuse_tracing_status,
+ get_otel_metrics_status,
+ shutdown_headroom_tracing,
+ shutdown_otel_metrics,
+ )
+ from headroom.pipeline import PipelineExtensionManager, PipelineStage
+ from headroom.providers.proxy_routes import register_provider_routes
+ from headroom.providers.registry import (
+ DEFAULT_ANTHROPIC_API_URL,
+ DEFAULT_CLOUDCODE_API_URL,
+ DEFAULT_GEMINI_API_URL,
+ DEFAULT_OPENAI_API_URL,
+ build_proxy_provider_runtime,
+ create_proxy_backend,
+ format_backend_status,
+ )
+
+ # =============================================================================
+ # Extracted modules (re-exported for backward compatibility)
+ # =============================================================================
+ from headroom.proxy.cost import (
+ _CACHE_ECONOMICS, # noqa: F401
+ CostTracker, # noqa: F401
+ _summarize_transforms, # noqa: F401
+ build_prefix_cache_stats, # noqa: F401
+ build_session_summary, # noqa: F401
+ merge_cost_stats, # noqa: F401
+ )
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS, # noqa: F401
+ MAX_COMPRESSION_CACHE_SESSIONS, # noqa: F401
+ MAX_MESSAGE_ARRAY_LENGTH, # noqa: F401
+ MAX_REQUEST_BODY_SIZE, # noqa: F401
+ MAX_SSE_BUFFER_SIZE, # noqa: F401
+ _get_image_compressor, # noqa: F401
+ _get_rtk_stats, # noqa: F401
+ _read_request_json, # noqa: F401
+ _setup_file_logging, # noqa: F401
+ is_anthropic_auth, # noqa: F401
+ jitter_delay_ms,
+ )
+ from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler
+
+ # Data models (extracted to headroom/proxy/models.py for maintainability)
+ from headroom.proxy.models import CacheEntry, ProxyConfig, RateLimitState, RequestLog # noqa: F401
+ from headroom.proxy.modes import (
+ PROXY_MODE_CACHE,
+ PROXY_MODE_TOKEN,
+ is_token_mode,
+ normalize_proxy_mode,
+ )
+ from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
+ from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
+ from headroom.proxy.request_logger import RequestLogger # noqa: F401
+ from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
+ from headroom.proxy.warmup import WarmupRegistry
+ from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
+ from headroom.subscription.base import get_quota_registry, reset_quota_registry
+ from headroom.subscription.codex_rate_limits import get_codex_rate_limit_state
+ from headroom.subscription.copilot_quota import get_copilot_quota_tracker
+ from headroom.subscription.tracker import (
+ configure_subscription_tracker,
+ get_subscription_tracker,
+ )
+ from headroom.telemetry import get_telemetry_collector
+ from headroom.telemetry.beacon import is_telemetry_enabled
+ from headroom.telemetry.toin import get_toin
+ from headroom.transforms import (
+ CacheAligner,
+ CodeAwareCompressor,
+ CodeCompressorConfig,
+ ContentRouter,
+ ContentRouterConfig,
+ IntelligentContextManager,
+ RollingWindow,
+ SmartCrusher,
+ Transform,
+ TransformPipeline,
+ is_tree_sitter_available,
+ )
+
+ AnyLLMBackend: Any = None
+ LiteLLMBackend: Any = None
+
+ fcntl: Any = None
+ try:
+ import fcntl as _fcntl
+
+ fcntl = _fcntl
+ HAS_FCNTL = True
+ except ImportError:
+ HAS_FCNTL = False
+
+ _build_prefix_cache_stats = build_prefix_cache_stats
+ _build_session_summary = build_session_summary
+ _merge_cost_stats = merge_cost_stats
+
+
+ logging.basicConfig(
+ level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+ )
+ logger = logging.getLogger("headroom.proxy")
+
+ # Always-on file logging to ~/.headroom/logs/ for `headroom perf` analysis
+ _setup_file_logging()
+
+
+ # Compression pipeline timeout in seconds
+
+
+ from headroom.proxy.handlers import ( # noqa: E402
+ AnthropicHandlerMixin,
+ BatchHandlerMixin,
+ GeminiHandlerMixin,
+ OpenAIHandlerMixin,
+ StreamingMixin,
+ )
+
+
+ class HeadroomProxy(
+ StreamingMixin,
+ AnthropicHandlerMixin,
+ OpenAIHandlerMixin,
+ GeminiHandlerMixin,
+ BatchHandlerMixin,
+ ):
+ """Production-ready Headroom optimization proxy."""
+
+ ANTHROPIC_API_URL = DEFAULT_ANTHROPIC_API_URL
+ OPENAI_API_URL = DEFAULT_OPENAI_API_URL
+ GEMINI_API_URL = DEFAULT_GEMINI_API_URL
+ CLOUDCODE_API_URL = DEFAULT_CLOUDCODE_API_URL
+
+ def __init__(self, config: ProxyConfig):
+ self.config = config
+ self.config.mode = normalize_proxy_mode(self.config.mode)
+ self.pipeline_extensions = PipelineExtensionManager(
+ hooks=config.hooks,
+ extensions=config.pipeline_extensions,
+ discover=config.discover_pipeline_extensions,
+ )
+
+ self.provider_runtime = build_proxy_provider_runtime(config)
+ api_targets = self.provider_runtime.api_targets
+
+ # Preserve the long-standing proxy compatibility surface while keeping
+ # provider_runtime as the source of truth for resolved upstream targets.
+ HeadroomProxy.ANTHROPIC_API_URL = api_targets.anthropic
+ HeadroomProxy.OPENAI_API_URL = api_targets.openai
+ HeadroomProxy.GEMINI_API_URL = api_targets.gemini
+ HeadroomProxy.CLOUDCODE_API_URL = api_targets.cloudcode
+ self.anthropic_provider = self.provider_runtime.pipeline_provider("anthropic")
+ self.openai_provider = self.provider_runtime.pipeline_provider("openai")
+
+ # Initialize transforms based on routing mode
+ # Choose context manager: IntelligentContextManager (smart) or RollingWindow (legacy)
+ context_manager: Transform # Can be either IntelligentContextManager or RollingWindow
+ if config.intelligent_context:
+ # Get TOIN instance for learned pattern integration
+ toin = get_toin() if config.intelligent_context_scoring else None
+ context_manager = IntelligentContextManager(
+ config=IntelligentContextConfig(
+ enabled=True,
+ keep_system=True,
+ keep_last_turns=config.keep_last_turns,
+ use_importance_scoring=config.intelligent_context_scoring,
+ toin_integration=config.intelligent_context_scoring,
+ compress_threshold=0.10 if config.intelligent_context_compress_first else 0.0,
+ ),
+ toin=toin,
+ )
+ self._context_manager_status = "intelligent"
+ else:
+ context_manager = RollingWindow(
+ RollingWindowConfig(
+ enabled=True,
+ keep_system=True,
+ keep_last_turns=config.keep_last_turns,
+ )
+ )
+ self._context_manager_status = "rolling_window"
+
+ if config.smart_routing:
+ # Smart routing: ContentRouter handles all content types intelligently
+ # It lazy-loads compressors only when needed
+ router_config = ContentRouterConfig(
+ enable_code_aware=config.code_aware_enabled,
+ tool_profiles=config.tool_profiles,
+ read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),
+ )
+ # Token mode: allow compression of older excluded-tool results
+ if is_token_mode(config.mode):
+ router_config.protect_recent_reads_fraction = 0.3
+ transforms = [
+ CacheAligner(CacheAlignerConfig(enabled=False)),
+ ContentRouter(router_config),
+ context_manager,
+ ]
+ self._code_aware_status = "lazy" if config.code_aware_enabled else "disabled"
+ else:
+ # Legacy mode: sequential pipeline
+ transforms = [
+ CacheAligner(CacheAlignerConfig(enabled=False)),
+ SmartCrusher(
+ SmartCrusherConfig( # type: ignore[arg-type]
+ enabled=True,
+ min_tokens_to_crush=config.min_tokens_to_crush,
+ max_items_after_crush=config.max_items_after_crush,
+ ),
+ ccr_config=CCRConfig(
+ enabled=config.ccr_inject_tool,
+ inject_retrieval_marker=config.ccr_inject_tool, # Add CCR markers
+ ),
+ ),
+ context_manager,
+ ]
+ # Add CodeAware if enabled and available
+ self._code_aware_status = self._setup_code_aware(config, transforms)
+
+ self.anthropic_pipeline = TransformPipeline(
+ transforms=transforms,
+ provider=self.anthropic_provider,
+ )
+ self.openai_pipeline = TransformPipeline(
+ transforms=transforms,
+ provider=self.openai_provider,
+ )
+
+ # Initialize components
+ self.cache = (
+ SemanticCache(
+ max_entries=config.cache_max_entries,
+ ttl_seconds=config.cache_ttl_seconds,
+ )
+ if config.cache_enabled
+ else None
+ )
+
+ self.rate_limiter = (
+ TokenBucketRateLimiter(
+ requests_per_minute=config.rate_limit_requests_per_minute,
+ tokens_per_minute=config.rate_limit_tokens_per_minute,
+ )
+ if config.rate_limit_enabled
+ else None
+ )
+
+ self.cost_tracker = (
+ CostTracker(
+ budget_limit_usd=config.budget_limit_usd,
+ budget_period=config.budget_period,
+ )
+ if config.cost_tracking_enabled
+ else None
+ )
+
+ self.metrics = PrometheusMetrics(cost_tracker=self.cost_tracker)
+
+ # Prefix cache tracking: freeze already-cached messages to avoid
+ # invalidating the provider's prefix cache with our transforms
+ from headroom.cache.prefix_tracker import PrefixFreezeConfig, SessionTrackerStore
+
+ self.session_tracker_store = SessionTrackerStore(
+ default_config=PrefixFreezeConfig(
+ enabled=config.prefix_freeze_enabled,
+ session_ttl_seconds=config.prefix_freeze_session_ttl,
+ )
+ )
+
+ # Compression cache store for token mode (session-scoped)
+ self._compression_caches: dict[str, CompressionCache] = {}
+
+ self.logger = (
+ RequestLogger(
+ log_file=config.log_file,
+ log_full_messages=config.log_full_messages,
+ )
+ if config.log_requests
+ else None
+ )
+
+ # Enterprise security plugin (loaded dynamically if available + licensed)
+ self.security = None
+
+ # HTTP client
+ self.http_client: httpx.AsyncClient | None = None
+
+ # Shared cold-start warmup registry (populated by startup()).
+ # Holds typed slots with loaded / loading / null / error status for
+ # each preloaded heavy asset. Exposed as ``proxy.warmup`` and
+ # serialized by the /debug/warmup route (Unit 5).
+ self.warmup: WarmupRegistry = WarmupRegistry()
+ # Unit 3: live registry of Codex WS sessions. Populated by
+ # ``handle_openai_responses_ws`` on accept; drained in its
+ # outermost ``finally``. Consumed by ``/debug/ws-sessions``.
+ self.ws_sessions: WebSocketSessionRegistry = WebSocketSessionRegistry()
+
+ # Unit 4: bounded pre-upstream concurrency for the Anthropic HTTP
+ # path. Caps how many ``handle_anthropic_messages`` calls may be
+ # running deep-copy / first-stage compression / memory-context
+ # lookup / upstream connect concurrently. ``/livez``, ``/readyz``,
+ # ``/health``, ``/metrics``, ``/stats``, and the Codex WS path are
+ # intentionally NOT gated by this semaphore.
+ #
+ # A value of ``0`` or negative disables the semaphore (unbounded
+ # mode); this is useful for the Unit 6 counter-factual where we
+ # deliberately reproduce the original starvation. The default is
+ # ``max(2, min(8, os.cpu_count() or 4))``.
+ _pre_upstream_cfg = config.anthropic_pre_upstream_concurrency
+ if _pre_upstream_cfg is None:
+ _pre_upstream_resolved = max(2, min(8, os.cpu_count() or 4))
+ else:
+ _pre_upstream_resolved = _pre_upstream_cfg
+ self.anthropic_pre_upstream_concurrency: int = _pre_upstream_resolved
+ self.anthropic_pre_upstream_acquire_timeout_seconds = float(
+ config.anthropic_pre_upstream_acquire_timeout_seconds
+ )
+ self.anthropic_pre_upstream_memory_context_timeout_seconds = float(
+ config.anthropic_pre_upstream_memory_context_timeout_seconds
+ )
+ if _pre_upstream_resolved > 0:
+ self.anthropic_pre_upstream_sem: asyncio.Semaphore | None = asyncio.Semaphore(
+ _pre_upstream_resolved
+ )
+ else:
+ self.anthropic_pre_upstream_sem = None
+
+ # Backend for Anthropic API (direct, LiteLLM, or any-llm)
+ # Supports: "anthropic" (direct), "bedrock", "vertex", "litellm-<provider>", or "anyllm"
+ self.anthropic_backend: Backend | None = create_proxy_backend(
+ backend=config.backend,
+ anyllm_provider=config.anyllm_provider,
+ bedrock_region=config.bedrock_region,
+ logger=logger,
+ anyllm_backend_cls=AnyLLMBackend,
+ litellm_backend_cls=LiteLLMBackend,
+ )
+
+ # Request counter for IDs
+ self._request_counter = 0
+ self._request_counter_lock = asyncio.Lock()
+
+ # CCR tool injectors (one per provider)
+ self.anthropic_tool_injector = CCRToolInjector(
+ provider="anthropic",
+ inject_tool=config.ccr_inject_tool,
+ inject_system_instructions=config.ccr_inject_system_instructions,
+ )
+ self.openai_tool_injector = CCRToolInjector(
+ provider="openai",
+ inject_tool=config.ccr_inject_tool,
+ inject_system_instructions=config.ccr_inject_system_instructions,
+ )
+
+ # CCR Response Handler (handles CCR tool calls automatically)
+ self.ccr_response_handler = (
+ CCRResponseHandler(
+ ResponseHandlerConfig(
+ enabled=True,
+ max_retrieval_rounds=config.ccr_max_retrieval_rounds,
+ )
+ )
+ if config.ccr_handle_responses
+ else None
+ )
+
+ # CCR Context Tracker (tracks compressed content across turns)
+ self.ccr_context_tracker = (
+ ContextTracker(
+ ContextTrackerConfig(
+ enabled=True,
+ proactive_expansion=config.ccr_proactive_expansion,
+ max_proactive_expansions=config.ccr_max_proactive_expansions,
+ )
+ )
+ if config.ccr_context_tracking
+ else None
+ )
+
+ # Turn counter for context tracking
+ self._turn_counter = 0
+
+ # Memory Handler (persistent user memory)
+ self.memory_handler: MemoryHandler | None = None
+ if config.memory_enabled:
+ # Resolve memory DB path: empty → project-scoped default
+ _mem_db_path = config.memory_db_path
+ if not _mem_db_path:
+ _mem_dir = Path.cwd() / ".headroom"
+ _mem_dir.mkdir(parents=True, exist_ok=True)
+ _mem_db_path = str(_mem_dir / "memory.db")
+ logger.info(f"Memory: Project-scoped DB at {_mem_db_path}")
+
+ memory_config = MemoryConfig(
+ enabled=True,
+ backend=config.memory_backend,
+ db_path=_mem_db_path,
+ inject_tools=config.memory_inject_tools,
+ use_native_tool=config.memory_use_native_tool,
+ inject_context=config.memory_inject_context,
+ top_k=config.memory_top_k,
+ min_similarity=config.memory_min_similarity,
+ qdrant_host=config.memory_qdrant_host,
+ qdrant_port=config.memory_qdrant_port,
+ neo4j_uri=config.memory_neo4j_uri,
+ neo4j_user=config.memory_neo4j_user,
+ neo4j_password=config.memory_neo4j_password,
+ bridge_enabled=config.memory_bridge_enabled,
+ bridge_md_paths=config.memory_bridge_md_paths,
+ bridge_md_format=config.memory_bridge_md_format,
+ bridge_auto_import=config.memory_bridge_auto_import,
+ bridge_export_path=config.memory_bridge_export_path,
+ )
+ self.memory_handler = MemoryHandler(
+ memory_config,
+ agent_type=config.traffic_learning_agent_type,
+ )
+
+ # Usage Reporter (license validation + phone-home for managed/enterprise)
+ self.usage_reporter: UsageReporter | None = None
+ if config.license_key:
+ from headroom.telemetry.reporter import UsageReporter
+
+ self.usage_reporter = UsageReporter(
+ license_key=config.license_key,
+ cloud_url=config.license_cloud_url,
+ report_interval=config.license_report_interval,
+ )
+
+ # Traffic Learner (live pattern extraction from proxy traffic)
+ # Only activates with --learn flag; requires --memory for backend
+ self.traffic_learner: TrafficLearner | None = None
+ self.traffic_learning_agent_type: str = config.traffic_learning_agent_type
+ if config.traffic_learning_enabled:
+ from headroom.memory.traffic_learner import TrafficLearner
+
+ self.traffic_learner = TrafficLearner(
+ user_id=os.environ.get("HEADROOM_USER_ID", os.environ.get("USER", "default")),
+ agent_type=config.traffic_learning_agent_type,
+ )
+
+ # Code graph file watcher (live reindex on file changes)
+ self.code_graph_watcher: CodeGraphWatcher | None = None # type: ignore[annotation-unchecked]
+ if config.code_graph_watcher:
+ from headroom.graph.watcher import CodeGraphWatcher
+
+ self.code_graph_watcher = CodeGraphWatcher(project_dir=Path.cwd())
+ if self.code_graph_watcher.start():
+ logger.info("Code graph: file watcher started")
+ else:
+ self.code_graph_watcher = None
+
+ self.pipeline_extensions.emit(
+ PipelineStage.SETUP,
+ operation="proxy.setup",
+ metadata={
+ "mode": self.config.mode,
+ "optimize": self.config.optimize,
+ "backend": self.config.backend,
+ "memory_enabled": self.config.memory_enabled,
+ },
+ )
+
+ def _get_compression_cache(self, session_id: str) -> CompressionCache:
+ """Get or create a CompressionCache for a session."""
+ if session_id not in self._compression_caches:
+ from headroom.cache.compression_cache import CompressionCache
+
+ # Evict oldest caches if at capacity
+ if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:
+ # Remove oldest quarter to amortize cleanup cost
+ oldest_keys = list(self._compression_caches.keys())[
+ : MAX_COMPRESSION_CACHE_SESSIONS // 4
+ ]
+ for key in oldest_keys:
+ del self._compression_caches[key]
+ logger.info(
+ "Evicted %d compression caches (exceeded %d max sessions)",
+ len(oldest_keys),
+ MAX_COMPRESSION_CACHE_SESSIONS,
+ )
+
+ self._compression_caches[session_id] = CompressionCache()
+ return self._compression_caches[session_id]
+
+ def _setup_code_aware(self, config: ProxyConfig, transforms: list) -> str:
+ """Set up code-aware compression if enabled.
+
+ Args:
+ config: Proxy configuration
+ transforms: Transform list to append to
+
+ Returns:
+ Status string for logging: 'enabled', 'disabled', 'available', 'unavailable'
+ """
+ if config.code_aware_enabled:
+ if is_tree_sitter_available():
+ code_config = CodeCompressorConfig(
+ preserve_imports=True,
+ preserve_signatures=True,
+ preserve_type_annotations=True,
+ )
+ # Insert before RollingWindow (which should be last)
+ transforms.insert(-1, CodeAwareCompressor(code_config))
+ return "enabled"
+ else:
+ logger.warning(
+ "Code-aware compression requested but tree-sitter not installed. "
+ "Install with: pip install headroom-ai[code]"
+ )
+ return "unavailable"
+ else:
+ if is_tree_sitter_available():
+ return "available" # Available but not enabled
+ return "disabled"
+
+ async def startup(self):
+ """Initialize async resources."""
+ self.pipeline_extensions.emit(
+ PipelineStage.PRE_START,
+ operation="proxy.startup",
+ metadata={"port": self.config.port, "host": self.config.host},
+ )
+ self.http_client = httpx.AsyncClient(
+ timeout=httpx.Timeout(
+ connect=self.config.connect_timeout_seconds,
+ read=self.config.request_timeout_seconds,
+ write=self.config.request_timeout_seconds,
+ pool=self.config.connect_timeout_seconds,
+ ),
+ limits=httpx.Limits(
+ max_connections=self.config.max_connections,
+ max_keepalive_connections=self.config.max_keepalive_connections,
+ ),
+ http2=self.config.http2,
+ )
+ logger.info("Headroom Proxy started")
+ logger.info(f"Optimization: {'ENABLED' if self.config.optimize else 'DISABLED'}")
+ self.config.mode = normalize_proxy_mode(self.config.mode)
+ logger.info(f"Mode: {self.config.mode}")
+ if self.config.mode == PROXY_MODE_TOKEN:
+ logger.info(" Prefix freeze: re-freeze after compression")
+ logger.info(" Read protection window: 30%% of excluded-tool messages")
+ logger.info(" CCR TTL: extended for session lifetime")
+ logger.info(" Compression cache: active")
+ if self.config.mode == PROXY_MODE_CACHE:
+ logger.info(" Prefix freeze: strict (all prior turns immutable)")
+ logger.info(" Mutations: latest turn only")
+ logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}")
+ logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}")
+ logger.info(
+ f"Connection Pool: max_connections={self.config.max_connections}, "
+ f"max_keepalive={self.config.max_keepalive_connections}, "
+ f"http2={'ENABLED' if self.config.http2 else 'DISABLED'}"
+ )
+
+ # Unit 4 pre-upstream concurrency announcement. Report the resolved
+ # value (auto-detected vs. explicit) so operators can correlate
+ # ``pre_upstream_wait_ms`` log lines with the configured cap.
+ if self.anthropic_pre_upstream_sem is None:
+ logger.info("Anthropic pre-upstream concurrency: unbounded (explicitly disabled)")
+ else:
+ _explicit = self.config.anthropic_pre_upstream_concurrency
+ _origin = "auto-detected" if _explicit is None else "explicit"
+ logger.info(
+ "Anthropic pre-upstream concurrency: %d (%s)",
+ self.anthropic_pre_upstream_concurrency,
+ _origin,
+ )
+ logger.info(
+ "Anthropic pre-upstream timeouts: acquire=%.1fs compression=%.1fs memory_context=%.1fs",
+ self.anthropic_pre_upstream_acquire_timeout_seconds,
+ float(COMPRESSION_TIMEOUT_SECONDS),
+ self.anthropic_pre_upstream_memory_context_timeout_seconds,
+ )
+
+ # Smart routing status
+ if self.config.smart_routing:
+ logger.info("Smart Routing: ENABLED (intelligent content detection)")
+ else:
+ logger.info("Smart Routing: DISABLED (legacy sequential mode)")
+
+ # Eagerly load ALL compressors, parsers, and detectors at startup
+ # This eliminates cold-start latency spikes on first requests.
+ # Iterate BOTH pipelines (Anthropic + OpenAI) and dedupe transforms
+ # by id() so shared-transform instances never load twice. The
+ # resulting status dict is merged into ``self.warmup`` so /debug/warmup
+ # (Unit 5) and /readyz have a single source of truth.
+ self._kompress_status = "not installed"
+ eager_status: dict[str, str] = {}
+
+ if self.config.optimize:
+ logger.info("Pre-loading compressors and parsers...")
+ seen_transform_ids: set[int] = set()
+ pipelines = (self.anthropic_pipeline, self.openai_pipeline)
+ for pipeline in pipelines:
+ for transform in pipeline.transforms:
+ if id(transform) in seen_transform_ids:
+ continue
+ seen_transform_ids.add(id(transform))
+ if not hasattr(transform, "eager_load_compressors"):
+ continue
+ try:
+ transform_status = transform.eager_load_compressors()
+ except Exception as exc:
+ logger.warning(
+ "Eager preload failed for %s: %s",
+ type(transform).__name__,
+ exc,
+ )
+ continue
+ if not isinstance(transform_status, dict):
+ continue
+ # Merge: later writers win only if the key wasn't set.
+ # Preload a transform ONCE — if another pipeline also has
+ # ``eager_load_compressors`` it contributes only new keys.
+ for key, value in transform_status.items():
+ eager_status.setdefault(key, value)
+ self.warmup.merge_transform_status(transform_status)
+
+ # Update internal status from eager loading results
+ if eager_status.get("kompress") == "enabled":
+ self._kompress_status = "enabled"
+ if eager_status.get("code_aware") == "enabled":
+ self._code_aware_status = "enabled"
+
+ # Log component status
+ if self._kompress_status == "enabled":
+ logger.info("Kompress: ENABLED (ModernBERT token compressor)")
+ elif self.config.optimize:
+ logger.info("Kompress: not installed (pip install headroom-ai[ml] for ML compression)")
+
+ if self._code_aware_status == "enabled":
+ logger.info("Code-Aware: ENABLED (AST-based compression)")
+ if "tree_sitter" in eager_status:
+ logger.info(f"Tree-Sitter: {eager_status['tree_sitter']}")
+ elif self._code_aware_status == "lazy":
+ logger.info("Code-Aware: LAZY (will load when code content detected)")
+ elif self._code_aware_status == "available":
+ logger.info("Code-Aware: available but disabled (use --code-aware)")
+ elif self._code_aware_status == "unavailable":
+ logger.info("Code-Aware: not installed (pip install headroom-ai[code])")
+ elif self._code_aware_status == "disabled":
+ logger.info("Code-Aware: DISABLED")
+
+ if eager_status.get("magika") == "enabled":
+ logger.info("Magika: ENABLED (ML content detection)")
+
+ if self.memory_handler:
+ self.warmup.memory_backend.mark_loading()
+ try:
+ await self.memory_handler.ensure_initialized()
+ except Exception as exc: # pragma: no cover - defensive
+ self.warmup.memory_backend.mark_error(str(exc))
+ logger.warning("Memory: backend initialization failed (startup continues): %s", exc)
+ memory_status = self.memory_handler.health_status()
+ if memory_status.get("initialized"):
+ self.warmup.memory_backend.mark_loaded(
+ handle=self.memory_handler,
+ backend=memory_status.get("backend"),
+ )
+ # Force one embed call so the ONNX graph is compiled now,
+ # not lazily during the first request. Best-effort — any
+ # failure is swallowed inside warmup_embedder.
+ self.warmup.memory_embedder.mark_loading()
+ warmed = await self.memory_handler.warmup_embedder()
+ if warmed:
+ self.warmup.memory_embedder.mark_loaded()
+ else:
+ # Not an error — e.g. qdrant-neo4j has no embedder slot
+ # we can reach, or the backend simply exposes no handle.
+ self.warmup.memory_embedder.mark_null()
+ else:
+ if self.warmup.memory_backend.status != "error":
+ self.warmup.memory_backend.mark_null()
+ self.warmup.memory_embedder.mark_null()
+ logger.info(
+ "Memory: ENABLED "
+ f"(backend={memory_status['backend']}, initialized={memory_status['initialized']})"
+ )
+ else:
+ logger.info("Memory: DISABLED")
+
+ # CCR status
+ ccr_features = []
+ if self.config.ccr_inject_tool:
+ ccr_features.append("tool_injection")
+ if self.config.ccr_handle_responses:
+ ccr_features.append("response_handling")
+ if self.config.ccr_context_tracking:
+ ccr_features.append("context_tracking")
+ if self.config.ccr_proactive_expansion:
+ ccr_features.append("proactive_expansion")
+ if ccr_features:
+ logger.info(f"CCR (Compress-Cache-Retrieve): ENABLED ({', '.join(ccr_features)})")
+ else:
+ logger.info("CCR: DISABLED")
+ logger.info(f"Savings history: {self.metrics.savings_tracker.storage_path}")
+
+ # Reset and rebuild the quota tracker registry for this server instance.
+ # reset_quota_registry() ensures a clean slate when the proxy is restarted
+ # (e.g. in tests that spin up multiple app instances in the same process).
+ reset_quota_registry()
+ registry = get_quota_registry()
+ tracker = configure_subscription_tracker(
+ poll_interval_s=self.config.subscription_poll_interval_s,
+ active_window_s=self.config.subscription_active_window_s,
+ enabled=self.config.subscription_tracking_enabled,
+ )
+ registry.register(tracker)
+ registry.register(get_codex_rate_limit_state())
+ registry.register(get_copilot_quota_tracker())
+ await registry.start_all()
+
+ if self.config.subscription_tracking_enabled:
+ logger.info(
+ "Subscription tracking: ENABLED "
+ f"(poll_interval={self.config.subscription_poll_interval_s}s, "
+ f"active_window={self.config.subscription_active_window_s}s)"
+ )
+ else:
+ logger.info("Subscription tracking: DISABLED")
+
+ copilot_tracker = get_copilot_quota_tracker()
+ if copilot_tracker.is_available():
+ logger.info("GitHub Copilot quota tracking: ENABLED")
+ else:
+ logger.info(
+ "GitHub Copilot quota tracking: DISABLED "
+ "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)"
+ )
+
+ # Log anonymous telemetry status so operators can see it in the log stream
+ if is_telemetry_enabled():
+ logger.info(
+ "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). "
+ "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry"
+ )
+ else:
+ logger.info("Anonymous telemetry: DISABLED")
+
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_START,
+ operation="proxy.startup",
+ metadata={
+ "port": self.config.port,
+ "host": self.config.host,
+ "warmup": self.warmup.to_dict(),
+ },
+ )
+
+ async def shutdown(self):
+ """Cleanup async resources."""
+ if self.http_client:
+ await self.http_client.aclose()
+ self.http_client = None
+
+ if self.memory_handler and hasattr(self.memory_handler, "close"):
+ await self.memory_handler.close()
+
+ # Stop all quota trackers via the registry
+ await get_quota_registry().stop_all()
+
+ # Print final stats
+ self._print_summary()
+
+ def _print_summary(self):
+ """Print session summary."""
+ m = self.metrics
+ logger.info("=" * 70)
+ logger.info("HEADROOM PROXY SESSION SUMMARY")
+ logger.info("=" * 70)
+ logger.info(f"Total requests: {m.requests_total}")
+ logger.info(f"Cached responses: {m.requests_cached}")
+ logger.info(f"Rate limited: {m.requests_rate_limited}")
+ logger.info(f"Failed: {m.requests_failed}")
+ logger.info(f"Input tokens: {m.tokens_input_total:,}")
+ logger.info(f"Output tokens: {m.tokens_output_total:,}")
+ logger.info(f"Tokens saved: {m.tokens_saved_total:,}")
+ if m.tokens_input_total > 0:
+ savings_pct = (
+ m.tokens_saved_total / (m.tokens_input_total + m.tokens_saved_total)
+ ) * 100
+ logger.info(f"Token savings: {savings_pct:.1f}%")
+ if m.latency_count > 0:
+ avg_latency = m.latency_sum_ms / m.latency_count
+ logger.info(f"Avg latency: {avg_latency:.0f}ms")
+ logger.info("=" * 70)
+
+ async def _next_request_id(self) -> str:
+ """Generate unique request ID."""
+ async with self._request_counter_lock:
+ self._request_counter += 1
+ return f"hr_{int(time.time())}_{self._request_counter:06d}"
+
+ def _extract_tags(self, headers: dict) -> dict[str, str]:
+ """Extract Headroom tags from headers."""
+ tags = {}
+ for key, value in headers.items():
+ if key.lower().startswith("x-headroom-"):
+ tag_name = key.lower().replace("x-headroom-", "")
+ tags[tag_name] = value
+ return tags
+
+ def _inject_system_context(
+ self,
+ messages: list[dict[str, Any]],
+ context: str,
+ body: dict[str, Any] | None = None,
+ ) -> list[dict[str, Any]]:
+ """Inject context into the system message/parameter.
+
+ For Anthropic API: Uses top-level 'system' parameter (not messages array).
+ For OpenAI API: Uses system role in messages array.
+
+ Args:
+ messages: The messages list.
+ context: Context to inject.
+ body: Optional request body to update system parameter (for Anthropic).
+
+ Returns:
+ Updated messages list.
+ """
+ messages = list(messages) # Copy to avoid mutation
+
+ # For Anthropic API: use top-level 'system' parameter
+ if body is not None:
+ existing_system = body.get("system", "")
+ if isinstance(existing_system, str):
+ body["system"] = (existing_system + "\n\n" + context).strip()
+ elif isinstance(existing_system, list):
+ # system is a list of content blocks (e.g., with cache_control).
+ # Append memory context as a new text block — never overwrite.
+ body["system"] = existing_system + [{"type": "text", "text": context}]
+ else:
+ # No existing system prompt — set as string
+ body["system"] = context
+ return messages
+
+ # For OpenAI API: use system role in messages
+ for i, msg in enumerate(messages):
+ if msg.get("role") == "system":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ messages[i] = {**msg, "content": content + "\n\n" + context}
+ return messages
+
+ # No system message found - prepend one
+ messages.insert(0, {"role": "system", "content": context})
+ return messages
+
+ async def _retry_request(
+ self,
+ method: str,
+ url: str,
+ headers: dict,
+ body: dict,
+ stream: bool = False,
+ ) -> httpx.Response:
+ """Make request with retry and exponential backoff."""
+ last_error = None
+
+ for attempt in range(self.config.retry_max_attempts):
+ try:
+ if stream:
+ # For streaming, we return early - retry happens at higher level
+ return await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
+ else:
+ response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
+
+ # Don't retry client errors (4xx)
+ if 400 <= response.status_code < 500:
+ return response
+
+ # Retry server errors (5xx)
+ if response.status_code >= 500:
+ raise httpx.HTTPStatusError(
+ f"Server error: {response.status_code}",
+ request=response.request,
+ response=response,
+ )
+
+ return response
+
+ except (httpx.ConnectError, httpx.ReadTimeout, httpx.HTTPStatusError) as e:
+ last_error = e
+
+ if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:
+ raise
+
+ # Exponential backoff with jitter
+ delay_with_jitter = jitter_delay_ms(
+ self.config.retry_base_delay_ms,
+ self.config.retry_max_delay_ms,
+ attempt,
+ )
+
+ logger.warning(
+ f"Request failed (attempt {attempt + 1}), retrying in {delay_with_jitter:.0f}ms: {e}"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ raise last_error # type: ignore[misc]
+
+
+ async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:
+ """Background task that logs TOIN stats periodically.
+
+ Args:
+ interval_seconds: How often to log stats (default: 5 minutes).
+ """
+ while True:
+ await asyncio.sleep(interval_seconds)
+ try:
+ toin = get_toin()
+ stats = toin.get_stats()
+ total_compressions = stats.get("total_compressions", 0)
+ if total_compressions > 0:
+ patterns = stats.get("patterns_tracked", 0)
+ retrievals = stats.get("total_retrievals", 0)
+ retrieval_rate = stats.get("global_retrieval_rate", 0.0)
+ logger.info(
+ "TOIN: %d patterns, %d compressions, %d retrievals, %.1f%% retrieval rate",
+ patterns,
+ total_compressions,
+ retrievals,
+ retrieval_rate * 100,
+ )
+ except Exception as e:
+ logger.debug("Failed to log TOIN stats: %s", e)
+
+
+ def _register_memory_components(proxy: HeadroomProxy, tracker: MemoryTracker) -> None:
+ """Register all memory-tracked components with the tracker.
+
+ This function is idempotent - it checks if components are already registered.
+
+ Args:
+ proxy: The HeadroomProxy instance.
+ tracker: The MemoryTracker instance.
+ """
+ # Register compression store (global singleton)
+ if "compression_store" not in tracker.registered_components:
+ store = get_compression_store()
+ tracker.register("compression_store", store.get_memory_stats)
+
+ # Register semantic cache (instance on proxy)
+ if proxy.cache and "semantic_cache" not in tracker.registered_components:
+ tracker.register("semantic_cache", proxy.cache.get_memory_stats)
+
+ # Register request logger (instance on proxy)
+ if proxy.logger and "request_logger" not in tracker.registered_components:
+ tracker.register("request_logger", proxy.logger.get_memory_stats)
+
+ # Register batch context store (global singleton)
+ if "batch_context_store" not in tracker.registered_components:
+ try:
+ from ..ccr.batch_store import get_batch_context_store
+
+ batch_store = get_batch_context_store()
+ if hasattr(batch_store, "get_memory_stats"):
+ tracker.register("batch_context_store", batch_store.get_memory_stats)
+ except ImportError:
+ pass
+
+ # Note: graph_store and vector_index are created per-user within the
+ # LocalMemoryBackend, not as global singletons. They would need to be
+ # registered when the memory system is initialized with specific backends.
+
+
+ def create_app(config: ProxyConfig | None = None) -> FastAPI:
+ """Create FastAPI application."""
+ if not FASTAPI_AVAILABLE:
+ raise ImportError("FastAPI required. Install: pip install fastapi uvicorn httpx")
+
+ from contextlib import asynccontextmanager
+
+ config = config or ProxyConfig()
+ proxy = HeadroomProxy(config)
+
+ # Telemetry beacon (anonymous aggregate stats).
+ # With uvicorn workers > 1, each worker runs the lifespan independently.
+ # We must ensure only ONE beacon runs across all workers — otherwise each
+ # worker creates its own beacon, spamming the telemetry table with N rows
+ # per cycle instead of 1 (all reading the same /stats from the same port).
+ #
+ # Strategy: use a file lock to ensure only the first worker starts the
+ # beacon. Other workers see the lock and skip.
+ from headroom.telemetry.beacon import TelemetryBeacon
+
+ _beacon = TelemetryBeacon(
+ port=config.port if hasattr(config, "port") else 8787,
+ sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy",
+ backend=config.backend if hasattr(config, "backend") else "anthropic",
+ )
+ from headroom import paths as _hr_paths
+
+ _beacon_lock_path = _hr_paths.beacon_lock_path(config.port)
+ _beacon_lock_fd: list = [None] # mutable holder for the lock file descriptor
+ _beacon_is_owner: list = [False]
+
+ def _try_acquire_beacon_lock() -> bool:
+ """Try to acquire the beacon file lock (non-blocking).
+
+ Returns True if this process is the beacon owner.
+ """
+ if not HAS_FCNTL:
+ return True
+
+ fd = None
+ try:
+ _beacon_lock_path.parent.mkdir(parents=True, exist_ok=True)
+ fd = open(_beacon_lock_path, "w") # noqa: SIM115
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ fd.write(str(os.getpid()))
+ fd.flush()
+ _beacon_lock_fd[0] = fd
+ return True
+ except OSError:
+ if fd is not None:
+ fd.close()
+ return False
+
+ def _release_beacon_lock() -> None:
+ """Release the beacon file lock."""
+ fd = _beacon_lock_fd[0]
+ if fd:
+ try:
+ if HAS_FCNTL:
+ fcntl.flock(fd, fcntl.LOCK_UN)
+ fd.close()
+ except Exception:
+ pass
+ _beacon_lock_fd[0] = None
+ try:
+ _beacon_lock_path.unlink(missing_ok=True)
+ except Exception:
+ pass
+
+ @asynccontextmanager
+ async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
+ configure_otel_metrics(OTelMetricsConfig.from_env(default_service_name="headroom-proxy"))
+ configure_langfuse_tracing(
+ LangfuseTracingConfig.from_env(default_service_name="headroom-proxy")
+ )
+
+ app.state.started_at = time.time()
+ app.state.ready = False
+ app.state.startup_error = None
+
+ try:
+ try:
+ # Startup
+ await proxy.startup()
+ asyncio.create_task(_log_toin_stats_periodically())
+ if proxy.usage_reporter:
+ await proxy.usage_reporter.start(proxy)
+ if proxy.traffic_learner:
+ await proxy.traffic_learner.start()
+
+ # Only start beacon if we acquire the lock (first worker wins)
+ _beacon_is_owner[0] = _try_acquire_beacon_lock()
+ if _beacon_is_owner[0]:
+ await _beacon.start()
+ else:
+ logger.debug("Beacon: skipping (another worker owns the lock)")
+
+ app.state.ready = True
+ yield
+ except Exception as exc:
+ app.state.startup_error = str(exc)
+ raise
+ finally:
+ app.state.ready = False
+ # Shutdown
+ if _beacon_is_owner[0]:
+ await _beacon.stop()
+ _release_beacon_lock()
+ if proxy.usage_reporter:
+ await proxy.usage_reporter.stop()
+ if proxy.traffic_learner:
+ await proxy.traffic_learner.stop()
+ if proxy.code_graph_watcher:
+ proxy.code_graph_watcher.stop()
+ await proxy.shutdown()
+ shutdown_headroom_tracing()
+ shutdown_otel_metrics()
+
+ app = FastAPI(
+ title="Headroom Proxy",
+ description="Production-ready LLM optimization proxy",
+ version=__version__,
+ lifespan=lifespan,
+ )
+ app.state.proxy = proxy
+ app.state.started_at = None
+ app.state.ready = False
+ app.state.startup_error = None
+
+ def _iso_utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+ def _uptime_seconds() -> float:
+ started_at = getattr(app.state, "started_at", None)
+ if not isinstance(started_at, int | float):
+ return 0.0
+ return round(max(0.0, time.time() - float(started_at)), 3)
+
+ def _component_health(
+ *,
+ enabled: bool,
+ ready: bool,
+ **details: Any,
+ ) -> dict[str, Any]:
+ status = "disabled" if not enabled else ("healthy" if ready else "unhealthy")
+ return {
+ "enabled": enabled,
+ "ready": (ready if enabled else True),
+ "status": status,
+ **details,
+ }
+
+ def _health_checks() -> dict[str, dict[str, Any]]:
+ memory_status = (
+ proxy.memory_handler.health_status()
+ if proxy.memory_handler
+ else {
+ "enabled": False,
+ "backend": None,
+ "initialized": False,
+ "native_tool": False,
+ "bridge_enabled": False,
+ }
+ )
+ memory_enabled = bool(memory_status.get("enabled", False))
+ memory_initialized = bool(memory_status.get("initialized", False))
+ return {
+ "startup": _component_health(
+ enabled=True,
+ ready=bool(getattr(app.state, "ready", False)),
+ error=getattr(app.state, "startup_error", None),
+ ),
+ "http_client": _component_health(
+ enabled=True,
+ ready=proxy.http_client is not None,
+ ),
+ "cache": _component_health(
+ enabled=config.cache_enabled,
+ ready=(proxy.cache is not None),
+ ),
+ "rate_limiter": _component_health(
+ enabled=config.rate_limit_enabled,
+ ready=(proxy.rate_limiter is not None),
+ ),
+ "memory": _component_health(
+ enabled=memory_enabled,
+ ready=memory_initialized,
+ backend=memory_status["backend"],
+ initialized=memory_initialized,
+ native_tool=bool(memory_status.get("native_tool", False)),
+ bridge_enabled=bool(memory_status.get("bridge_enabled", False)),
+ ),
+ }
+
+ def _runtime_payload() -> dict[str, Any]:
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ ws_active_sessions = ws_registry.active_count() if ws_registry is not None else 0
+ ws_active_relay_tasks = (
+ ws_registry.active_relay_task_count() if ws_registry is not None else 0
+ )
+ return {
+ "anthropic_pre_upstream": {
+ "enabled": proxy.anthropic_pre_upstream_sem is not None,
+ "resolved_concurrency": proxy.anthropic_pre_upstream_concurrency,
+ "source": (
+ "auto" if config.anthropic_pre_upstream_concurrency is None else "explicit"
+ ),
+ "acquire_timeout_seconds": proxy.anthropic_pre_upstream_acquire_timeout_seconds,
+ "compression_timeout_seconds": float(COMPRESSION_TIMEOUT_SECONDS),
+ "memory_context_timeout_seconds": (
+ proxy.anthropic_pre_upstream_memory_context_timeout_seconds
+ ),
+ "codex_ws_gated": False,
+ },
+ "websocket_sessions": {
+ "active_sessions": ws_active_sessions,
+ "active_relay_tasks": ws_active_relay_tasks,
+ },
+ }
+
+ def _health_payload(*, include_config: bool) -> dict[str, Any]:
+ checks = _health_checks()
+ ready = all(check["ready"] for check in checks.values())
+ payload: dict[str, Any] = {
+ "service": "headroom-proxy",
+ "status": "healthy" if ready else "unhealthy",
+ "ready": ready,
+ "version": __version__,
+ "timestamp": _iso_utc_now(),
+ "uptime_seconds": _uptime_seconds(),
+ "checks": checks,
+ "runtime": _runtime_payload(),
+ }
+ deployment_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE")
+ if deployment_profile:
+ payload["deployment"] = {
+ "profile": deployment_profile,
+ "preset": os.environ.get("HEADROOM_DEPLOYMENT_PRESET"),
+ "runtime": os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME"),
+ "supervisor": os.environ.get("HEADROOM_DEPLOYMENT_SUPERVISOR"),
+ "scope": os.environ.get("HEADROOM_DEPLOYMENT_SCOPE"),
+ }
+ if include_config:
+ payload["config"] = {
+ "backend": config.backend,
+ "optimize": config.optimize,
+ "cache": config.cache_enabled,
+ "rate_limit": config.rate_limit_enabled,
+ "memory": config.memory_enabled,
+ "learn": config.traffic_learning_enabled,
+ "code_graph": config.code_graph_watcher,
+ "pid": os.getpid(),
+ }
+ return payload
+
+ # CORS
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+
+ # X-Headroom-Stack: SDK adapters (TS openai/anthropic/etc.) tag their
+ # requests so telemetry can segment by integration surface. Registered
+ # before extension middleware so any extension-level auth/guards run
+ # outermost and we don't count requests they reject.
+ @app.middleware("http")
+ async def _record_headroom_stack(request, call_next):
+ if request.url.path.startswith("/v1/"):
+ stack = request.headers.get("x-headroom-stack")
+ if stack:
+ try:
+ proxy.metrics.record_stack(stack)
+ except Exception:
+ logger.debug("record_stack failed", exc_info=True)
+ return await call_next(request)
+
+ # Third-party proxy extensions (Enterprise, custom plugins). Discovered via
- # the `headroom.proxy_extension` entry-point group. An extension that raises
- # from its install() is a deliberate fail-closed signal and aborts startup.
++ # the `headroom.proxy_extension` entry-point group, but **opt-in only**:
++ # only names listed in config.proxy_extensions (CLI: --proxy-extension,
++ # env: HEADROOM_PROXY_EXTENSIONS) actually get installed. Discovery alone
++ # never runs third-party code. An extension that raises from its install()
++ # is a deliberate fail-closed signal and aborts startup.
+ from headroom.proxy.extensions import install_all as _install_extensions
+
- _install_extensions(app, config)
++ _install_extensions(app, config, enabled=getattr(config, "proxy_extensions", None))
+
+ # Health & Metrics
+ @app.get("/livez")
+ async def livez():
+ return JSONResponse(
+ status_code=200,
+ content={
+ "service": "headroom-proxy",
+ "status": "healthy",
+ "alive": True,
+ "version": __version__,
+ "timestamp": _iso_utc_now(),
+ "uptime_seconds": _uptime_seconds(),
+ },
+ )
+
+ @app.get("/readyz")
+ async def readyz():
+ payload = _health_payload(include_config=False)
+ return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)
+
+ @app.get("/health")
+ async def health():
+ payload = _health_payload(include_config=True)
+ return JSONResponse(status_code=200, content=payload)
+
+ # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —
+ # debug endpoints are invisible to external scanners.
+ from headroom.proxy.debug_introspection import (
+ collect_tasks as _collect_tasks,
+ )
+ from headroom.proxy.loopback_guard import require_loopback as _require_loopback
+
+ @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])
+ async def debug_tasks(stack: bool = False):
+ """Enumerate running asyncio tasks.
+
+ Default is cheap — ``stack_depth`` is ``null`` in every entry so
+ a storm snapshot does not walk 50+ coroutine frames synchronously.
+ Pass ``?stack=true`` to compute ``stack_depth`` for each task
+ (useful for single-shot human debugging).
+ """
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ return JSONResponse(
+ status_code=200,
+ content=_collect_tasks(ws_registry, with_stack_depth=stack),
+ )
+
+ @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])
+ async def debug_ws_sessions():
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ snapshot = ws_registry.snapshot() if ws_registry is not None else []
+ return JSONResponse(status_code=200, content=snapshot)
+
+ @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])
+ async def debug_warmup():
+ warmup_registry = getattr(proxy, "warmup", None)
+ payload = warmup_registry.to_dict() if warmup_registry is not None else {}
+ payload["runtime"] = _runtime_payload()
+ return JSONResponse(status_code=200, content=payload)
+
+ @app.get("/dashboard", response_class=HTMLResponse)
+ async def dashboard():
+ """Serve the Headroom dashboard UI."""
+ return get_dashboard_html()
+
+ DASHBOARD_STATS_CACHE_TTL_SECONDS = 5.0
+ _stats_snapshot_lock = asyncio.Lock()
+ _stats_snapshot: dict[str, Any] = {"expires_at": 0.0, "value": None}
+
+ async def _build_stats_payload() -> dict[str, Any]:
+ """Build the full `/stats` response payload.
+
+ This is the main stats endpoint - it aggregates data from all subsystems:
+ - Request metrics (total, cached, failed, by model/provider)
+ - Token usage and savings
+ - Cost tracking
+ - Canonical persisted display_session metrics for downstream dashboards
+ - Compression (CCR) statistics
+ - Telemetry/TOIN (data flywheel) statistics
+ - Cache and rate limiter stats
+ """
+ m = proxy.metrics
+
+ # Calculate average latency
+ avg_latency_ms = round(m.latency_sum_ms / m.latency_count, 2) if m.latency_count > 0 else 0
+ min_latency_ms = (
+ round(m.latency_min_ms, 2)
+ if m.latency_count > 0 and m.latency_min_ms != float("inf")
+ else 0
+ )
+ max_latency_ms = round(m.latency_max_ms, 2) if m.latency_count > 0 else 0
+
+ # Calculate Headroom overhead (optimization time only, excludes pass-through requests)
+ avg_overhead_ms = (
+ round(m.overhead_sum_ms / m.overhead_count, 2) if m.overhead_count > 0 else 0
+ )
+ min_overhead_ms = (
+ round(m.overhead_min_ms, 2)
+ if m.overhead_count > 0 and m.overhead_min_ms != float("inf")
+ else 0
+ )
+ max_overhead_ms = round(m.overhead_max_ms, 2) if m.overhead_count > 0 else 0
+
+ # Calculate TTFB (time to first byte)
+ avg_ttfb_ms = round(m.ttfb_sum_ms / m.ttfb_count, 2) if m.ttfb_count > 0 else 0
+ min_ttfb_ms = (
+ round(m.ttfb_min_ms, 2) if m.ttfb_count > 0 and m.ttfb_min_ms != float("inf") else 0
+ )
+ max_ttfb_ms = round(m.ttfb_max_ms, 2) if m.ttfb_count > 0 else 0
+
+ # Get compression store stats
+ store = get_compression_store()
+ compression_stats = store.get_stats()
+
+ # Get telemetry/TOIN stats
+ telemetry = get_telemetry_collector()
+ telemetry_stats = telemetry.get_stats()
+
+ # Get feedback loop stats
+ feedback = get_compression_feedback()
+ feedback_stats = feedback.get_stats()
+
+ # Build prefix cache stats once (used in both prefix_cache and cost)
+ prefix_cache_stats = _build_prefix_cache_stats(m, proxy.cost_tracker)
+
+ # Fetch CLI filtering savings (rtk — tokens avoided before reaching context)
+ cli_filtering_stats = _get_rtk_stats()
+ cli_tokens_avoided = (
+ cli_filtering_stats.get("tokens_saved", 0) if cli_filtering_stats else 0
+ )
+
+ # Calculate total tokens before compression
+ total_tokens_before = m.tokens_input_total + m.tokens_saved_total
+
+ # Build human-readable summary
+ summary = _build_session_summary(
+ proxy, m, prefix_cache_stats, cli_tokens_avoided, total_tokens_before
+ )
+ # DEBUG: log the summary payload for external upsert consumers
+ try:
+ logger.debug("/stats summary data: %r", summary)
+ except Exception:
+ logger.warning("Failed to log /stats summary payload")
+
+ # Compression cache stats (token mode)
+ compression_cache_stats: dict = {}
+ if proxy.config.mode == PROXY_MODE_TOKEN and proxy._compression_caches:
+ total_entries = 0
+ total_hits = 0
+ total_misses = 0
+ total_tokens_saved = 0
+ for cache in proxy._compression_caches.values():
+ s = cache.get_stats()
+ total_entries += s.get("entries", 0)
+ total_hits += s.get("hits", 0)
+ total_misses += s.get("misses", 0)
+ total_tokens_saved += s.get("total_tokens_saved", 0)
+ compression_cache_stats = {
+ "mode": PROXY_MODE_TOKEN,
+ "active_sessions": len(proxy._compression_caches),
+ "total_entries": total_entries,
+ "total_hits": total_hits,
+ "total_misses": total_misses,
+ "hit_rate": round(total_hits / max(1, total_hits + total_misses) * 100, 1),
+ "total_tokens_saved": total_tokens_saved,
+ }
+ else:
+ compression_cache_stats = {"mode": proxy.config.mode}
+
+ # Build unified savings summary (all layers)
+ compression_tokens = m.tokens_saved_total
+ cache_net_usd = prefix_cache_stats.get("totals", {}).get("net_savings_usd", 0.0)
+ total_tokens_all_layers = compression_tokens + cli_tokens_avoided
+ persistent_savings = m.savings_tracker.stats_preview()
+ display_session = persistent_savings.get("display_session", {})
+
+ return {
+ "summary": summary,
+ "savings": {
+ "total_tokens": total_tokens_all_layers,
+ "by_layer": {
+ "cli_filtering": {
+ "tokens": cli_tokens_avoided,
+ "description": "Tokens avoided by CLI output filtering (rtk) before reaching context",
+ },
+ "compression": {
+ "tokens": compression_tokens,
+ "description": "Tokens removed by proxy compression (SmartCrusher, ContentRouter, etc.)",
+ },
+ "prefix_cache": {
+ "discount_usd": round(cache_net_usd, 4),
+ "description": (
+ "Cost discount from provider prefix caching. "
+ "Headroom's CacheAligner improves hit rates; "
+ "baseline caching is provider-native."
+ ),
+ },
+ },
+ },
+ "requests": {
+ "total": m.requests_total,
+ "cached": m.requests_cached,
+ "rate_limited": m.requests_rate_limited,
+ "failed": m.requests_failed,
+ "by_provider": dict(m.requests_by_provider),
+ "by_model": dict(m.requests_by_model),
+ "by_stack": dict(m.requests_by_stack),
+ },
+ "tokens": {
+ "input": m.tokens_input_total,
+ "output": m.tokens_output_total,
+ "saved": m.tokens_saved_total,
+ "cli_tokens_avoided": cli_tokens_avoided,
+ "total_before_compression": total_tokens_before,
+ "savings_percent": round(
+ (m.tokens_saved_total / total_tokens_before * 100)
+ if total_tokens_before > 0
+ else 0,
+ 2,
+ ),
+ },
+ "latency": {
+ "average_ms": avg_latency_ms,
+ "min_ms": min_latency_ms,
+ "max_ms": max_latency_ms,
+ "total_requests": m.latency_count,
+ },
+ "overhead": {
+ "average_ms": avg_overhead_ms,
+ "min_ms": min_overhead_ms,
+ "max_ms": max_overhead_ms,
+ },
+ "ttfb": {
+ "average_ms": avg_ttfb_ms,
+ "min_ms": min_ttfb_ms,
+ "max_ms": max_ttfb_ms,
+ },
+ "pipeline_timing": {
+ name: {
+ "average_ms": round(
+ m.transform_timing_sum[name] / m.transform_timing_count[name], 2
+ ),
+ "max_ms": round(m.transform_timing_max[name], 2),
+ "count": m.transform_timing_count[name],
+ }
+ for name in sorted(m.transform_timing_sum.keys())
+ }
+ if m.transform_timing_sum
+ else {},
+ "waste_signals": dict(m.waste_signals_total) if m.waste_signals_total else {},
+ "savings_history": m.savings_history[-100:], # Last 100 data points
+ "display_session": display_session,
+ "persistent_savings": persistent_savings,
+ "prefix_cache": prefix_cache_stats,
+ "cost": _merge_cost_stats(
+ proxy.cost_tracker.stats() if proxy.cost_tracker else None,
+ prefix_cache_stats,
+ cli_tokens_avoided=cli_tokens_avoided,
+ ),
+ "compression": {
+ "ccr_entries": compression_stats.get("entry_count", 0),
+ "ccr_max_entries": compression_stats.get("max_entries", 0),
+ "original_tokens_cached": compression_stats.get("total_original_tokens", 0),
+ "compressed_tokens_cached": compression_stats.get("total_compressed_tokens", 0),
+ "ccr_retrievals": compression_stats.get("total_retrievals", 0),
+ },
+ "compression_cache": compression_cache_stats,
+ "anon_telemetry_shipping": is_telemetry_enabled(),
+ "telemetry": {
+ "enabled": telemetry_stats.get("enabled", False),
+ "total_compressions": telemetry_stats.get("total_compressions", 0),
+ "total_retrievals": telemetry_stats.get("total_retrievals", 0),
+ "global_retrieval_rate": round(telemetry_stats.get("global_retrieval_rate", 0), 4),
+ "tool_signatures_tracked": telemetry_stats.get("tool_signatures_tracked", 0),
+ "avg_compression_ratio": round(telemetry_stats.get("avg_compression_ratio", 0), 4),
+ "avg_token_reduction": round(telemetry_stats.get("avg_token_reduction", 0), 4),
+ },
+ "otel": get_otel_metrics_status(),
+ "langfuse": get_langfuse_tracing_status(),
+ "feedback_loop": {
+ "tools_tracked": feedback_stats.get("tools_tracked", 0),
+ "total_compressions": feedback_stats.get("total_compressions", 0),
+ "total_retrievals": feedback_stats.get("total_retrievals", 0),
+ "global_retrieval_rate": round(feedback_stats.get("global_retrieval_rate", 0), 4),
+ "tools_with_high_retrieval": sum(
+ 1
+ for p in feedback_stats.get("tool_patterns", {}).values()
+ if p.get("retrieval_rate", 0) > 0.3
+ ),
+ },
+ "toin": get_toin().get_stats(),
+ "cli_filtering": cli_filtering_stats,
+ "cache": await proxy.cache.stats() if proxy.cache else None,
+ "rate_limiter": await proxy.rate_limiter.stats() if proxy.rate_limiter else None,
+ "recent_requests": proxy.logger.get_recent(10) if proxy.logger else [],
+ "log_full_messages": proxy.config.log_full_messages if proxy else False,
+ **get_quota_registry().get_all_stats(),
+ }
+
+ async def _get_cached_stats_payload() -> dict[str, Any]:
+ """Return a short-TTL cached `/stats` snapshot for dashboard polling."""
+ now = time.monotonic()
+ cached_payload = cast(dict[str, Any] | None, _stats_snapshot.get("value"))
+ if cached_payload is not None and now < float(_stats_snapshot["expires_at"]):
+ return cached_payload
+
+ async with _stats_snapshot_lock:
+ now = time.monotonic()
+ cached_payload = cast(dict[str, Any] | None, _stats_snapshot.get("value"))
+ if cached_payload is not None and now < float(_stats_snapshot["expires_at"]):
+ return cached_payload
+
+ payload = await _build_stats_payload()
+ _stats_snapshot["value"] = payload
+ _stats_snapshot["expires_at"] = time.monotonic() + DASHBOARD_STATS_CACHE_TTL_SECONDS
+ return payload
+
+ @app.get("/stats")
+ async def stats(cached: bool = False):
+ """Get comprehensive proxy statistics.
+
+ This is the main stats endpoint - it aggregates data from all subsystems:
+ - Request metrics (total, cached, failed, by model/provider)
+ - Token usage and savings
+ - Cost tracking
+ - Canonical persisted display_session metrics for downstream dashboards
+ - Compression (CCR) statistics
+ - Telemetry/TOIN (data flywheel) statistics
+ - Cache and rate limiter stats
+
+ Use ``?cached=1`` for the dashboard fast path. That returns a short-TTL
+ snapshot to avoid rebuilding the full payload on every UI poll.
+ """
+ if cached:
+ return await _get_cached_stats_payload()
+ return await _build_stats_payload()
+
+ @app.get("/stats-history")
+ async def stats_history(
+ format: Literal["json", "csv"] = "json",
+ series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
+ history_mode: Literal["compact", "full", "none"] = "compact",
+ ):
+ """Get durable proxy compression history plus display-session state."""
+ if format == "csv":
+ filename = f"headroom-stats-history-{series}.csv"
+ return Response(
+ content=proxy.metrics.savings_tracker.export_csv(series=series),
+ media_type="text/csv; charset=utf-8",
+ headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+ )
+
+ return proxy.metrics.savings_tracker.history_response(history_mode=history_mode)
+
+ @app.get("/transformations/feed")
+ async def transformations_feed(limit: int = 20):
+ """Get recent message transformations for the live feed.
+
+ Returns empty list if log_full_messages is disabled (messages are not stored).
+ """
+ if limit > 100:
+ limit = 100
+
+ transformations = []
+ log_full_messages = proxy.config.log_full_messages if proxy else False
+
+ if proxy and proxy.logger:
+ logs = proxy.logger.get_recent_with_messages(limit)
+ for log in logs:
+ transformations.append(
+ {
+ "request_id": log.get("request_id"),
+ "timestamp": log.get("timestamp"),
+ "provider": log.get("provider"),
+ "model": log.get("model"),
+ "input_tokens_original": log.get("input_tokens_original"),
+ "input_tokens_optimized": log.get("input_tokens_optimized"),
+ "tokens_saved": log.get("tokens_saved"),
+ "savings_percent": log.get("savings_percent"),
+ "transforms_applied": log.get("transforms_applied", []),
+ "request_messages": log.get("request_messages"),
+ "response_content": log.get("response_content"),
+ "turn_id": log.get("turn_id"),
+ }
+ )
+
+ return {"transformations": transformations, "log_full_messages": log_full_messages}
+
+ @app.get("/subscription-window")
+ async def subscription_window():
+ """Current Anthropic subscription window utilisation and Headroom contribution."""
+ tracker = get_subscription_tracker()
+ if tracker is None:
+ return JSONResponse(
+ status_code=503,
+ content={"error": "Subscription tracking is not enabled"},
+ )
+ return JSONResponse(content=tracker.state)
+
+ @app.get("/quota")
+ async def quota():
+ """Unified quota/rate-limit stats for all registered providers (Anthropic, Codex, Copilot)."""
+ return JSONResponse(content=get_quota_registry().get_all_stats())
+
+ @app.get("/metrics")
+ async def metrics():
+ """Prometheus metrics endpoint."""
+ return PlainTextResponse(
+ await proxy.metrics.export(),
+ media_type="text/plain; version=0.0.4",
+ )
+
+ # Debug endpoints
+ @app.get("/debug/memory")
+ async def debug_memory():
+ """Get detailed memory usage statistics.
+
+ Returns memory usage for all tracked components including:
+ - Process-level memory (RSS, VMS, percent)
+ - Per-component memory usage and budgets
+ - Cache hit/miss statistics
+ - Total tracked vs target budget
+
+ This endpoint is useful for debugging memory issues and
+ monitoring memory budgets.
+ """
+ from ..memory.tracker import MemoryTracker
+
+ tracker = MemoryTracker.get()
+
+ # Register components if not already registered
+ _register_memory_components(proxy, tracker)
+
+ report = tracker.get_report()
+ return report.to_dict()
+
+ @app.post("/cache/clear")
+ async def clear_cache():
+ """Clear the response cache."""
+ if proxy.cache:
+ await proxy.cache.clear()
+ return {"status": "cleared"}
+ return {"status": "cache disabled"}
+
+ # CCR (Compress-Cache-Retrieve) endpoints
+ @app.post("/v1/retrieve")
+ async def ccr_retrieve(request: Request):
+ """Retrieve original content from CCR compression cache.
+
+ This is the "Retrieve" part of CCR (Compress-Cache-Retrieve).
+ When SmartCrusher compresses tool outputs, the original data is cached.
+ LLMs can call this endpoint to get more data if needed.
+
+ Request body:
+ hash (str): Hash key from compression marker (required)
+ query (str): Optional search query to filter results
+
+ Response:
+ Full retrieval: {"hash": "...", "original_content": "...", ...}
+ Search: {"hash": "...", "query": "...", "results": [...], "count": N}
+ """
+ data = await request.json()
+ hash_key = data.get("hash")
+ query = data.get("query")
+
+ if not hash_key:
+ raise HTTPException(status_code=400, detail="hash required")
+
+ store = get_compression_store()
+
+ if query:
+ # Search within cached content
+ results = store.search(hash_key, query)
+ return {
+ "hash": hash_key,
+ "query": query,
+ "results": results,
+ "count": len(results),
+ }
+ else:
+ # Return full original content
+ entry = store.retrieve(hash_key)
+ if entry:
+ return {
+ "hash": hash_key,
+ "original_content": entry.original_content,
+ "original_tokens": entry.original_tokens,
+ "original_item_count": entry.original_item_count,
+ "compressed_item_count": entry.compressed_item_count,
+ "tool_name": entry.tool_name,
+ "retrieval_count": entry.retrieval_count,
+ }
+ raise HTTPException(
+ status_code=404, detail="Entry not found or expired (TTL: 5 minutes)"
+ )
+
+ @app.get("/v1/retrieve/stats")
+ async def ccr_stats():
+ """Get CCR compression store statistics."""
+ store = get_compression_store()
+ stats = store.get_stats()
+ events = store.get_retrieval_events(limit=20)
+ return {
+ "store": stats,
+ "recent_retrievals": [
+ {
+ "hash": e.hash,
+ "query": e.query,
+ "items_retrieved": e.items_retrieved,
+ "total_items": e.total_items,
+ "tool_name": e.tool_name,
+ "retrieval_type": e.retrieval_type,
+ }
+ for e in events
+ ],
+ }
+
+ @app.get("/v1/feedback")
+ async def ccr_feedback():
+ """Get CCR feedback loop statistics and learned patterns.
+
+ This endpoint exposes the feedback loop's learned patterns for monitoring
+ and debugging. It shows:
+ - Per-tool retrieval rates (high = compress less aggressively)
+ - Common search queries per tool
+ - Queried fields (suggest what to preserve)
+
+ Use this to understand how well compression is working and whether
+ the feedback loop is adjusting appropriately.
+ """
+ feedback = get_compression_feedback()
+ stats = feedback.get_stats()
+ return {
+ "feedback": stats,
+ "hints_example": {
+ tool_name: {
+ "hints": {
+ "max_items": hints.max_items
+ if (hints := feedback.get_compression_hints(tool_name))
+ else 15,
+ "suggested_items": hints.suggested_items if hints else None,
+ "skip_compression": hints.skip_compression if hints else False,
+ "preserve_fields": hints.preserve_fields if hints else [],
+ "reason": hints.reason if hints else "",
+ }
+ }
+ for tool_name in list(stats.get("tool_patterns", {}).keys())[:5]
+ },
+ }
+
+ @app.get("/v1/feedback/{tool_name}")
+ async def ccr_feedback_for_tool(tool_name: str):
+ """Get compression hints for a specific tool.
+
+ Returns feedback-based hints that would be used for compressing
+ this tool's output.
+ """
+ feedback = get_compression_feedback()
+ hints = feedback.get_compression_hints(tool_name)
+ patterns = feedback.get_all_patterns().get(tool_name)
+
+ return {
+ "tool_name": tool_name,
+ "hints": {
+ "max_items": hints.max_items,
+ "min_items": hints.min_items,
+ "suggested_items": hints.suggested_items,
+ "aggressiveness": hints.aggressiveness,
+ "skip_compression": hints.skip_compression,
+ "preserve_fields": hints.preserve_fields,
+ "reason": hints.reason,
+ },
+ "pattern": {
+ "total_compressions": patterns.total_compressions if patterns else 0,
+ "total_retrievals": patterns.total_retrievals if patterns else 0,
+ "retrieval_rate": patterns.retrieval_rate if patterns else 0.0,
+ "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0,
+ "search_rate": patterns.search_rate if patterns else 0.0,
+ "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [],
+ "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [],
+ }
+ if patterns
+ else None,
+ }
+
+ # Telemetry endpoints (Data Flywheel)
+ @app.get("/v1/telemetry")
+ async def telemetry_stats():
+ """Get telemetry statistics for the data flywheel.
+
+ This endpoint exposes privacy-preserving telemetry data that powers
+ the data flywheel - learning optimal compression strategies across
+ tool types based on usage patterns.
+
+ What's collected (anonymized):
+ - Tool output structure patterns (field types, not values)
+ - Compression decisions and ratios
+ - Retrieval patterns (rate, type, not content)
+ - Strategy effectiveness
+
+ What's NOT collected:
+ - Actual data values
+ - User identifiers
+ - Queries or search terms
+ - File paths or tool names (hashed by default)
+ """
+ telemetry = get_telemetry_collector()
+ return telemetry.get_stats()
+
+ @app.get("/v1/telemetry/export")
+ async def telemetry_export():
+ """Export full telemetry data for aggregation.
+
+ This endpoint exports all telemetry data in a format suitable for
+ cross-user aggregation. The data is privacy-preserving - no actual
+ values are included, only structural patterns and statistics.
+
+ Use this for:
+ - Building a central learning service
+ - Sharing learned patterns across instances
+ - Analysis and debugging
+ """
+ telemetry = get_telemetry_collector()
+ return telemetry.export_stats()
+
+ @app.post("/v1/telemetry/import")
+ async def telemetry_import(request: Request):
+ """Import telemetry data from another source.
+
+ This allows merging telemetry from multiple sources for cross-user
+ learning. The imported data is merged with existing statistics.
+
+ Request body: Telemetry export data from /v1/telemetry/export
+ """
+ telemetry = get_telemetry_collector()
+ data = await request.json()
+ telemetry.import_stats(data)
+ return {"status": "imported", "current_stats": telemetry.get_stats()}
+
+ @app.get("/v1/telemetry/tools")
+ async def telemetry_tools():
+ """Get telemetry statistics for all tracked tool signatures.
+
+ Returns statistics per tool signature (anonymized), including:
+ - Compression ratios and strategy usage
+ - Retrieval rates (high = compression too aggressive)
+ - Learned recommendations
+ """
+ telemetry = get_telemetry_collector()
+ all_stats = telemetry.get_all_tool_stats()
+ return {
+ "tool_count": len(all_stats),
+ "tools": {sig_hash: stats.to_dict() for sig_hash, stats in all_stats.items()},
+ }
+
+ @app.get("/v1/telemetry/tools/{signature_hash}")
+ async def telemetry_tool_detail(signature_hash: str):
+ """Get detailed telemetry for a specific tool signature.
+
+ Includes learned recommendations if enough data has been collected.
+ """
+ telemetry = get_telemetry_collector()
+ stats = telemetry.get_tool_stats(signature_hash)
+ recommendations = telemetry.get_recommendations(signature_hash)
+
+ if stats is None:
+ raise HTTPException(
+ status_code=404, detail=f"No telemetry found for signature: {signature_hash}"
+ )
+
+ return {
+ "signature_hash": signature_hash,
+ "stats": stats.to_dict(),
+ "recommendations": recommendations,
+ }
+
+ # TOIN (Tool Output Intelligence Network) endpoints
+ @app.get("/v1/toin/stats")
+ async def toin_stats():
+ """Get overall TOIN statistics.
+
+ Returns aggregated statistics from the Tool Output Intelligence Network,
+ which learns optimal compression strategies across all tool types.
+
+ Response includes:
+ - enabled: Whether TOIN is enabled
+ - patterns_tracked: Number of unique tool patterns being tracked
+ - total_compressions: Total compression events recorded
+ - total_retrievals: Total retrieval events recorded
+ - global_retrieval_rate: Overall retrieval rate (high = compression too aggressive)
+ - patterns_with_recommendations: Patterns with enough data for recommendations
+ """
+ toin = get_toin()
+ return toin.get_stats()
+
+ @app.get("/v1/toin/patterns")
+ async def toin_patterns(limit: int = 20):
+ """List TOIN patterns with most samples.
+
+ Returns patterns sorted by sample_size descending. Use this to see
+ which tool types have the most data and their learned behaviors.
+
+ Query params:
+ limit: Maximum number of patterns to return (default 20)
+
+ Response includes for each pattern:
+ - hash: Truncated tool signature hash (12 chars)
+ - compressions: Total compression events
+ - retrievals: Total retrieval events
+ - retrieval_rate: Percentage of compressions that triggered retrieval
+ - confidence: Confidence level in recommendations (0.0-1.0)
+ - skip_recommended: Whether TOIN recommends skipping compression
+ - optimal_max_items: Learned optimal max_items setting
+ """
+ toin = get_toin()
+ exported = toin.export_patterns()
+ patterns_data = exported.get("patterns", {})
+
+ # Convert to list and sort by sample_size
+ patterns_list = []
+ for sig_hash, pattern_dict in patterns_data.items():
+ sample_size = pattern_dict.get("sample_size", 0)
+ total_compressions = pattern_dict.get("total_compressions", 0)
+ total_retrievals = pattern_dict.get("total_retrievals", 0)
+ retrieval_rate = (
+ total_retrievals / total_compressions if total_compressions > 0 else 0.0
+ )
+
+ patterns_list.append(
+ {
+ "hash": sig_hash[:12],
+ "compressions": total_compressions,
+ "retrievals": total_retrievals,
+ "retrieval_rate": f"{retrieval_rate:.1%}",
+ "confidence": round(pattern_dict.get("confidence", 0.0), 3),
+ "skip_recommended": pattern_dict.get("skip_compression_recommended", False),
+ "optimal_max_items": pattern_dict.get("optimal_max_items", 20),
+ "sample_size": sample_size,
+ }
+ )
+
+ # Sort by sample_size descending
+ patterns_list.sort(key=lambda p: p["sample_size"], reverse=True)
+
+ # Remove sample_size from output (used only for sorting)
+ for p in patterns_list:
+ del p["sample_size"]
+
+ return patterns_list[:limit]
+
+ @app.get("/v1/toin/pattern/{hash_prefix}")
+ async def toin_pattern_detail(hash_prefix: str):
+ """Get detailed TOIN pattern info by hash prefix.
+
+ Searches for a pattern where the tool signature hash starts with
+ the provided prefix. Returns full pattern details if found.
+
+ Path params:
+ hash_prefix: Beginning of the tool signature hash (min 4 chars recommended)
+
+ Response: Full pattern.to_dict() with all learned statistics and recommendations.
+ """
+ toin = get_toin()
+ exported = toin.export_patterns()
+ patterns_data = exported.get("patterns", {})
+
+ # Search for pattern with matching hash prefix
+ for sig_hash, pattern_dict in patterns_data.items():
+ if sig_hash.startswith(hash_prefix):
+ return pattern_dict
+
+ raise HTTPException(
+ status_code=404, detail=f"No TOIN pattern found with hash starting with: {hash_prefix}"
+ )
+
+ @app.get("/v1/retrieve/{hash_key}")
+ async def ccr_retrieve_get(hash_key: str, query: str | None = None):
+ """GET version of CCR retrieve for easier testing."""
+ store = get_compression_store()
+
+ if query:
+ results = store.search(hash_key, query)
+ return {
+ "hash": hash_key,
+ "query": query,
+ "results": results,
+ "count": len(results),
+ }
+ else:
+ entry = store.retrieve(hash_key)
+ if entry:
+ return {
+ "hash": hash_key,
+ "original_content": entry.original_content,
+ "original_tokens": entry.original_tokens,
+ "original_item_count": entry.original_item_count,
+ "compressed_item_count": entry.compressed_item_count,
+ "tool_name": entry.tool_name,
+ "retrieval_count": entry.retrieval_count,
+ }
+ raise HTTPException(status_code=404, detail="Entry not found or expired")
+
+ # CCR Tool Call Handler - for agent frameworks to call when LLM uses headroom_retrieve
+ @app.post("/v1/retrieve/tool_call")
+ async def ccr_handle_tool_call(request: Request):
+ """Handle a CCR tool call from an LLM response.
+
+ This endpoint accepts tool call formats from various providers and returns
+ a properly formatted tool result. Agent frameworks can use this to handle
+ CCR tool calls without implementing the retrieval logic themselves.
+
+ Request body (Anthropic format):
+ {
+ "tool_call": {
+ "id": "toolu_123",
+ "name": "headroom_retrieve",
+ "input": {"hash": "abc123", "query": "optional search"}
+ },
+ "provider": "anthropic"
+ }
+
+ Request body (OpenAI format):
+ {
+ "tool_call": {
+ "id": "call_123",
+ "function": {
+ "name": "headroom_retrieve",
+ "arguments": "{\"hash\": \"abc123\"}"
+ }
+ },
+ "provider": "openai"
+ }
+
+ Response:
+ {
+ "tool_result": {...}, # Formatted for the provider
+ "success": true,
+ "data": {...} # Raw retrieval data
+ }
+ """
+ data = await request.json()
+ tool_call = data.get("tool_call", {})
+ provider = data.get("provider", "anthropic")
+
+ # Parse the tool call
+ hash_key, query = parse_tool_call(tool_call, provider)
+
+ if hash_key is None:
+ raise HTTPException(
+ status_code=400, detail=f"Invalid tool call or not a {CCR_TOOL_NAME} call"
+ )
+
+ # Perform retrieval
+ store = get_compression_store()
+
+ if query:
+ results = store.search(hash_key, query)
+ retrieval_data = {
+ "hash": hash_key,
+ "query": query,
+ "results": results,
+ "count": len(results),
+ }
+ else:
+ entry = store.retrieve(hash_key)
+ if entry:
+ retrieval_data = {
+ "hash": hash_key,
+ "original_content": entry.original_content,
+ "original_item_count": entry.original_item_count,
+ "compressed_item_count": entry.compressed_item_count,
+ }
+ else:
+ retrieval_data = {
+ "error": "Entry not found or expired (TTL: 5 minutes)",
+ "hash": hash_key,
+ }
+
+ # Format tool result for provider
+ tool_call_id = tool_call.get("id", "")
+ result_content = json.dumps(retrieval_data, indent=2)
+
+ if provider == "anthropic":
+ tool_result = {
+ "type": "tool_result",
+ "tool_use_id": tool_call_id,
+ "content": result_content,
+ }
+ elif provider == "openai":
+ tool_result = {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": result_content,
+ }
+ else:
+ tool_result = {
+ "tool_call_id": tool_call_id,
+ "content": result_content,
+ }
+
+ return {
+ "tool_result": tool_result,
+ "success": "error" not in retrieval_data,
+ "data": retrieval_data,
+ }
+
+ # Compression-only endpoint (for TypeScript SDK and other HTTP clients)
+ @app.post("/v1/compress")
+ async def compress_messages(request: Request):
+ return await proxy.handle_compress(request)
+
+ register_provider_routes(app, proxy)
+
+ return app
+
+
+ def _get_code_aware_banner_status(config: ProxyConfig) -> str:
+ """Get code-aware compression status line for banner."""
+ if config.code_aware_enabled:
+ if is_tree_sitter_available():
+ return "ENABLED (AST-based)"
+ else:
+ return "NOT INSTALLED (pip install headroom-ai[code])"
+ else:
+ if is_tree_sitter_available():
+ return "DISABLED (remove --no-code-aware to enable)"
+ return "DISABLED"
+
+
+ def run_server(
+ config: ProxyConfig | None = None,
+ workers: int = 1,
+ limit_concurrency: int = 1000,
+ ):
+ """Run the proxy server.
+
+ Args:
+ config: Proxy configuration
+ workers: Number of worker processes (use N for multi-core scaling)
+ limit_concurrency: Max concurrent connections before 503 response
+ """
+ if not FASTAPI_AVAILABLE:
+ print("ERROR: FastAPI required. Install: pip install fastapi uvicorn httpx")
+ sys.exit(1)
+
+ config = config or ProxyConfig()
+ app = create_app(config)
+
+ code_aware_status = _get_code_aware_banner_status(config)
+
+ # Format connection pool info
+ pool_info = f"max={config.max_connections}, keepalive={config.max_keepalive_connections}"
+ http2_status = "ENABLED" if config.http2 else "DISABLED"
+
+ backend_status = format_backend_status(
+ backend=config.backend,
+ anyllm_provider=config.anyllm_provider,
+ bedrock_region=config.bedrock_region,
+ )
+
+ print(f"""
+ ╔══════════════════════════════════════════════════════════════════════╗
+ ║ HEADROOM PROXY SERVER ║
+ ╠══════════════════════════════════════════════════════════════════════╣
+ ║ Version: 1.0.0 ║
+ ║ Listening: http://{config.host}:{config.port:<5} ║
+ ║ Workers: {workers:<3} Concurrency Limit: {limit_concurrency:<5} ║
+ ║ Backend: {backend_status:<59}║
+ ╠══════════════════════════════════════════════════════════════════════╣
+ ║ FEATURES: ║
+ ║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║
+ ║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║
+ ║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║
+ ║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║
+ ║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║
+ ║ Code-Aware: {code_aware_status:<52}║
+ ║ HTTP/2: {http2_status:<52}║
+ ║ Conn Pool: {pool_info:<52}║
+ ╠══════════════════════════════════════════════════════════════════════╣
+ ║ USAGE: ║
+ ║ Claude Code: ANTHROPIC_BASE_URL=http://{config.host}:{config.port} claude ║
+ ║ Cursor: Set base URL in settings ║
+ ╠══════════════════════════════════════════════════════════════════════╣
+ ║ ENDPOINTS: ║
+ ║ /livez Process liveness ║
+ ║ /readyz Traffic readiness ║
+ ║ /health Aggregate health ║
+ ║ /stats Detailed statistics ║
+ ║ /metrics Prometheus metrics ║
+ ║ /cache/clear Clear response cache ║
+ ║ /v1/retrieve CCR: Retrieve compressed content ║
+ ║ /v1/retrieve/stats CCR: Compression store stats ║
+ ║ /v1/retrieve/tool_call CCR: Handle LLM tool calls ║
+ ║ /v1/feedback CCR: Feedback loop stats & patterns ║
+ ║ /v1/feedback/{{tool}} CCR: Compression hints for a tool ║
+ ║ /v1/telemetry Data flywheel: Telemetry stats ║
+ ║ /v1/telemetry/export Data flywheel: Export for aggregation ║
+ ║ /v1/telemetry/tools Data flywheel: Per-tool stats ║
+ ║ /v1/toin/stats TOIN: Overall intelligence stats ║
+ ║ /v1/toin/patterns TOIN: List learned patterns ║
+ ║ /v1/toin/pattern/{{hash}} TOIN: Pattern details by hash ║
+ ╚══════════════════════════════════════════════════════════════════════╝
+ """)
+
+ uvicorn.run(
+ app,
+ host=config.host,
+ port=config.port,
+ log_level="warning",
+ workers=workers if workers > 1 else None, # None = single process (default)
+ limit_concurrency=limit_concurrency,
+ # Defense-in-depth: the loopback guard for /debug/* endpoints trusts
+ # request.client.host. uvicorn's ProxyHeadersMiddleware rewrites that
+ # from X-Forwarded-For when FORWARDED_ALLOW_IPS is broader than the
+ # default. Disabling proxy_headers here guarantees the guard sees the
+ # real peer address regardless of env.
+ proxy_headers=False,
+ )
+
+
+ def _get_env_bool(name: str, default: bool) -> bool:
+ """Get boolean from environment variable."""
+ val = os.environ.get(name)
+ if val is None:
+ return default
+ return val.lower() in ("true", "1", "yes", "on")
+
+
+ def _get_env_int(name: str, default: int) -> int:
+ """Get integer from environment variable."""
+ val = os.environ.get(name)
+ if val is None:
+ return default
+ try:
+ return int(val)
+ except ValueError:
+ return default
+
+
+ def _get_env_float(name: str, default: float) -> float:
+ """Get float from environment variable."""
+ val = os.environ.get(name)
+ if val is None:
+ return default
+ try:
+ return float(val)
+ except ValueError:
+ return default
+
+
+ def _get_env_str(name: str, default: str) -> str:
+ """Get string from environment variable."""
+ return os.environ.get(name, default)
+
+
+ def _parse_tool_profiles(cli_profiles: list[str]) -> dict[str, Any]:
+ """Parse tool profiles from CLI args and HEADROOM_TOOL_PROFILES env var.
+
+ Format: ToolName:level (e.g., Grep:conservative, Bash:moderate)
+ Env var format: comma-separated (e.g., "Grep:conservative,Bash:moderate")
+
+ Returns:
+ Dict mapping tool names to CompressionProfile instances.
+ """
+ from headroom.config import PROFILE_PRESETS, CompressionProfile
+
+ profiles: dict[str, CompressionProfile] = {}
+ raw_entries: list[str] = list(cli_profiles)
+
+ # Also check env var
+ env_val = os.environ.get("HEADROOM_TOOL_PROFILES", "")
+ if env_val:
+ raw_entries.extend(e.strip() for e in env_val.split(",") if e.strip())
+
+ for entry in raw_entries:
+ if ":" not in entry:
+ logger.warning("Invalid tool profile format (expected ToolName:level): %s", entry)
+ continue
+ tool_name, level = entry.split(":", 1)
+ tool_name = tool_name.strip()
+ level = level.strip().lower()
+
+ if level in PROFILE_PRESETS:
+ profiles[tool_name] = PROFILE_PRESETS[level]
+ else:
+ logger.warning(
+ "Unknown profile level '%s' for tool '%s'. Use: conservative, moderate, aggressive",
+ level,
+ tool_name,
+ )
+
+ return profiles
+
+
+ if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Headroom Proxy Server")
+
+ # Server
+ parser.add_argument("--host", default="127.0.0.1")
+ parser.add_argument("--port", type=int, default=8787)
+ parser.add_argument(
+ "--openai-api-url", help=f"Custom OpenAI API URL (default: {DEFAULT_OPENAI_API_URL})"
+ )
+ parser.add_argument(
+ "--anthropic-api-url",
+ help=f"Custom Anthropic API URL (default: {DEFAULT_ANTHROPIC_API_URL})",
+ )
+
+ # Backend (anthropic direct, bedrock, openrouter, anyllm, or litellm-<provider>)
+ parser.add_argument(
+ "--backend",
+ default="anthropic",
+ help=(
+ "Backend: 'anthropic' (direct), 'bedrock' (AWS), 'openrouter', "
+ "'anyllm' (any-llm), or 'litellm-<provider>' (e.g., litellm-hosted_vllm, litellm-vertex)"
+ ),
+ )
+ parser.add_argument(
+ "--bedrock-region",
+ default="us-west-2",
+ help="AWS region for Bedrock backend (default: us-west-2)",
+ )
+ parser.add_argument(
+ "--bedrock-profile",
+ help="AWS profile for Bedrock backend (default: use default credentials)",
+ )
+ parser.add_argument(
+ "--openrouter-api-key",
+ help="OpenRouter API key (or set OPENROUTER_API_KEY env var)",
+ )
+ parser.add_argument(
+ "--anyllm-provider",
+ default="openai",
+ help="any-llm provider: openai, anthropic, mistral, groq, ollama, bedrock, etc. (default: openai)",
+ )
+
+ # Connection pool (scalability)
+ parser.add_argument(
+ "--max-connections",
+ type=int,
+ default=500,
+ help="Max connections to upstream APIs (default: 500)",
+ )
+ parser.add_argument(
+ "--max-keepalive", type=int, default=100, help="Max keepalive connections (default: 100)"
+ )
+ parser.add_argument(
+ "--no-http2",
+ action="store_true",
+ help="Disable HTTP/2 (enabled by default for better throughput)",
+ )
+ parser.add_argument(
+ "--workers",
+ type=int,
+ default=1,
+ help="Number of worker processes (default: 1, use N for multi-core)",
+ )
+ parser.add_argument(
+ "--limit-concurrency",
+ type=int,
+ default=1000,
+ help="Max concurrent connections before 503 (default: 1000)",
+ )
+
+ # Optimization
+ parser.add_argument("--no-optimize", action="store_true", help="Disable optimization")
+ parser.add_argument("--min-tokens", type=int, default=500, help="Min tokens to crush")
+ parser.add_argument("--max-items", type=int, default=50, help="Max items after crush")
+ parser.add_argument(
+ "--tool-profile",
+ action="append",
+ default=[],
+ help="Per-tool compression profile: ToolName:level (e.g., Grep:conservative, Bash:moderate, WebFetch:aggressive). "
+ "Can be specified multiple times. Also settable via HEADROOM_TOOL_PROFILES env var.",
+ )
+
+ # Caching
+ parser.add_argument("--no-cache", action="store_true", help="Disable caching")
+ parser.add_argument("--cache-ttl", type=int, default=3600, help="Cache TTL seconds")
+
+ # Rate limiting
+ parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")
+ parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")
+ parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")
+
+ # Cost
+ parser.add_argument("--budget", type=float, help="Budget limit in USD")
+ parser.add_argument("--budget-period", choices=["hourly", "daily", "monthly"], default="daily")
+
+ # Logging
+ parser.add_argument("--log-file", help="Log file path")
+ parser.add_argument("--log-messages", action="store_true", help="Log full messages")
+
+ # Smart routing (content-aware compression)
+ parser.add_argument(
+ "--no-smart-routing",
+ action="store_true",
+ help="Disable smart routing (use legacy sequential pipeline)",
+ )
+
+ # Code-aware compression
+ parser.add_argument(
+ "--code-aware",
+ action="store_true",
+ help="Enable AST-based code compression (requires: pip install headroom-ai[code])",
+ )
+ parser.add_argument(
+ "--no-code-aware",
+ action="store_true",
+ help="Disable code-aware compression",
+ )
+
+ args = parser.parse_args()
+
+ # Environment variable defaults (HEADROOM_* prefix)
+ # CLI args override env vars, env vars override ProxyConfig defaults
+ env_smart_routing = _get_env_bool("HEADROOM_SMART_ROUTING", True)
+ env_code_aware = _get_env_bool("HEADROOM_CODE_AWARE_ENABLED", True)
+ env_optimize = _get_env_bool("HEADROOM_OPTIMIZE", True)
+ env_cache = _get_env_bool("HEADROOM_CACHE_ENABLED", True)
+ env_rate_limit = _get_env_bool("HEADROOM_RATE_LIMIT_ENABLED", True)
+
+ # Determine settings: CLI flags override env vars
+ # --no-X explicitly disables, --X explicitly enables, neither uses env var
+ smart_routing = env_smart_routing if not args.no_smart_routing else False
+ code_aware_enabled = (
+ env_code_aware
+ if not (args.code_aware or args.no_code_aware)
+ else (args.code_aware or not args.no_code_aware)
+ )
+ optimize = env_optimize if not args.no_optimize else False
+ cache_enabled = env_cache if not args.no_cache else False
+ rate_limit_enabled = env_rate_limit if not args.no_rate_limit else False
+
+ # Set OpenRouter API key from CLI if provided
+ if hasattr(args, "openrouter_api_key") and args.openrouter_api_key:
+ os.environ["OPENROUTER_API_KEY"] = args.openrouter_api_key
+
+ # Parse per-tool compression profiles from CLI and env var
+ tool_profiles = _parse_tool_profiles(args.tool_profile)
+
+ config = ProxyConfig(
+ host=_get_env_str("HEADROOM_HOST", args.host),
+ port=_get_env_int("HEADROOM_PORT", args.port),
+ openai_api_url=_get_env_str("OPENAI_TARGET_API_URL", args.openai_api_url),
+ anthropic_api_url=_get_env_str("ANTHROPIC_TARGET_API_URL", args.anthropic_api_url),
+ # Backend settings
+ backend=_get_env_str("HEADROOM_BACKEND", args.backend), # type: ignore[arg-type]
+ bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", args.bedrock_region),
+ bedrock_profile=args.bedrock_profile or os.environ.get("AWS_PROFILE"),
+ anyllm_provider=_get_env_str("HEADROOM_ANYLLM_PROVIDER", args.anyllm_provider),
+ optimize=optimize,
+ min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", args.min_tokens),
+ max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", args.max_items),
+ cache_enabled=cache_enabled,
+ cache_ttl_seconds=_get_env_int("HEADROOM_CACHE_TTL", args.cache_ttl),
+ rate_limit_enabled=rate_limit_enabled,
+ rate_limit_requests_per_minute=_get_env_int("HEADROOM_RPM", args.rpm),
+ rate_limit_tokens_per_minute=_get_env_int("HEADROOM_TPM", args.tpm),
+ budget_limit_usd=args.budget,
+ budget_period=args.budget_period,
+ log_file=_get_env_str("HEADROOM_LOG_FILE", args.log_file)
+ if args.log_file
+ else os.environ.get("HEADROOM_LOG_FILE"),
+ log_full_messages=args.log_messages or _get_env_bool("HEADROOM_LOG_MESSAGES", False),
+ smart_routing=smart_routing,
+ code_aware_enabled=code_aware_enabled,
+ # Connection pool settings
+ max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),
+ max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),
+ http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),
+ tool_profiles=tool_profiles if tool_profiles else None,
+ mode=normalize_proxy_mode(_get_env_str("HEADROOM_MODE", PROXY_MODE_TOKEN)),
+ )
+
+ # Get worker and concurrency settings
+ workers = _get_env_int("HEADROOM_WORKERS", args.workers)
+ limit_concurrency = _get_env_int("HEADROOM_LIMIT_CONCURRENCY", args.limit_concurrency)
+
+ run_server(config, workers=workers, limit_concurrency=limit_concurrency)