Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat: make token_headroom the default mode + fix Gemini handler bug
Tejas Chopra9c5580baeefa
diff --git a/headroom/ccr/mcp_server.py b/headroom/ccr/mcp_server.pyindex 833c4bc3d..1e53d2674 100644--- a/headroom/ccr/mcp_server.py+++ b/headroom/ccr/mcp_server.py@@ -76,7 +76,7 @@ def _format_session_summary(summary: dict[str, Any], local_stats: dict[str, Any]lines.append("Headroom Session Summary")lines.append("=" * 40)- mode = summary.get("mode", "cost_savings")+ mode = summary.get("mode", "token_headroom")api_reqs = summary.get("api_requests", 0)model = summary.get("primary_model", "unknown")lines.append(f"Mode: {mode} | {api_reqs} API requests | {model}")diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 236cf5a7e..8cf0f8af8 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -14,7 +14,7 @@ from .main import main"--mode",default=None,type=click.Choice(["cost_savings", "token_headroom"]),- help="Optimization mode: cost_savings (preserve prefix cache) or token_headroom (compress for session extension). Default: cost_savings. Env: HEADROOM_MODE",+ help="Optimization mode: token_headroom (compress for session extension) or cost_savings (preserve prefix cache). Default: token_headroom. Env: HEADROOM_MODE",)@click.option("--no-optimize", is_flag=True, help="Disable optimization (passthrough mode)")@click.option("--no-cache", is_flag=True, help="Disable semantic caching")@@ -184,7 +184,7 @@ def proxy(effective_anyllm_provider = os.environ.get("HEADROOM_ANYLLM_PROVIDER") or anyllm_provider# Resolve mode: CLI flag > env var > default- effective_mode = mode or os.environ.get("HEADROOM_MODE", "cost_savings")+ effective_mode = mode or os.environ.get("HEADROOM_MODE", "token_headroom")config = ProxyConfig(host=host,diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 50d0f4a80..6d2a1fa80 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -527,6 +527,21 @@ def _build_session_summary(# Maximum request body size (100MB - increased to support image-heavy requests)MAX_REQUEST_BODY_SIZE = 100 * 1024 * 1024+# Maximum SSE buffer size (10MB - prevents memory exhaustion from malformed streams)+MAX_SSE_BUFFER_SIZE = 10 * 1024 * 1024++# Maximum message array length (prevents DoS from deeply nested payloads)+MAX_MESSAGE_ARRAY_LENGTH = 10000++# Maximum compression cache sessions (prevents unbounded memory growth)+MAX_COMPRESSION_CACHE_SESSIONS = 500++# Maximum rate limiter buckets (prevents DoS via spoofed API keys)+MAX_RATE_LIMITER_BUCKETS = 1000++# Compression pipeline timeout in seconds+COMPRESSION_TIMEOUT_SECONDS = 30+# =============================================================================# Data Models@@ -605,10 +620,10 @@ class ProxyConfig:bedrock_profile: str | None = None # AWS profile (optional)anyllm_provider: str = "openai" # any-llm provider (openai, mistral, groq, etc.)- # Optimization mode: "cost_savings" (default) or "token_headroom"- # cost_savings: preserve prefix cache for cost reduction+ # Optimization mode: "token_headroom" (default) or "cost_savings"# token_headroom: compress older messages for session extension- mode: str = "cost_savings"+ # cost_savings: preserve prefix cache for cost reduction+ mode: str = "token_headroom"# Optimizationoptimize: bool = True@@ -872,6 +887,19 @@ class TokenBucketRateLimiter:)self._lock = asyncio.Lock()+ async def _cleanup_stale_buckets(self) -> None:+ """Remove buckets that haven't been used in the last 10 minutes."""+ now = time.time()+ stale_threshold = now - 600 # 10 minutes+ stale_keys = [+ k for k, v in self._request_buckets.items() if v.last_update < stale_threshold+ ]+ for k in stale_keys:+ del self._request_buckets[k]+ self._token_buckets.pop(k, None)+ if stale_keys:+ logger.debug(f"Cleaned up {len(stale_keys)} stale rate limiter buckets")+def _refill(self, state: RateLimitState, rate_per_minute: float) -> float:"""Refill bucket based on elapsed time."""now = time.time()@@ -884,6 +912,9 @@ class TokenBucketRateLimiter:async def check_request(self, key: str = "default") -> tuple[bool, float]:"""Check if request is allowed. Returns (allowed, wait_seconds)."""async with self._lock:+ # Prevent unbounded bucket growth from spoofed keys+ if len(self._request_buckets) > MAX_RATE_LIMITER_BUCKETS:+ await self._cleanup_stale_buckets()state = self._request_buckets[key]available = self._refill(state, self.requests_per_minute)@@ -1747,6 +1778,20 @@ class HeadroomProxy:if session_id not in self._compression_caches:from headroom.cache.compression_cache import CompressionCache+ # Evict oldest caches if at capacity+ if len(self._compression_caches) >= MAX_COMPRESSION_CACHE_SESSIONS:+ # Remove oldest quarter to amortize cleanup cost+ oldest_keys = list(self._compression_caches.keys())[+ : MAX_COMPRESSION_CACHE_SESSIONS // 4+ ]+ for key in oldest_keys:+ del self._compression_caches[key]+ logger.info(+ "Evicted %d compression caches (exceeded %d max sessions)",+ len(oldest_keys),+ MAX_COMPRESSION_CACHE_SESSIONS,+ )+self._compression_caches[session_id] = CompressionCache()return self._compression_caches[session_id]@@ -1834,7 +1879,7 @@ class HeadroomProxy:logger.warning(f"Unknown HEADROOM_MODE '{self.config.mode}', falling back to 'cost_savings'")- self.config.mode = "cost_savings"+ self.config.mode = "token_headroom"logger.info(f"Mode: {self.config.mode}")if self.config.mode == "token_headroom":logger.info(" Prefix freeze: re-freeze after compression")@@ -2104,6 +2149,21 @@ class HeadroomProxy:)model = body.get("model", "unknown")messages = body.get("messages", [])++ # Validate message array size+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": f"Message array too large ({len(messages)} messages). "+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",+ },+ },+ )+stream = body.get("stream", False)# Image compression (before text optimization)@@ -2127,7 +2187,9 @@ class HeadroomProxy:# Rate limitingif self.rate_limiter:- rate_key = headers.get("x-api-key", "default")[:16]+ api_key = headers.get("x-api-key", "")+ client_ip = request.client.host if request.client else "unknown"+ rate_key = f"{api_key[:16]}:{client_ip}" if api_key else client_ipallowed, wait_seconds = await self.rate_limiter.check_request(rate_key)if not allowed:await self.metrics.record_rate_limited()@@ -2211,6 +2273,7 @@ class HeadroomProxy:prefix_tracker = self.session_tracker_store.get_or_create(session_id, "anthropic")frozen_message_count = prefix_tracker.get_frozen_message_count()+ _compression_failed = Falseif self.config.optimize and messages:try:context_limit = self.anthropic_provider.get_context_limit(model)@@ -2229,12 +2292,17 @@ class HeadroomProxy:# Re-freeze boundary: consecutive stable messages from startfrozen_message_count = comp_cache.compute_frozen_count(messages)- result = self.anthropic_pipeline.apply(- messages=working_messages,- model=model,- model_limit=context_limit,- frozen_message_count=frozen_message_count,- biases=biases,+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.anthropic_pipeline.apply(+ messages=working_messages,+ model=model,+ model_limit=context_limit,+ frozen_message_count=frozen_message_count,+ biases=biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,)# Cache newly compressed messages (index-aligned diff)@@ -2250,12 +2318,17 @@ class HeadroomProxy:# original_tokens was set at line ~2183 from uncompressed messages.optimized_tokens = result.tokens_afterelse:- result = self.anthropic_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- frozen_message_count=frozen_message_count,- biases=biases,+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.anthropic_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ frozen_message_count=frozen_message_count,+ biases=biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,)if result.messages != messages:@@ -2269,6 +2342,8 @@ class HeadroomProxy:waste_signals_dict = result.waste_signals.to_dict()except Exception as e:logger.warning(f"Optimization failed: {e}")+ # Flag compression failure for observability+ _compression_failed = Truetokens_saved = max(0, original_tokens - optimized_tokens)optimization_latency = (time.time() - start_time) * 1000@@ -2811,6 +2886,8 @@ class HeadroomProxy:response_headers["x-headroom-transforms"] = ",".join(transforms_applied)if cache_hit:response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"return Response(content=response.content,@@ -4234,7 +4311,7 @@ class HeadroomProxy:)async def generate():- nonlocal body # May need to modify for continuation requests+ nonlocal body, memory_enabled # May need to modify for continuation requests# For memory mode, we buffer the response to check for tool callsbuffered_chunks: list[bytes] = []@@ -4255,10 +4332,27 @@ class HeadroomProxy:chunk_str = chunk.decode("utf-8", errors="ignore")stream_state["sse_buffer"] += chunk_str+ # Safety: prevent unbounded buffer growth+ if len(stream_state["sse_buffer"]) > MAX_SSE_BUFFER_SIZE:+ logger.error(+ "SSE buffer exceeded maximum size (%d bytes), "+ "truncating to prevent memory exhaustion",+ MAX_SSE_BUFFER_SIZE,+ )+ stream_state["sse_buffer"] = stream_state["sse_buffer"][+ -MAX_SSE_BUFFER_SIZE // 2 :+ ]+if memory_enabled:# Buffer for memory tool detectionbuffered_chunks.append(chunk)full_sse_data += chunk_str+ if len(full_sse_data) > MAX_SSE_BUFFER_SIZE:+ logger.warning(+ "Memory-mode SSE buffer exceeded maximum size, "+ "disabling memory detection for this request"+ )+ memory_enabled = Falseelse:# Immediate streaming when memory not enabledyield chunk@@ -4696,6 +4790,21 @@ class HeadroomProxy:)model = body.get("model", "unknown")messages = body.get("messages", [])++ # Validate message array size+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Message array too large ({len(messages)} messages). "+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",+ "type": "invalid_request_error",+ "code": "invalid_request",+ }+ },+ )+stream = body.get("stream", False)# Image compression (before text optimization)@@ -4778,6 +4887,7 @@ class HeadroomProxy:)openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()+ _compression_failed = Falseif self.config.optimize and messages:try:context_limit = self.openai_provider.get_context_limit(model)@@ -4791,12 +4901,17 @@ class HeadroomProxy:# Re-freeze boundaryopenai_frozen_count = comp_cache.compute_frozen_count(messages)- result = self.openai_pipeline.apply(- messages=working_messages,- model=model,- model_limit=context_limit,- frozen_message_count=openai_frozen_count,- biases=_hook_biases,+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=working_messages,+ model=model,+ model_limit=context_limit,+ frozen_message_count=openai_frozen_count,+ biases=_hook_biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,)if result.messages != working_messages:@@ -4810,12 +4925,17 @@ class HeadroomProxy:# so tokens_saved captures both Zone 1 + Zone 2 savings.optimized_tokens = result.tokens_afterelse:- result = self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- frozen_message_count=openai_frozen_count,- biases=_hook_biases,+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ frozen_message_count=openai_frozen_count,+ biases=_hook_biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,)if result.messages != messages:@@ -4829,6 +4949,8 @@ class HeadroomProxy:waste_signals_dict = result.waste_signals.to_dict()except Exception as e:logger.warning(f"Optimization failed: {e}")+ # Flag compression failure for observability+ _compression_failed = Truetokens_saved = max(0, original_tokens - optimized_tokens)optimization_latency = (time.time() - start_time) * 1000@@ -5052,6 +5174,8 @@ class HeadroomProxy:response_headers["x-headroom-transforms"] = ",".join(transforms_applied)if cache_read_tokens > 0:response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"return Response(content=response.content,@@ -5941,6 +6065,7 @@ class HeadroomProxy:optimized_messages = messagesoptimized_tokens = original_tokens+ _compression_failed = Falseif self.config.optimize and messages:try:# Use OpenAI pipeline (similar message format)@@ -5959,6 +6084,7 @@ class HeadroomProxy:if result.waste_signals:waste_signals_dict = result.waste_signals.to_dict()except Exception as e:+ _compression_failed = Truelogger.warning(f"[{request_id}] Gemini optimization failed: {e}")tokens_saved = max(0, original_tokens - optimized_tokens)@@ -6074,6 +6200,8 @@ class HeadroomProxy:response_headers["x-headroom-transforms"] = ",".join(transforms_applied)if cache_read_tokens > 0:response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"return Response(content=response.content,@@ -6541,7 +6669,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:"total_tokens_saved": total_tokens_saved,}else:- compression_cache_stats = {"mode": "cost_savings"}+ compression_cache_stats = {"mode": "token_headroom"}# Build unified savings summary (all layers)compression_tokens = m.tokens_saved_total@@ -7846,7 +7974,7 @@ if __name__ == "__main__":max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),http2=not args.no_http2 and _get_env_bool("HEADROOM_HTTP2", True),tool_profiles=tool_profiles if tool_profiles else None,- mode=_get_env_str("HEADROOM_MODE", "cost_savings"),+ mode=_get_env_str("HEADROOM_MODE", "token_headroom"),)# Get worker and concurrency settings