Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Refactor: split server.py into handler mixins (Steps 5-9)

chopratejase8ab444f09e9
diff --git a/headroom/proxy/handlers/__init__.py b/headroom/proxy/handlers/__init__.py
new file mode 100644
index 000000000..d0d1c8ce7
--- /dev/null
+++ b/headroom/proxy/handlers/__init__.py
@@ -0,0 +1,20 @@
+"""Handler mixins for HeadroomProxy.
+
+Each mixin class contains methods extracted from HeadroomProxy that handle
+requests for a specific provider or concern. The mixins rely on HeadroomProxy's
+__init__ for all self.* attributes (duck typing).
+"""
+
+from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
+from headroom.proxy.handlers.batch import BatchHandlerMixin
+from headroom.proxy.handlers.gemini import GeminiHandlerMixin
+from headroom.proxy.handlers.openai import OpenAIHandlerMixin
+from headroom.proxy.handlers.streaming import StreamingMixin
+
+__all__ = [
+ "AnthropicHandlerMixin",
+ "BatchHandlerMixin",
+ "GeminiHandlerMixin",
+ "OpenAIHandlerMixin",
+ "StreamingMixin",
+]
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
new file mode 100644
index 000000000..848de1710
--- /dev/null
+++ b/headroom/proxy/handlers/anthropic.py
@@ -0,0 +1,1435 @@
+"""Anthropic handler mixin for HeadroomProxy.
+
+Contains all Anthropic Messages API handlers including batch operations.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from fastapi import Request
+ from fastapi.responses import Response, StreamingResponse
+
+import httpx
+
+logger = logging.getLogger("headroom.proxy")
+
+
+class AnthropicHandlerMixin:
+ """Mixin providing Anthropic API handler methods for HeadroomProxy."""
+
+ async def handle_anthropic_messages(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle Anthropic /v1/messages endpoint."""
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.cache.compression_store import get_compression_store
+ from headroom.ccr import CCRToolInjector
+ from headroom.proxy.cost import _summarize_transforms
+ from headroom.proxy.helpers import (
+ MAX_MESSAGE_ARRAY_LENGTH,
+ MAX_REQUEST_BODY_SIZE,
+ _get_image_compressor,
+ _read_request_json,
+ )
+ from headroom.proxy.models import RequestLog
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "type": "error",
+ "error": {
+ "type": "request_too_large",
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ },
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": f"Invalid request body: {e!s}",
+ },
+ },
+ )
+ model = body.get("model", "unknown")
+ messages = body.get("messages", [])
+
+ # Validate message array size
+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": f"Message array too large ({len(messages)} messages). "
+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
+ },
+ },
+ )
+
+ stream = body.get("stream", False)
+
+ # Bypass: skip ALL compression, TOIN learning, and CCR injection
+ # when the caller explicitly opts out via header.
+ # Prevents Headroom from corrupting sub-agent API calls
+ # (e.g., Claude Code sub-agents that inherit ANTHROPIC_BASE_URL).
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ if _bypass:
+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")
+
+ # Image compression (before text optimization)
+ if self.config.image_optimize and messages and not _bypass:
+ compressor = _get_image_compressor()
+ if compressor and compressor.has_images(messages):
+ messages = compressor.compress(messages, provider="anthropic")
+ if compressor.last_result:
+ logger.info(
+ f"Image compression: {compressor.last_result.technique.value} "
+ f"({compressor.last_result.savings_percent:.0f}% saved, "
+ f"{compressor.last_result.original_tokens} -> "
+ f"{compressor.last_result.compressed_tokens} tokens)"
+ )
+
+ # Extract headers and tags
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ tags = self._extract_tags(headers)
+
+ # Rate limiting
+ if self.rate_limiter:
+ api_key = headers.get("x-api-key", "")
+ client_ip = request.client.host if request.client else "unknown"
+ rate_key = f"{api_key[:16]}:{client_ip}" if api_key else client_ip
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited()
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ headers={"Retry-After": str(int(wait_seconds) + 1)},
+ )
+
+ # Budget check
+ if self.cost_tracker:
+ allowed, remaining = self.cost_tracker.check_budget()
+ if not allowed:
+ raise HTTPException(
+ status_code=429,
+ detail=f"Budget exceeded for {self.config.budget_period} period",
+ )
+
+ # Memory: Get user ID when memory is enabled (fallback to "default" for simple DevEx)
+ memory_user_id: str | None = None
+ if self.memory_handler:
+ memory_user_id = headers.get("x-headroom-user-id", "default")
+
+ # Check cache (non-streaming only)
+ cache_hit = False
+ if self.cache and not stream:
+ cached = await self.cache.get(messages, model)
+ if cached:
+ cache_hit = True
+ optimization_latency = (time.time() - start_time) * 1000
+
+ await self.metrics.record_request(
+ provider="anthropic",
+ model=model,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0, # Savings already counted when response was cached
+ latency_ms=optimization_latency,
+ cached=True,
+ )
+
+ # Remove compression headers from cached response
+ response_headers = dict(cached.response_headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=cached.response_body,
+ headers=response_headers,
+ media_type="application/json",
+ )
+
+ # Count original tokens
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Hook: pre_compress — let hooks modify messages before compression
+
+ if self.config.hooks:
+ from headroom.hooks import CompressContext
+
+ _hook_ctx = CompressContext(
+ model=model,
+ user_query=extract_user_query(messages),
+ provider="anthropic",
+ )
+ try:
+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)
+ except Exception as e:
+ logger.debug(f"[{request_id}] pre_compress hook error: {e}")
+
+ # Apply optimization
+ transforms_applied = []
+ pipeline_timing: dict[str, float] = {}
+ waste_signals_dict: dict[str, int] | None = None
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ # Get prefix cache tracker for this session
+ session_id = self.session_tracker_store.compute_session_id(request, model, messages)
+ prefix_tracker = self.session_tracker_store.get_or_create(session_id, "anthropic")
+ frozen_message_count = prefix_tracker.get_frozen_message_count()
+
+ _compression_failed = False
+ original_messages = messages # Preserve for 400-retry fallback
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+ if self.config.optimize and messages and not _bypass and _license_ok:
+ try:
+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
+
+ context_limit = self.anthropic_provider.get_context_limit(model)
+ biases = (
+ self.config.hooks.compute_biases(messages, _hook_ctx)
+ if self.config.hooks
+ else None
+ )
+
+ if self.config.mode == "token_headroom":
+ comp_cache = self._get_compression_cache(session_id)
+
+ # Zone 1: Swap cached compressed versions into working copy
+ working_messages = comp_cache.apply_cached(messages)
+
+ # Re-freeze boundary: consecutive stable messages from start
+ frozen_message_count = comp_cache.compute_frozen_count(messages)
+
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.anthropic_pipeline.apply(
+ messages=working_messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(working_messages),
+ frozen_message_count=frozen_message_count,
+ biases=biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ # Cache newly compressed messages (index-aligned diff)
+ if result.messages != working_messages:
+ comp_cache.update_from_result(messages, result.messages)
+
+ # Always use pipeline result — Zone 1 swaps are already applied
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)
+ # so tokens_saved captures both Zone 1 + Zone 2 savings.
+ # original_tokens was set at line ~2183 from uncompressed messages.
+ optimized_tokens = result.tokens_after
+ else:
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.anthropic_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ frozen_message_count=frozen_message_count,
+ biases=biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+
+ if result.waste_signals:
+ waste_signals_dict = result.waste_signals.to_dict()
+ except Exception as e:
+ logger.warning(f"Optimization failed: {e}")
+ # Flag compression failure for observability
+ _compression_failed = True
+
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Hook: post_compress — let hooks observe compression results
+ if self.config.hooks and tokens_saved > 0:
+ from headroom.hooks import CompressEvent
+
+ try:
+ self.config.hooks.post_compress(
+ CompressEvent(
+ tokens_before=original_tokens,
+ tokens_after=optimized_tokens,
+ tokens_saved=tokens_saved,
+ compression_ratio=tokens_saved / original_tokens
+ if original_tokens > 0
+ else 0,
+ transforms_applied=transforms_applied,
+ model=model,
+ user_query=_hook_ctx.user_query if self.config.hooks else "",
+ provider="anthropic",
+ )
+ )
+ except Exception as e:
+ logger.debug(f"[{request_id}] post_compress hook error: {e}")
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = body.get("tools")
+ _original_tools = tools # Preserve for diagnostic / future retry
+ if (
+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
+ ) and not _bypass:
+ # Create fresh injector to avoid state leakage between requests
+ injector = CCRToolInjector(
+ provider="anthropic",
+ inject_tool=self.config.ccr_inject_tool,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ optimized_messages, tools, was_injected = injector.process_request(
+ optimized_messages, tools
+ )
+
+ if injector.has_compressed_content:
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
+ )
+ else:
+ logger.debug(
+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
+ )
+
+ # Track compression in context tracker for multi-turn awareness
+ if self.ccr_context_tracker:
+ self._turn_counter += 1
+ for hash_key in injector.detected_hashes:
+ # Get compression metadata from store
+ store = get_compression_store()
+ entry = store.get_metadata(hash_key)
+ if entry:
+ self.ccr_context_tracker.track_compression(
+ hash_key=hash_key,
+ turn_number=self._turn_counter,
+ tool_name=entry.get("tool_name"),
+ original_count=entry.get("original_item_count", 0),
+ compressed_count=entry.get("compressed_item_count", 0),
+ query_context=entry.get("query_context", ""),
+ sample_content=entry.get("compressed_content", "")[:500],
+ )
+
+ # CCR Proactive Expansion: Check if current query needs expanded context
+ if self.ccr_context_tracker and self.config.ccr_proactive_expansion:
+ # Extract user query from messages
+ user_query = ""
+ for msg in reversed(messages):
+ if msg.get("role") == "user":
+ content = msg.get("content", "")
+ if isinstance(content, str):
+ user_query = content
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ user_query = block.get("text", "")
+ break
+ break
+
+ if user_query:
+ recommendations = self.ccr_context_tracker.analyze_query(
+ user_query, self._turn_counter
+ )
+ if recommendations:
+ expansions = self.ccr_context_tracker.execute_expansions(recommendations)
+ if expansions:
+ # Add expanded context to the system message or as additional context
+ expansion_text = self.ccr_context_tracker.format_expansions_for_context(
+ expansions
+ )
+ logger.info(
+ f"[{request_id}] CCR: Proactively expanded {len(expansions)} context(s) "
+ f"based on query relevance"
+ )
+ # Append to the last user message
+ if optimized_messages and optimized_messages[-1].get("role") == "user":
+ last_msg = optimized_messages[-1]
+ content = last_msg.get("content", "")
+ if isinstance(content, str):
+ optimized_messages[-1] = {
+ **last_msg,
+ "content": content + "\n\n" + expansion_text,
+ }
+
+ # Traffic Learner: Extract patterns from inbound tool results
+ if self.traffic_learner:
+ try:
+ # Wire backend on first use (lazy init after memory handler is ready)
+ if (
+ self.traffic_learner._backend is None
+ and self.memory_handler
+ and self.memory_handler.initialized
+ and self.memory_handler.backend
+ ):
+ self.traffic_learner.set_backend(self.memory_handler.backend)
+
+ # Extract tool results from messages and learn from them
+ tool_results = self.traffic_learner.extract_tool_results_from_messages(
+ optimized_messages
+ )
+ for tr in tool_results[-5:]: # Only recent results
+ await self.traffic_learner.on_tool_result(
+ tool_name=tr["tool_name"],
+ tool_input=tr["input"],
+ tool_output=tr["output"],
+ is_error=tr["is_error"],
+ )
+
+ # Also extract preference signals from user messages
+ await self.traffic_learner.on_messages(optimized_messages)
+ except Exception as e:
+ logger.debug(f"[{request_id}] Traffic learner: {e}")
+
+ # Memory: Inject context and tools
+ if self.memory_handler and memory_user_id:
+ # Search and inject memory context
+ if self.memory_handler.config.inject_context:
+ try:
+ memory_context = await self.memory_handler.search_and_format_context(
+ memory_user_id, optimized_messages
+ )
+ if memory_context:
+ optimized_messages = self._inject_system_context(
+ optimized_messages, memory_context, body=body
+ )
+ logger.info(
+ f"[{request_id}] Memory: Injected {len(memory_context)} chars of context"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory: Context injection failed: {e}")
+
+ # Inject memory tools
+ if self.memory_handler.config.inject_tools:
+ tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "anthropic")
+ if mem_tools_injected:
+ tool_names = [
+ t.get("name") or t.get("type", "")
+ for t in tools
+ if t.get("name", "").startswith("memory")
+ or t.get("type", "").startswith("memory")
+ ]
+ logger.info(f"[{request_id}] Memory: Injected tools: {tool_names}")
+
+ # Add beta headers for native memory tool
+ beta_headers = self.memory_handler.get_beta_headers()
+ if beta_headers:
+ for key, value in beta_headers.items():
+ # Merge with existing beta header if present
+ existing = headers.get(key, "")
+ if existing and value not in existing:
+ headers[key] = f"{existing},{value}"
+ else:
+ headers[key] = value
+ logger.info(
+ f"[{request_id}] Memory: Added beta header: {key}={headers[key]}"
+ )
+
+ # Query Echo: disabled — hurts prefix caching in long conversations.
+ # The echo changes every turn, invalidating the cached prefix.
+ # To re-enable, uncomment and set query_echo_enabled on ProxyConfig.
+
+ # Update body
+ body["messages"] = optimized_messages
+ if tools is not None:
+ body["tools"] = tools
+
+ # Forward request - use Bedrock backend if configured, otherwise direct API
+ if self.anthropic_backend is not None:
+ # Route through Bedrock backend
+ try:
+ if stream:
+ return await self._stream_response_bedrock(
+ body,
+ headers,
+ "anthropic",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+ else:
+ backend_response = await self.anthropic_backend.send_message(body, headers)
+
+ if backend_response.error:
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+
+ # Track metrics
+ total_latency = (time.time() - start_time) * 1000
+ usage = backend_response.body.get("usage", {})
+ output_tokens = usage.get("output_tokens", 0)
+
+ _backend_name = (
+ self.anthropic_backend.name if self.anthropic_backend else "anthropic"
+ )
+ await self.metrics.record_request(
+ provider=_backend_name,
+ model=model,
+ input_tokens=optimized_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ cached=False,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
+
+ # Log request
+ if self.logger:
+ self.logger.log(
+ RequestLog(
+ request_id=request_id,
+ timestamp=datetime.now().isoformat(),
+ provider=_backend_name,
+ model=model,
+ input_tokens_original=original_tokens,
+ input_tokens_optimized=optimized_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ savings_percent=(tokens_saved / original_tokens * 100)
+ if original_tokens > 0
+ else 0,
+ optimization_latency_ms=optimization_latency,
+ total_latency_ms=total_latency,
+ tags=tags,
+ cache_hit=False,
+ transforms_applied=transforms_applied,
+ request_messages=body.get("messages")
+ if self.config.log_full_messages
+ else None,
+ )
+ )
+
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+ except Exception as e:
+ logger.error(f"[{request_id}] Bedrock backend error: {e}")
+ return JSONResponse(
+ status_code=500,
+ content={
+ "type": "error",
+ "error": {"type": "api_error", "message": str(e)},
+ },
+ )
+
+ # Direct Anthropic API
+ url = f"{self.ANTHROPIC_API_URL}/v1/messages"
+
+ try:
+ if stream:
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "anthropic",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ memory_user_id=memory_user_id,
+ pipeline_timing=pipeline_timing,
+ prefix_tracker=prefix_tracker,
+ )
+ else:
+ response = await self._retry_request("POST", url, headers, body)
+
+ # Full diagnostic dump on upstream errors.
+ # Writes pre/post compression messages, tools, and error
+ # to ~/.headroom/logs/debug_400/ for offline analysis.
+ if response.status_code >= 400:
+ try:
+ err_body = response.json()
+ err_msg = err_body.get("error", {}).get("message", "")
+ err_type = err_body.get("error", {}).get("type", "")
+ except Exception:
+ err_body = {"raw": response.text[:2000]}
+ err_msg = str(response.text[:500])
+ err_type = "parse_error"
+
+ logger.warning(
+ f"[{request_id}] UPSTREAM_ERROR "
+ f"status={response.status_code} "
+ f"error_type={err_type} "
+ f"error_msg={err_msg!r} "
+ f"model={model} "
+ f"compressed={'yes' if transforms_applied else 'no'} "
+ f"transforms={transforms_applied} "
+ f"original_tokens={original_tokens} "
+ f"optimized_tokens={optimized_tokens} "
+ f"message_count={len(body.get('messages', []))} "
+ f"stream={stream}"
+ )
+
+ # Dump full request details to debug file
+ try:
+ debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"
+ debug_dir.mkdir(parents=True, exist_ok=True)
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+ debug_file = debug_dir / f"{ts}_{request_id}.json"
+
+ # Sanitize headers (redact API keys)
+ safe_headers = {}
+ for k, v in headers.items():
+ if k.lower() in ("x-api-key", "authorization"):
+ safe_headers[k] = v[:12] + "..." if v else ""
+ else:
+ safe_headers[k] = v
+
+ debug_payload = {
+ "request_id": request_id,
+ "timestamp": datetime.now().isoformat(),
+ "status_code": response.status_code,
+ "error_response": err_body,
+ "model": model,
+ "stream": stream,
+ "headers": safe_headers,
+ "compression": {
+ "was_compressed": bool(transforms_applied),
+ "transforms": transforms_applied,
+ "original_tokens": original_tokens,
+ "optimized_tokens": optimized_tokens,
+ "tokens_saved": tokens_saved,
+ "compression_failed": _compression_failed,
+ },
+ "tools_sent": body.get("tools"),
+ "tool_count": len(body.get("tools") or []),
+ "original_tool_count": len(_original_tools or []),
+ "messages_sent": body.get("messages"),
+ "message_count": len(body.get("messages", [])),
+ "original_messages": (
+ original_messages
+ if original_messages is not body.get("messages")
+ else "__same_as_sent__"
+ ),
+ "original_message_count": len(original_messages),
+ "system_prompt": body.get("system"),
+ }
+
+ with open(debug_file, "w") as f:
+ json.dump(debug_payload, f, indent=2, default=str)
+
+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
+ except Exception as dump_err:
+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
+
+ # Parse response for CCR handling
+ resp_json = None
+ try:
+ resp_json = response.json()
+ except (json.JSONDecodeError, ValueError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to parse response JSON for CCR handling: {e}"
+ )
+
+ # CCR Response Handling: Handle headroom_retrieve tool calls automatically
+ if (
+ self.ccr_response_handler
+ and resp_json
+ and response.status_code == 200
+ and self.ccr_response_handler.has_ccr_tool_calls(resp_json, "anthropic")
+ ):
+ logger.info(f"[{request_id}] CCR: Detected retrieval tool call, handling...")
+
+ # Create API call function for continuation
+ # Use a fresh client to avoid potential decompression state issues
+ async def api_call_fn(
+ msgs: list[dict], tls: list[dict] | None
+ ) -> dict[str, Any]:
+ continuation_body = {
+ **body,
+ "messages": msgs,
+ }
+ if tls is not None:
+ continuation_body["tools"] = tls
+
+ # Use clean headers for continuation
+ continuation_headers = {
+ k: v
+ for k, v in headers.items()
+ if k.lower()
+ not in (
+ "content-encoding",
+ "transfer-encoding",
+ "accept-encoding",
+ "content-length",
+ )
+ }
+
+ # Reuse main client for CCR continuations (connection pooling)
+ logger.info(f"CCR: Making continuation request with {len(msgs)} messages")
+ assert self.http_client is not None, "HTTP client not initialized"
+ try:
+ cont_response = await self.http_client.post(
+ url,
+ json=continuation_body,
+ headers=continuation_headers,
+ timeout=httpx.Timeout(120.0), # Override timeout for CCR
+ )
+ logger.info(
+ f"CCR: Got response status={cont_response.status_code}, "
+ f"content-encoding={cont_response.headers.get('content-encoding')}"
+ )
+ result: dict[str, Any] = cont_response.json()
+ logger.info("CCR: Parsed JSON successfully")
+ return result
+ except Exception as e:
+ resp_headers: str | dict[str, str] = "N/A"
+ try:
+ resp_headers = dict(cont_response.headers)
+ except Exception:
+ pass
+ logger.error(
+ f"CCR: API call failed: {e}, response headers: {resp_headers}"
+ )
+ raise
+
+ # Handle CCR tool calls
+ try:
+ final_resp_json = await self.ccr_response_handler.handle_response(
+ resp_json,
+ optimized_messages,
+ tools,
+ api_call_fn,
+ provider="anthropic",
+ )
+ # Update response content with final response
+ resp_json = final_resp_json
+ # Remove encoding headers since content is now uncompressed JSON
+ ccr_response_headers = {
+ k: v
+ for k, v in response.headers.items()
+ if k.lower() not in ("content-encoding", "content-length")
+ }
+ try:
+ ccr_content = json.dumps(final_resp_json).encode()
+ except (TypeError, ValueError) as json_err:
+ logger.warning(
+ f"[{request_id}] CCR: JSON serialization failed: {json_err}"
+ )
+ ccr_content = json.dumps(resp_json).encode()
+ response = httpx.Response(
+ status_code=200,
+ content=ccr_content,
+ headers=ccr_response_headers,
+ )
+ logger.info(f"[{request_id}] CCR: Retrieval handled successfully")
+ except Exception as e:
+ import traceback
+
+ logger.warning(
+ f"[{request_id}] CCR: Response handling failed: {e}\n"
+ f"Traceback: {traceback.format_exc()}"
+ )
+ # Continue with original response
+
+ # Memory: Handle memory tool calls in response
+ if (
+ self.memory_handler
+ and memory_user_id
+ and resp_json
+ and response.status_code == 200
+ and self.memory_handler.has_memory_tool_calls(resp_json, "anthropic")
+ ):
+ logger.info(f"[{request_id}] Memory: Detected memory tool call, handling...")
+
+ try:
+ # Execute memory tool calls
+ tool_results = await self.memory_handler.handle_memory_tool_calls(
+ resp_json, memory_user_id, "anthropic"
+ )
+
+ if tool_results:
+ # Create continuation messages
+ assistant_msg = {
+ "role": "assistant",
+ "content": resp_json.get("content", []),
+ }
+ user_msg = {
+ "role": "user",
+ "content": tool_results,
+ }
+
+ continuation_messages = optimized_messages + [assistant_msg, user_msg]
+
+ # Make continuation API call
+ continuation_body = {**body, "messages": continuation_messages}
+ if tools:
+ continuation_body["tools"] = tools
+
+ cont_response = await self._retry_request(
+ "POST", url, headers, continuation_body
+ )
+
+ # Update response with continuation
+ resp_json = cont_response.json()
+ response = cont_response
+ logger.info(
+ f"[{request_id}] Memory: Tool calls handled, continuation complete"
+ )
+
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory: Tool call handling failed: {e}")
+ # Continue with original response
+
+ total_latency = (time.time() - start_time) * 1000
+
+ # Parse response for output token count and cache metrics
+ output_tokens = 0
+ cr_tokens = 0
+ cw_tokens = 0
+ uncached_input_tokens = 0
+ if resp_json:
+ usage = resp_json.get("usage", {})
+ output_tokens = usage.get("output_tokens", 0)
+ cr_tokens = usage.get("cache_read_input_tokens", 0)
+ cw_tokens = usage.get("cache_creation_input_tokens", 0)
+ uncached_input_tokens = usage.get("input_tokens", 0)
+
+ # Update prefix cache tracker for next turn
+ prefix_tracker.update_from_response(
+ cache_read_tokens=cr_tokens,
+ cache_write_tokens=cw_tokens,
+ messages=optimized_messages,
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(
+ model,
+ tokens_saved,
+ optimized_tokens,
+ cache_read_tokens=cr_tokens,
+ cache_write_tokens=cw_tokens,
+ uncached_tokens=uncached_input_tokens,
+ )
+
+ # Cache response
+ if self.cache and response.status_code == 200:
+ await self.cache.set(
+ messages,
+ model,
+ response.content,
+ dict(response.headers),
+ tokens_saved=tokens_saved,
+ )
+
+ # Record metrics — use optimized_tokens (what we sent), not API's
+ # input_tokens which is just the non-cached portion with prompt caching
+ await self.metrics.record_request(
+ provider="anthropic",
+ model=model,
+ input_tokens=optimized_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ waste_signals=waste_signals_dict,
+ cache_read_tokens=cr_tokens,
+ cache_write_tokens=cw_tokens,
+ uncached_input_tokens=uncached_input_tokens,
+ )
+
+ # Log request
+ if self.logger:
+ self.logger.log(
+ RequestLog(
+ request_id=request_id,
+ timestamp=datetime.now().isoformat(),
+ provider="anthropic",
+ model=model,
+ input_tokens_original=original_tokens,
+ input_tokens_optimized=optimized_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ savings_percent=(tokens_saved / original_tokens * 100)
+ if original_tokens > 0
+ else 0,
+ optimization_latency_ms=optimization_latency,
+ total_latency_ms=total_latency,
+ tags=tags,
+ cache_hit=cache_hit,
+ transforms_applied=transforms_applied,
+ waste_signals=waste_signals_dict,
+ request_messages=messages if self.config.log_full_messages else None,
+ )
+ )
+
+ # Structured perf log line for `headroom perf` analysis
+ num_msgs = len(messages)
+ resp_usage = resp_json.get("usage", {}) if resp_json else {}
+ cr = resp_usage.get("cache_read_input_tokens", 0)
+ cw = resp_usage.get("cache_creation_input_tokens", 0)
+ chp = round(cr / (cr + cw) * 100) if (cr + cw) > 0 else 0
+ timing_str = (
+ " ".join(f"{k}={v:.0f}ms" for k, v in pipeline_timing.items())
+ if pipeline_timing
+ else ""
+ )
+ logger.info(
+ f"[{request_id}] PERF "
+ f"model={model} msgs={num_msgs} "
+ f"tok_before={original_tokens} tok_after={optimized_tokens} "
+ f"tok_saved={tokens_saved} "
+ f"cache_read={cr} cache_write={cw} cache_hit_pct={chp} "
+ f"opt_ms={optimization_latency:.0f} "
+ f"transforms={_summarize_transforms(transforms_applied)}"
+ f"{' timing=' + timing_str if timing_str else ''}"
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Inject Headroom compression metrics (for SaaS metering)
+ response_headers["x-headroom-tokens-before"] = str(original_tokens)
+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
+ response_headers["x-headroom-model"] = model
+ if transforms_applied:
+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
+ if cache_hit:
+ response_headers["x-headroom-cached"] = "true"
+ if _compression_failed:
+ response_headers["x-headroom-compression-failed"] = "true"
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ except Exception as e:
+ await self.metrics.record_failed()
+ # Log full error details internally for debugging
+ logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")
+
+ # Try fallback if enabled
+ if self.config.fallback_enabled and self.config.fallback_provider == "openai":
+ logger.info(f"[{request_id}] Attempting fallback to OpenAI")
+ # Convert to OpenAI format and retry
+ # (simplified - would need message format conversion)
+
+ # Return sanitized error message to client (don't expose internal details)
+ return JSONResponse(
+ status_code=502,
+ content={
+ "type": "error",
+ "error": {
+ "type": "api_error",
+ "message": "An error occurred while processing your request. Please try again.",
+ },
+ },
+ )
+
+ async def handle_anthropic_batch_create(
+ self,
+ request: Request,
+ ) -> Response:
+ """Handle Anthropic POST /v1/messages/batches endpoint with compression.
+
+ Anthropic batch format:
+ {
+ "requests": [
+ {
+ "custom_id": "req-1",
+ "params": {
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 1024,
+ "messages": [{"role": "user", "content": "Hello"}]
+ }
+ },
+ ...
+ ]
+ }
+
+ This method applies compression to each request's messages before forwarding.
+ """
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.ccr import CCRToolInjector
+ from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "type": "error",
+ "error": {
+ "type": "request_too_large",
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ },
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": f"Invalid request body: {e!s}",
+ },
+ },
+ )
+
+ requests_list = body.get("requests", [])
+ if not requests_list:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": "Missing or empty 'requests' field in batch request",
+ },
+ },
+ )
+
+ # Extract headers
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+
+ # Track compression stats across all batch requests
+ total_original_tokens = 0
+ total_optimized_tokens = 0
+ total_tokens_saved = 0
+ compressed_requests = []
+ pipeline_timing: dict[str, float] = {}
+
+ # Apply compression to each request in the batch
+ for batch_req in requests_list:
+ custom_id = batch_req.get("custom_id", "")
+ params = batch_req.get("params", {})
+ messages = params.get("messages", [])
+ model = params.get("model", "unknown")
+
+ if not messages or not self.config.optimize:
+ # No messages or optimization disabled - pass through unchanged
+ compressed_requests.append(batch_req)
+ continue
+
+ # Apply optimization
+ try:
+ context_limit = self.anthropic_provider.get_context_limit(model)
+ result = self.anthropic_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+
+ optimized_messages = result.messages
+ for k, v in result.timing.items():
+ pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v
+ # Use pipeline's token counts for consistency with pipeline logs
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+ total_original_tokens += original_tokens
+ total_optimized_tokens += optimized_tokens
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+ total_tokens_saved += tokens_saved
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = params.get("tools")
+ if self.config.ccr_inject_tool and tokens_saved > 0:
+ injector = CCRToolInjector(
+ provider="anthropic",
+ inject_tool=True,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ optimized_messages, tools, was_injected = injector.process_request(
+ optimized_messages, tools
+ )
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for batch request '{custom_id}'"
+ )
+
+ # Create compressed batch request
+ compressed_params = {**params, "messages": optimized_messages}
+ if tools is not None:
+ compressed_params["tools"] = tools
+ compressed_requests.append(
+ {
+ "custom_id": custom_id,
+ "params": compressed_params,
+ }
+ )
+
+ if tokens_saved > 0:
+ logger.debug(
+ f"[{request_id}] Batch request '{custom_id}': "
+ f"{original_tokens:,} -> {optimized_tokens:,} tokens "
+ f"(saved {tokens_saved:,})"
+ )
+
+ except Exception as e:
+ logger.warning(
+ f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}"
+ )
+ # Pass through unchanged on failure
+ compressed_requests.append(batch_req)
+ total_optimized_tokens += original_tokens
+
+ # Update body with compressed requests
+ body["requests"] = compressed_requests
+
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Forward request to Anthropic
+ url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches"
+
+ try:
+ response = await self._retry_request("POST", url, headers, body)
+
+ # Record metrics
+ await self.metrics.record_request(
+ provider="anthropic",
+ model="batch",
+ input_tokens=total_optimized_tokens,
+ output_tokens=0,
+ tokens_saved=total_tokens_saved,
+ latency_ms=optimization_latency,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+
+ # Log compression stats
+ if total_tokens_saved > 0:
+ savings_percent = (
+ (total_tokens_saved / total_original_tokens * 100)
+ if total_original_tokens > 0
+ else 0
+ )
+ logger.info(
+ f"[{request_id}] Batch ({len(compressed_requests)} requests): "
+ f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "
+ f"(saved {total_tokens_saved:,}, {savings_percent:.1f}%)"
+ )
+
+ # Store batch context for CCR result processing
+ if response.status_code == 200 and self.config.ccr_inject_tool:
+ try:
+ response_data = response.json()
+ batch_id = response_data.get("id")
+ if batch_id:
+ await self._store_anthropic_batch_context(
+ batch_id,
+ requests_list,
+ headers.get("x-api-key"),
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Failed to store batch context: {e}")
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ except Exception as e:
+ await self.metrics.record_failed()
+ logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "type": "error",
+ "error": {
+ "type": "api_error",
+ "message": "An error occurred while processing your batch request. Please try again.",
+ },
+ },
+ )
+
+ async def handle_anthropic_batch_passthrough(
+ self,
+ request: Request,
+ batch_id: str | None = None,
+ ) -> Response:
+ """Handle Anthropic batch passthrough endpoints.
+
+ Used for:
+ - GET /v1/messages/batches - List batches
+ - GET /v1/messages/batches/{batch_id} - Get batch
+ - GET /v1/messages/batches/{batch_id}/results - Get batch results
+ - POST /v1/messages/batches/{batch_id}/cancel - Cancel batch
+ """
+ from fastapi.responses import Response
+
+ start_time = time.time()
+ path = request.url.path
+ url = f"{self.ANTHROPIC_API_URL}{path}"
+
+ # Preserve query string parameters (e.g., limit, after_id for list endpoint)
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+
+ body = await request.body()
+
+ response = await self.http_client.request( # type: ignore[union-attr]
+ method=request.method,
+ url=url,
+ headers=headers,
+ content=body,
+ )
+
+ # Track metrics
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider="anthropic",
+ model="passthrough:batches",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ )
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def _store_anthropic_batch_context(
+ self,
+ batch_id: str,
+ requests_list: list[dict[str, Any]],
+ api_key: str | None,
+ ) -> None:
+ """Store batch context for CCR result processing.
+
+ Args:
+ batch_id: The batch ID from the API response.
+ requests_list: The original batch requests.
+ api_key: The API key for continuation calls.
+ """
+ from headroom.ccr import BatchContext, BatchRequestContext, get_batch_context_store
+
+ store = get_batch_context_store()
+ context = BatchContext(
+ batch_id=batch_id,
+ provider="anthropic",
+ api_key=api_key,
+ api_base_url=self.ANTHROPIC_API_URL,
+ )
+
+ for batch_req in requests_list:
+ custom_id = batch_req.get("custom_id", "")
+ params = batch_req.get("params", {})
+ context.add_request(
+ BatchRequestContext(
+ custom_id=custom_id,
+ messages=params.get("messages", []),
+ tools=params.get("tools"),
+ model=params.get("model", ""),
+ extras={
+ "max_tokens": params.get("max_tokens", 4096),
+ "system": params.get("system"),
+ },
+ )
+ )
+
+ await store.store(context)
+ logger.debug(f"Stored batch context for {batch_id} with {len(requests_list)} requests")
+
+ async def handle_anthropic_batch_results(
+ self,
+ request: Request,
+ batch_id: str,
+ ) -> Response:
+ """Handle Anthropic batch results with CCR post-processing.
+
+ This endpoint:
+ 1. Fetches raw results from Anthropic
+ 2. Detects CCR tool calls in each result
+ 3. Executes retrieval and makes continuation calls
+ 4. Returns processed results with complete responses
+ """
+ from fastapi.responses import Response
+
+ from headroom.ccr import BatchResultProcessor, get_batch_context_store
+
+ start_time = time.time()
+
+ # Forward request to get raw results
+ url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches/{batch_id}/results"
+
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+
+ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]
+
+ if response.status_code != 200:
+ # Error - pass through
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Parse results - Anthropic batch results are JSONL format
+ raw_content = response.content.decode("utf-8")
+ results = []
+ for line in raw_content.strip().split("\n"):
+ if line.strip():
+ try:
+ results.append(json.loads(line))
+ except json.JSONDecodeError:
+ continue
+
+ if not results:
+ # No results to process
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Check if we have context and CCR processing is enabled
+ store = get_batch_context_store()
+ batch_context = await store.get(batch_id)
+
+ if batch_context is None or not self.config.ccr_inject_tool:
+ # No context or CCR disabled - pass through
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Process results with CCR handler
+ processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]
+ processed = await processor.process_results(batch_id, results, "anthropic")
+
+ # Convert back to JSONL format
+ processed_lines = []
+ for p in processed:
+ processed_lines.append(json.dumps(p.result))
+ if p.was_processed:
+ logger.info(
+ f"CCR: Processed batch result {p.custom_id} "
+ f"({p.continuation_rounds} continuation rounds)"
+ )
+
+ processed_content = "\n".join(processed_lines)
+
+ # Track metrics
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider="anthropic",
+ model="batch:ccr-processed",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ )
+
+ return Response(
+ content=processed_content.encode("utf-8"),
+ status_code=200,
+ media_type="application/jsonl",
+ )
diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.py
new file mode 100644
index 000000000..27276c351
--- /dev/null
+++ b/headroom/proxy/handlers/batch.py
@@ -0,0 +1,995 @@
+"""Batch handler mixin for HeadroomProxy.
+
+Contains all batch API handlers for Google and OpenAI batch operations.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from fastapi import Request
+ from fastapi.responses import Response
+
+logger = logging.getLogger("headroom.proxy")
+
+
+class BatchHandlerMixin:
+ """Mixin providing batch API handler methods for HeadroomProxy."""
+
+ async def handle_google_batch_create(
+ self,
+ request: Request,
+ model: str,
+ ) -> Response:
+ """Handle Google POST /v1beta/models/{model}:batchGenerateContent endpoint.
+
+ Google batch format:
+ {
+ "batch": {
+ "display_name": "my-batch",
+ "input_config": {
+ "requests": {
+ "requests": [
+ {
+ "request": {"contents": [{"parts": [{"text": "..."}]}]},
+ "metadata": {"key": "request-1"}
+ }
+ ]
+ }
+ }
+ }
+ }
+
+ This method applies compression to each request's contents before forwarding.
+ """
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.ccr import CCRToolInjector
+ from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "code": 413,
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "status": "INVALID_ARGUMENT",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "code": 400,
+ "message": f"Invalid request body: {e!s}",
+ "status": "INVALID_ARGUMENT",
+ }
+ },
+ )
+
+ # Extract batch config
+ batch_config = body.get("batch", {})
+ input_config = batch_config.get("input_config", {})
+ requests_wrapper = input_config.get("requests", {})
+ requests_list = requests_wrapper.get("requests", [])
+
+ if not requests_list:
+ # No inline requests - might be using file input, pass through
+ logger.debug(f"[{request_id}] Google batch: No inline requests, passing through")
+ return await self._google_batch_passthrough(request, model, body)
+
+ # Extract headers
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+
+ # Track compression stats
+ total_original_tokens = 0
+ total_optimized_tokens = 0
+ total_tokens_saved = 0
+ compressed_requests = []
+ pipeline_timing: dict[str, float] = {}
+
+ # Apply compression to each request in the batch
+ for idx, batch_req in enumerate(requests_list):
+ req_content = batch_req.get("request", {})
+ metadata = batch_req.get("metadata", {})
+ contents = req_content.get("contents", [])
+
+ if not contents or not self.config.optimize:
+ # No contents or optimization disabled - pass through unchanged
+ compressed_requests.append(batch_req)
+ continue
+
+ # Convert Google format to messages for compression
+ system_instruction = req_content.get("systemInstruction")
+ messages, preserved_indices = self._gemini_contents_to_messages(
+ contents, system_instruction
+ )
+
+ # Store original content entries that have non-text parts before compression
+ preserved_contents = {idx: contents[idx] for idx in preserved_indices}
+
+ # Early exit if ALL content has non-text parts (nothing to compress)
+ if len(preserved_indices) == len(contents):
+ # All content has non-text parts, skip compression
+ compressed_requests.append(batch_req)
+ continue
+
+ # Apply optimization
+ try:
+ # Default context limit for most models
+ context_limit = 128000
+
+ # Use OpenAI pipeline (similar message format after conversion)
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+
+ optimized_messages = result.messages
+ for k, v in result.timing.items():
+ pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v
+ # Use pipeline's token counts for consistency with pipeline logs
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+ total_original_tokens += original_tokens
+ total_optimized_tokens += optimized_tokens
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+ total_tokens_saved += tokens_saved
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = req_content.get("tools")
+ # Extract existing function declarations if present
+ existing_funcs = None
+ if tools:
+ for tool in tools:
+ if "functionDeclarations" in tool:
+ existing_funcs = tool["functionDeclarations"]
+ break
+
+ if self.config.ccr_inject_tool and tokens_saved > 0:
+ injector = CCRToolInjector(
+ provider="google",
+ inject_tool=True,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ optimized_messages, injected_funcs, was_injected = injector.process_request(
+ optimized_messages, existing_funcs
+ )
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for Google batch request {idx}"
+ )
+ existing_funcs = injected_funcs
+
+ # Convert back to Google contents format
+ optimized_contents, optimized_sys_inst = self._messages_to_gemini_contents(
+ optimized_messages
+ )
+
+ # Restore preserved content entries that had non-text parts
+ for orig_idx, original_content in preserved_contents.items():
+ if orig_idx < len(optimized_contents):
+ optimized_contents[orig_idx] = original_content
+
+ # Create compressed batch request
+ compressed_req_content = {**req_content, "contents": optimized_contents}
+ if optimized_sys_inst:
+ compressed_req_content["systemInstruction"] = optimized_sys_inst
+ if existing_funcs is not None:
+ compressed_req_content["tools"] = [{"functionDeclarations": existing_funcs}]
+
+ compressed_req = {
+ "request": compressed_req_content,
+ "metadata": metadata,
+ }
+
+ compressed_requests.append(compressed_req)
+
+ if tokens_saved > 0:
+ logger.debug(
+ f"[{request_id}] Google batch request {idx}: "
+ f"{original_tokens:,} -> {optimized_tokens:,} tokens "
+ f"(saved {tokens_saved:,})"
+ )
+
+ except Exception as e:
+ logger.warning(
+ f"[{request_id}] Optimization failed for Google batch request {idx}: {e}"
+ )
+ # Pass through unchanged on failure
+ compressed_requests.append(batch_req)
+ total_optimized_tokens += original_tokens
+
+ # Update body with compressed requests
+ body["batch"]["input_config"]["requests"]["requests"] = compressed_requests
+
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Forward request to Google
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"
+
+ # Add API key to URL if present in headers
+ api_key = headers.pop("x-goog-api-key", None)
+ if api_key:
+ url = f"{url}?key={api_key}"
+
+ try:
+ response = await self._retry_request("POST", url, headers, body)
+
+ # Record metrics
+ await self.metrics.record_request(
+ provider="google",
+ model=f"batch:{model}",
+ input_tokens=total_optimized_tokens,
+ output_tokens=0,
+ tokens_saved=total_tokens_saved,
+ latency_ms=optimization_latency,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+
+ # Log compression stats
+ if total_tokens_saved > 0:
+ savings_percent = (
+ (total_tokens_saved / total_original_tokens * 100)
+ if total_original_tokens > 0
+ else 0
+ )
+ logger.info(
+ f"[{request_id}] Google batch compression: "
+ f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "
+ f"({savings_percent:.1f}% saved across {len(requests_list)} requests)"
+ )
+
+ # Store batch context for CCR result processing
+ if response.status_code == 200 and self.config.ccr_inject_tool:
+ try:
+ response_data = response.json()
+ batch_name = response_data.get("name")
+ if batch_name:
+ await self._store_google_batch_context(
+ batch_name,
+ requests_list,
+ model,
+ api_key,
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Failed to store Google batch context: {e}")
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ except Exception as e:
+ logger.error(f"[{request_id}] Google batch request failed: {e}")
+ return JSONResponse(
+ status_code=500,
+ content={
+ "error": {
+ "code": 500,
+ "message": f"Failed to forward batch request: {e!s}",
+ "status": "INTERNAL",
+ }
+ },
+ )
+
+ async def _google_batch_passthrough(
+ self,
+ request: Request,
+ model: str,
+ body: dict | None = None,
+ ) -> Response:
+ """Pass through Google batch request without modification."""
+ from fastapi.responses import Response
+
+ start_time = time.time()
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"
+
+ # Add API key to URL if present in headers
+ api_key = headers.pop("x-goog-api-key", None)
+ if api_key:
+ url = f"{url}?key={api_key}"
+
+ if body is None:
+ body_content = await request.body()
+ else:
+ body_content = json.dumps(body).encode()
+
+ response = await self.http_client.post( # type: ignore[union-attr]
+ url,
+ headers=headers,
+ content=body_content,
+ )
+
+ # Track metrics
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider="google",
+ model=f"passthrough:batch:{model}",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ )
+
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def handle_google_batch_passthrough(
+ self,
+ request: Request,
+ batch_name: str | None = None,
+ ) -> Response:
+ """Handle Google batch passthrough endpoints.
+
+ Used for:
+ - GET /v1beta/batches/{batch_name} - Get batch status
+ - POST /v1beta/batches/{batch_name}:cancel - Cancel batch
+ - DELETE /v1beta/batches/{batch_name} - Delete batch
+ """
+ from fastapi.responses import Response
+
+ start_time = time.time()
+ path = request.url.path
+ url = f"{self.GEMINI_API_URL}{path}"
+
+ # Preserve query string parameters
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+
+ # Handle API key
+ api_key = headers.pop("x-goog-api-key", None)
+ if api_key:
+ if "?" in url:
+ url = f"{url}&key={api_key}"
+ else:
+ url = f"{url}?key={api_key}"
+
+ body = await request.body()
+
+ response = await self.http_client.request( # type: ignore[union-attr]
+ method=request.method,
+ url=url,
+ headers=headers,
+ content=body,
+ )
+
+ # Track metrics
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider="google",
+ model="passthrough:batches",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ )
+
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def _store_google_batch_context(
+ self,
+ batch_name: str,
+ requests_list: list[dict[str, Any]],
+ model: str,
+ api_key: str | None,
+ ) -> None:
+ """Store Google batch context for CCR result processing.
+
+ Args:
+ batch_name: The batch name from the API response.
+ requests_list: The original batch requests.
+ model: The model used for the batch.
+ api_key: The API key for continuation calls.
+ """
+ from headroom.ccr import BatchContext, BatchRequestContext, get_batch_context_store
+
+ store = get_batch_context_store()
+ context = BatchContext(
+ batch_id=batch_name,
+ provider="google",
+ api_key=api_key,
+ api_base_url=self.GEMINI_API_URL,
+ )
+
+ for batch_req in requests_list:
+ metadata = batch_req.get("metadata", {})
+ custom_id = metadata.get("key", "")
+ req_content = batch_req.get("request", {})
+ contents = req_content.get("contents", [])
+ system_instruction = req_content.get("systemInstruction")
+
+ # Convert contents to messages format for CCR handler
+ messages, _ = self._gemini_contents_to_messages(contents, system_instruction)
+
+ # Extract system instruction text if present
+ sys_text = None
+ if system_instruction:
+ parts = system_instruction.get("parts", [])
+ if parts and isinstance(parts[0], dict):
+ sys_text = parts[0].get("text")
+
+ context.add_request(
+ BatchRequestContext(
+ custom_id=custom_id,
+ messages=messages,
+ tools=req_content.get("tools"),
+ model=model,
+ system_instruction=sys_text,
+ )
+ )
+
+ await store.store(context)
+ logger.debug(
+ f"Stored Google batch context for {batch_name} with {len(requests_list)} requests"
+ )
+
+ async def handle_google_batch_results(
+ self,
+ request: Request,
+ batch_name: str,
+ ) -> Response:
+ """Handle Google batch results with CCR post-processing.
+
+ Google batch results endpoint returns the batch operation status.
+ When status is SUCCEEDED, results are embedded in the response.
+ This handler processes CCR tool calls in those results.
+ """
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.ccr import BatchResultProcessor, get_batch_context_store
+
+ start_time = time.time()
+
+ # Forward request to get batch status/results
+ url = f"{self.GEMINI_API_URL}/v1beta/{batch_name}"
+
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+
+ # Handle API key
+ api_key = headers.pop("x-goog-api-key", None)
+ if api_key:
+ if "?" in url:
+ url = f"{url}&key={api_key}"
+ else:
+ url = f"{url}?key={api_key}"
+
+ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]
+
+ if response.status_code != 200:
+ # Error - pass through
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Parse response
+ try:
+ response_data = response.json()
+ except json.JSONDecodeError:
+ # Not JSON - pass through
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Check if batch has results (state must be SUCCEEDED)
+ metadata = response_data.get("metadata", {})
+ state = metadata.get("state")
+
+ if state != "SUCCEEDED":
+ # Batch not complete - pass through
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Extract results from response
+ # Google embeds results in the batch response
+ results = response_data.get("response", {}).get("responses", [])
+
+ if not results:
+ # No results to process
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Check if we have context and CCR processing is enabled
+ store = get_batch_context_store()
+ batch_context = await store.get(batch_name)
+
+ if batch_context is None or not self.config.ccr_inject_tool:
+ # No context or CCR disabled - pass through
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Process results with CCR handler
+ processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]
+ processed = await processor.process_results(batch_name, results, "google")
+
+ # Update response with processed results
+ processed_results = [p.result for p in processed]
+ response_data["response"]["responses"] = processed_results
+
+ for p in processed:
+ if p.was_processed:
+ logger.info(
+ f"CCR: Processed Google batch result {p.custom_id} "
+ f"({p.continuation_rounds} continuation rounds)"
+ )
+
+ # Track metrics
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider="google",
+ model="batch:ccr-processed",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ )
+
+ return JSONResponse(content=response_data, status_code=200)
+
+ async def handle_batch_create(self, request: Request) -> Response:
+ """Handle POST /v1/batches - Create a batch with compression.
+
+ Flow:
+ 1. Parse request to get input_file_id
+ 2. Download the JSONL file content from OpenAI
+ 3. Parse each line and compress the messages
+ 4. Create a new compressed JSONL file
+ 5. Upload compressed file to OpenAI
+ 6. Create batch with the new compressed file_id
+ 7. Return batch object with compression stats in metadata
+ """
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.proxy.helpers import _read_request_json
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+
+ input_file_id = body.get("input_file_id")
+ endpoint = body.get("endpoint")
+ completion_window = body.get("completion_window", "24h")
+ metadata = body.get("metadata", {})
+
+ if not input_file_id:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": "input_file_id is required",
+ "type": "invalid_request_error",
+ "code": "missing_parameter",
+ }
+ },
+ )
+
+ if not endpoint:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": "endpoint is required",
+ "type": "invalid_request_error",
+ "code": "missing_parameter",
+ }
+ },
+ )
+
+ # Only compress chat completions endpoint
+ if endpoint != "/v1/chat/completions":
+ # Pass through for other endpoints
+ return await self._batch_passthrough(request, body)
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+
+ try:
+ # Step 1: Download the input file from OpenAI
+ logger.info(f"[{request_id}] Batch: Downloading input file {input_file_id}")
+ file_content = await self._download_openai_file(input_file_id, headers)
+
+ if file_content is None:
+ return JSONResponse(
+ status_code=404,
+ content={
+ "error": {
+ "message": f"Failed to download file {input_file_id}",
+ "type": "invalid_request_error",
+ "code": "file_not_found",
+ }
+ },
+ )
+
+ # Step 2: Parse and compress each line
+ logger.info(f"[{request_id}] Batch: Compressing JSONL content")
+ compressed_lines, stats = await self._compress_batch_jsonl(file_content, request_id)
+
+ if stats["total_requests"] == 0:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": "No valid requests found in input file",
+ "type": "invalid_request_error",
+ "code": "empty_file",
+ }
+ },
+ )
+
+ # Step 3: Create compressed JSONL content
+ compressed_content = "\n".join(compressed_lines)
+
+ # Step 4: Upload compressed file to OpenAI
+ logger.info(f"[{request_id}] Batch: Uploading compressed file")
+ new_file_id = await self._upload_openai_file(
+ compressed_content, f"compressed_{input_file_id}.jsonl", headers
+ )
+
+ if new_file_id is None:
+ return JSONResponse(
+ status_code=500,
+ content={
+ "error": {
+ "message": "Failed to upload compressed file",
+ "type": "server_error",
+ "code": "upload_failed",
+ }
+ },
+ )
+
+ # Step 5: Create batch with compressed file
+ logger.info(f"[{request_id}] Batch: Creating batch with compressed file {new_file_id}")
+
+ # Add compression stats to metadata
+ compression_metadata = {
+ **metadata,
+ "headroom_compressed": "true",
+ "headroom_original_file_id": input_file_id,
+ "headroom_total_requests": str(stats["total_requests"]),
+ "headroom_tokens_saved": str(stats["total_tokens_saved"]),
+ "headroom_original_tokens": str(stats["total_original_tokens"]),
+ "headroom_compressed_tokens": str(stats["total_compressed_tokens"]),
+ "headroom_savings_percent": f"{stats['savings_percent']:.1f}",
+ }
+
+ batch_body = {
+ "input_file_id": new_file_id,
+ "endpoint": endpoint,
+ "completion_window": completion_window,
+ "metadata": compression_metadata,
+ }
+
+ url = f"{self.OPENAI_API_URL}/v1/batches"
+ response = await self.http_client.post(url, json=batch_body, headers=headers) # type: ignore[union-attr]
+
+ total_latency = (time.time() - start_time) * 1000
+
+ # Log compression stats
+ logger.info(
+ f"[{request_id}] Batch created: {stats['total_requests']} requests, "
+ f"{stats['total_original_tokens']:,} -> {stats['total_compressed_tokens']:,} tokens "
+ f"(saved {stats['total_tokens_saved']:,} tokens, {stats['savings_percent']:.1f}%) "
+ f"in {total_latency:.0f}ms"
+ )
+
+ # Record metrics
+ await self.metrics.record_request(
+ provider="openai",
+ model="batch",
+ input_tokens=stats["total_compressed_tokens"],
+ output_tokens=0,
+ tokens_saved=stats["total_tokens_saved"],
+ latency_ms=total_latency,
+ )
+
+ # Return response with compression info in headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ response_headers["x-headroom-tokens-saved"] = str(stats["total_tokens_saved"])
+ response_headers["x-headroom-savings-percent"] = f"{stats['savings_percent']:.1f}"
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ except Exception as e:
+ logger.error(f"[{request_id}] Batch creation failed: {type(e).__name__}: {e}")
+ await self.metrics.record_failed()
+ return JSONResponse(
+ status_code=500,
+ content={
+ "error": {
+ "message": "An error occurred while processing the batch request",
+ "type": "server_error",
+ "code": "batch_processing_error",
+ }
+ },
+ )
+
+ async def _download_openai_file(self, file_id: str, headers: dict) -> str | None:
+ """Download file content from OpenAI."""
+ url = f"{self.OPENAI_API_URL}/v1/files/{file_id}/content"
+ try:
+ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]
+ if response.status_code == 200:
+ return str(response.text)
+ logger.error(f"Failed to download file {file_id}: {response.status_code}")
+ return None
+ except Exception as e:
+ logger.error(f"Error downloading file {file_id}: {e}")
+ return None
+
+ async def _upload_openai_file(self, content: str, filename: str, headers: dict) -> str | None:
+ """Upload a file to OpenAI for batch processing."""
+ url = f"{self.OPENAI_API_URL}/v1/files"
+
+ # Prepare multipart form data
+ # We need to use httpx's files parameter for multipart upload
+ files = {
+ "file": (filename, content.encode("utf-8"), "application/jsonl"),
+ }
+ data = {
+ "purpose": "batch",
+ }
+
+ # Remove content-type from headers (httpx will set it for multipart)
+ upload_headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}
+
+ try:
+ response = await self.http_client.post( # type: ignore[union-attr]
+ url, files=files, data=data, headers=upload_headers
+ )
+ if response.status_code == 200:
+ result = response.json()
+ file_id: str | None = result.get("id")
+ return file_id
+ logger.error(f"Failed to upload file: {response.status_code} - {response.text}")
+ return None
+ except Exception as e:
+ logger.error(f"Error uploading file: {e}")
+ return None
+
+ async def _compress_batch_jsonl(self, content: str, request_id: str) -> tuple[list[str], dict]:
+ """Compress messages in each line of a batch JSONL file.
+
+ Returns:
+ Tuple of (compressed_lines, stats_dict)
+ """
+ from headroom.ccr import CCRToolInjector
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ lines = content.strip().split("\n")
+ compressed_lines = []
+ total_original_tokens = 0
+ total_compressed_tokens = 0
+ total_requests = 0
+ errors = 0
+
+ tokenizer = get_tokenizer("gpt-4") # Use gpt-4 tokenizer for batch
+
+ for i, line in enumerate(lines):
+ if not line.strip():
+ continue
+
+ try:
+ request_obj = json.loads(line)
+ body = request_obj.get("body", {})
+ messages = body.get("messages", [])
+ model = body.get("model", "gpt-4")
+
+ if not messages:
+ # No messages to compress, pass through
+ compressed_lines.append(line)
+ total_requests += 1
+ continue
+
+ # Compress messages using the OpenAI pipeline
+ if self.config.optimize:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ compressed_messages = result.messages
+ # Use pipeline's token counts for consistency with pipeline logs
+ original_tokens = result.tokens_before
+ compressed_tokens = result.tokens_after
+ except Exception as e:
+ logger.warning(f"[{request_id}] Compression failed for line {i}: {e}")
+ compressed_messages = messages
+ original_tokens = tokenizer.count_messages(messages)
+ compressed_tokens = original_tokens
+ else:
+ compressed_messages = messages
+ original_tokens = tokenizer.count_messages(messages)
+ compressed_tokens = original_tokens
+
+ total_original_tokens += original_tokens
+ total_compressed_tokens += compressed_tokens
+ tokens_saved = original_tokens - compressed_tokens
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = body.get("tools")
+ if self.config.ccr_inject_tool and tokens_saved > 0:
+ injector = CCRToolInjector(
+ provider="openai",
+ inject_tool=True,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ compressed_messages, tools, was_injected = injector.process_request(
+ compressed_messages, tools
+ )
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for batch line {i}"
+ )
+
+ # Update body with compressed messages
+ body["messages"] = compressed_messages
+ if tools is not None:
+ body["tools"] = tools
+ request_obj["body"] = body
+
+ compressed_lines.append(json.dumps(request_obj))
+ total_requests += 1
+
+ except json.JSONDecodeError as e:
+ logger.warning(f"[{request_id}] Invalid JSON on line {i}: {e}")
+ errors += 1
+ # Keep original line on error
+ compressed_lines.append(line)
+ total_requests += 1
+
+ total_tokens_saved = total_original_tokens - total_compressed_tokens
+ savings_percent = (
+ (total_tokens_saved / total_original_tokens * 100) if total_original_tokens > 0 else 0
+ )
+
+ stats = {
+ "total_requests": total_requests,
+ "total_original_tokens": total_original_tokens,
+ "total_compressed_tokens": total_compressed_tokens,
+ "total_tokens_saved": total_tokens_saved,
+ "savings_percent": savings_percent,
+ "errors": errors,
+ }
+
+ return compressed_lines, stats
+
+ async def _batch_passthrough(self, request: Request, body: dict) -> Response:
+ """Pass through batch request to OpenAI without compression."""
+ from fastapi.responses import Response
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+
+ url = f"{self.OPENAI_API_URL}/v1/batches"
+ response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
+
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def handle_batch_list(self, request: Request) -> Response:
+ """Handle GET /v1/batches - List batches (passthrough)."""
+ return await self.handle_passthrough(request, self.OPENAI_API_URL)
+
+ async def handle_batch_get(self, request: Request, batch_id: str) -> Response:
+ """Handle GET /v1/batches/{batch_id} - Get batch (passthrough)."""
+ return await self.handle_passthrough(request, self.OPENAI_API_URL)
+
+ async def handle_batch_cancel(self, request: Request, batch_id: str) -> Response:
+ """Handle POST /v1/batches/{batch_id}/cancel - Cancel batch (passthrough)."""
+ return await self.handle_passthrough(request, self.OPENAI_API_URL)
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py
new file mode 100644
index 000000000..3c1f4bb14
--- /dev/null
+++ b/headroom/proxy/handlers/gemini.py
@@ -0,0 +1,649 @@
+"""Gemini handler mixin for HeadroomProxy.
+
+Contains all Google Gemini API handlers including format conversion utilities.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from fastapi import Request
+ from fastapi.responses import JSONResponse, Response, StreamingResponse
+
+logger = logging.getLogger("headroom.proxy")
+
+
+class GeminiHandlerMixin:
+ """Mixin providing Gemini API handler methods for HeadroomProxy."""
+
+ def _has_non_text_parts(self, content: dict) -> bool:
+ """Check if a Gemini content entry has non-text parts.
+
+ Non-text parts include:
+ - inlineData: Base64-encoded images/media
+ - fileData: File references (URI + MIME type)
+ - functionCall: Function calls from model
+ - functionResponse: Responses to function calls
+
+ Args:
+ content: A single Gemini content entry with 'parts' list.
+
+ Returns:
+ True if any part contains non-text data.
+ """
+ parts = content.get("parts", [])
+ for part in parts:
+ if any(
+ key in part
+ for key in ("inlineData", "fileData", "functionCall", "functionResponse")
+ ):
+ return True
+ return False
+
+ def _gemini_contents_to_messages(
+ self, contents: list[dict], system_instruction: dict | None = None
+ ) -> tuple[list[dict], set[int]]:
+ """Convert Gemini contents[] format to OpenAI messages[] format for optimization.
+
+ Gemini format:
+ contents: [{"role": "user", "parts": [{"text": "..."}]}]
+ systemInstruction: {"parts": [{"text": "..."}]}
+
+ OpenAI format:
+ messages: [{"role": "user", "content": "..."}]
+
+ Returns:
+ Tuple of (messages, preserved_indices) where preserved_indices contains
+ the indices of content entries that have non-text parts (images, function
+ calls, etc.) and should not be compressed.
+ """
+ messages = []
+ preserved_indices: set[int] = set()
+
+ # Add system instruction as system message
+ if system_instruction:
+ parts = system_instruction.get("parts", [])
+ text_parts = [p.get("text", "") for p in parts if "text" in p]
+ if text_parts:
+ messages.append({"role": "system", "content": "\n".join(text_parts)})
+
+ # Convert contents to messages
+ for idx, content in enumerate(contents):
+ # Track content entries with non-text parts
+ if self._has_non_text_parts(content):
+ preserved_indices.add(idx)
+
+ role = content.get("role", "user")
+ # Map Gemini roles to OpenAI roles
+ if role == "model":
+ role = "assistant"
+
+ parts = content.get("parts", [])
+ text_parts = [p.get("text", "") for p in parts if "text" in p]
+
+ if text_parts:
+ messages.append({"role": role, "content": "\n".join(text_parts)})
+
+ return messages, preserved_indices
+
+ def _messages_to_gemini_contents(self, messages: list[dict]) -> tuple[list[dict], dict | None]:
+ """Convert OpenAI messages[] format back to Gemini contents[] format.
+
+ Returns:
+ (contents, system_instruction) tuple
+ """
+ contents = []
+ system_instruction = None
+
+ for msg in messages:
+ role = msg.get("role", "user")
+ content = msg.get("content", "")
+
+ if role == "system":
+ # Extract as systemInstruction
+ system_instruction = {"parts": [{"text": content}]}
+ else:
+ # Map OpenAI roles to Gemini roles
+ gemini_role = "model" if role == "assistant" else "user"
+ contents.append({"role": gemini_role, "parts": [{"text": content}]})
+
+ return contents, system_instruction
+
+ async def handle_gemini_generate_content(
+ self,
+ request: Request,
+ model: str,
+ ) -> Response | StreamingResponse:
+ """Handle Gemini native /v1beta/models/{model}:generateContent endpoint.
+
+ Gemini's native API differs from OpenAI:
+ - Input: `contents[]` with `parts[]` instead of `messages`
+ - System: `systemInstruction` instead of system message
+ - Auth: `x-goog-api-key` header instead of `Authorization: Bearer`
+ - Output: `candidates[].content.parts[].text`
+ """
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "code": 413,
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "code": 400,
+ }
+ },
+ )
+
+ contents = body.get("contents", [])
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ tags = self._extract_tags(headers)
+
+ # Rate limiting (use Gemini API key)
+ if self.rate_limiter:
+ rate_key = headers.get("x-goog-api-key", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited()
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Convert Gemini format to messages for optimization
+ system_instruction = body.get("systemInstruction")
+ messages, preserved_indices = self._gemini_contents_to_messages(
+ contents, system_instruction
+ )
+
+ # Store original content entries that have non-text parts before compression
+ preserved_contents = {idx: contents[idx] for idx in preserved_indices}
+
+ # Early exit if ALL content has non-text parts (nothing to compress)
+ if len(preserved_indices) == len(contents):
+ # All content has non-text parts, skip compression entirely
+ # Just forward the request as-is
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"
+ query_params = dict(request.query_params)
+ is_streaming = query_params.get("alt") == "sse"
+ if "key" in query_params:
+ url += f"?key={query_params['key']}"
+
+ if is_streaming:
+ stream_url = (
+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
+ )
+ if "key" in query_params:
+ stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
+ return await self._stream_response(
+ stream_url,
+ headers,
+ body,
+ "gemini",
+ model,
+ request_id,
+ 0,
+ 0,
+ 0,
+ [],
+ tags,
+ 0,
+ )
+ else:
+ response = await self._retry_request("POST", url, headers, body)
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Token counting
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Optimization
+ transforms_applied: list[str] = []
+ waste_signals_dict: dict[str, int] | None = None
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ _compression_failed = False
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+ if self.config.optimize and messages and _license_ok:
+ try:
+ # Use OpenAI pipeline (similar message format)
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ # Use pipeline's token counts for consistency with pipeline logs
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+ if result.waste_signals:
+ waste_signals_dict = result.waste_signals.to_dict()
+ except Exception as e:
+ _compression_failed = True
+ logger.warning(f"[{request_id}] Gemini optimization failed: {e}")
+
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Query Echo: disabled — hurts prefix caching in long conversations.
+
+ # Convert back to Gemini format if optimized
+ if optimized_messages != messages:
+ optimized_contents, optimized_system = self._messages_to_gemini_contents(
+ optimized_messages
+ )
+
+ # Restore preserved content entries that had non-text parts
+ for orig_idx, original_content in preserved_contents.items():
+ if orig_idx < len(optimized_contents):
+ optimized_contents[orig_idx] = original_content
+
+ body["contents"] = optimized_contents
+ if optimized_system:
+ body["systemInstruction"] = optimized_system
+ elif "systemInstruction" in body:
+ del body["systemInstruction"]
+
+ # Build URL - model is extracted from path
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"
+
+ # Check if streaming requested via query param
+ query_params = dict(request.query_params)
+ is_streaming = query_params.get("alt") == "sse"
+
+ # Preserve API key in query params if present
+ if "key" in query_params:
+ url += f"?key={query_params['key']}"
+
+ try:
+ if is_streaming:
+ # For streaming, use streamGenerateContent endpoint
+ stream_url = (
+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
+ )
+ if "key" in query_params:
+ stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
+
+ return await self._stream_response(
+ stream_url,
+ headers,
+ body,
+ "gemini",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ )
+ else:
+ response = await self._retry_request("POST", url, headers, body)
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = optimized_tokens # fallback
+ output_tokens = 0
+ cache_read_tokens = 0
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usageMetadata", {})
+ total_input_tokens = usage.get("promptTokenCount", optimized_tokens)
+ output_tokens = usage.get("candidatesTokenCount", 0)
+ # Gemini returns cachedContentTokenCount for context-cached tokens
+ # These are charged at 10-25% of the input price depending on model
+ cache_read_tokens = usage.get("cachedContentTokenCount", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from Gemini response: {e}"
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(
+ model,
+ tokens_saved,
+ optimized_tokens,
+ cache_read_tokens=cache_read_tokens,
+ )
+
+ await self.metrics.record_request(
+ provider="gemini",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ waste_signals=waste_signals_dict,
+ cache_read_tokens=cache_read_tokens,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] Gemini {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens)"
+ )
+ else:
+ logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ # Inject Headroom compression metrics (for SaaS metering)
+ response_headers["x-headroom-tokens-before"] = str(original_tokens)
+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
+ response_headers["x-headroom-model"] = model
+ if transforms_applied:
+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
+ if cache_read_tokens > 0:
+ response_headers["x-headroom-cached"] = "true"
+ if _compression_failed:
+ response_headers["x-headroom-compression-failed"] = "true"
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed()
+ logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "code": 502,
+ }
+ },
+ )
+
+ async def handle_gemini_stream_generate_content(
+ self,
+ request: Request,
+ model: str,
+ ) -> StreamingResponse | JSONResponse:
+ """Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+ from headroom.tokenizers import get_tokenizer
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "code": 400,
+ }
+ },
+ )
+
+ contents = body.get("contents", [])
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ tags = self._extract_tags(headers)
+
+ # Token counting
+ tokenizer = get_tokenizer(model)
+ original_tokens = 0
+ for content in contents:
+ parts = content.get("parts", [])
+ for part in parts:
+ if "text" in part:
+ original_tokens += tokenizer.count_text(part["text"])
+
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Build URL with SSE param
+ query_params = dict(request.query_params)
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
+ if "key" in query_params:
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
+
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "gemini",
+ model,
+ request_id,
+ original_tokens,
+ original_tokens,
+ 0, # tokens_saved
+ [], # transforms_applied
+ tags,
+ optimization_latency,
+ )
+
+ async def handle_gemini_count_tokens(
+ self,
+ request: Request,
+ model: str,
+ ) -> Response:
+ """Handle Gemini /v1beta/models/{model}:countTokens endpoint with compression.
+
+ This endpoint counts tokens AFTER applying compression, so users can see
+ how many tokens they'll actually use after optimization.
+
+ The request format is the same as generateContent:
+ {"contents": [...], "systemInstruction": {...}}
+ """
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.proxy.helpers import _read_request_json
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "code": 400,
+ }
+ },
+ )
+
+ contents = body.get("contents", [])
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+
+ # Convert Gemini format to messages for optimization
+ system_instruction = body.get("systemInstruction")
+ messages, preserved_indices = self._gemini_contents_to_messages(
+ contents, system_instruction
+ )
+
+ # Store original content entries that have non-text parts before compression
+ preserved_contents = {idx: contents[idx] for idx in preserved_indices}
+
+ # Early exit if ALL content has non-text parts (nothing to compress)
+ if len(preserved_indices) == len(contents):
+ # All content has non-text parts, skip compression entirely
+ # Just forward the countTokens request as-is
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"
+ query_params = dict(request.query_params)
+ if "key" in query_params:
+ url += f"?key={query_params['key']}"
+
+ response = await self._retry_request("POST", url, headers, body)
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ # Token counting (original)
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Apply compression using the same pipeline as generateContent
+ transforms_applied: list[str] = []
+ optimized_messages = messages
+
+ if self.config.optimize and messages:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ except Exception as e:
+ logger.warning(f"[{request_id}] Gemini countTokens optimization failed: {e}")
+
+ # Convert back to Gemini format for the API call
+ if optimized_messages != messages:
+ optimized_contents, optimized_system = self._messages_to_gemini_contents(
+ optimized_messages
+ )
+
+ # Restore preserved content entries that had non-text parts
+ for orig_idx, original_content in preserved_contents.items():
+ if orig_idx < len(optimized_contents):
+ optimized_contents[orig_idx] = original_content
+
+ body["contents"] = optimized_contents
+ if optimized_system:
+ body["systemInstruction"] = optimized_system
+ elif "systemInstruction" in body:
+ del body["systemInstruction"]
+
+ # Build URL
+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"
+
+ # Preserve API key in query params if present
+ query_params = dict(request.query_params)
+ if "key" in query_params:
+ url += f"?key={query_params['key']}"
+
+ try:
+ response = await self._retry_request("POST", url, headers, body)
+ total_latency = (time.time() - start_time) * 1000
+
+ # Parse response to get token count
+ compressed_tokens = 0
+ try:
+ resp_json = response.json()
+ compressed_tokens = resp_json.get("totalTokens", 0)
+ except (json.JSONDecodeError, ValueError) as e:
+ logger.debug(f"[{request_id}] Failed to parse Gemini token count response: {e}")
+
+ # Track stats
+ tokens_saved = (
+ max(0, original_tokens - compressed_tokens) if compressed_tokens > 0 else 0
+ )
+
+ await self.metrics.record_request(
+ provider="gemini",
+ model=model,
+ input_tokens=compressed_tokens,
+ output_tokens=0,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] Gemini countTokens {model}: {original_tokens:,} → {compressed_tokens:,} "
+ f"(saved {tokens_saved:,} tokens, transforms: {transforms_applied})"
+ )
+ else:
+ logger.info(
+ f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens"
+ )
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed()
+ logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "code": 502,
+ }
+ },
+ )
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
new file mode 100644
index 000000000..4bdcc1ec1
--- /dev/null
+++ b/headroom/proxy/handlers/openai.py
@@ -0,0 +1,1320 @@
+"""OpenAI handler mixin for HeadroomProxy.
+
+Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import logging
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from fastapi import Request, WebSocket
+ from fastapi.responses import JSONResponse, Response, StreamingResponse
+
+
+logger = logging.getLogger("headroom.proxy")
+
+
+class OpenAIHandlerMixin:
+ """Mixin providing OpenAI API handler methods for HeadroomProxy."""
+
+ async def handle_openai_chat(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle OpenAI /v1/chat/completions endpoint."""
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.ccr import CCRToolInjector
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS,
+ MAX_MESSAGE_ARRAY_LENGTH,
+ MAX_REQUEST_BODY_SIZE,
+ _get_image_compressor,
+ _read_request_json,
+ )
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+ model = body.get("model", "unknown")
+ messages = body.get("messages", [])
+
+ # Validate message array size
+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Message array too large ({len(messages)} messages). "
+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
+ "type": "invalid_request_error",
+ "code": "invalid_request",
+ }
+ },
+ )
+
+ stream = body.get("stream", False)
+
+ # Bypass: skip ALL compression for explicit opt-out
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ if _bypass:
+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")
+
+ # Image compression (before text optimization)
+ if self.config.image_optimize and messages and not _bypass:
+ compressor = _get_image_compressor()
+ if compressor and compressor.has_images(messages):
+ messages = compressor.compress(messages, provider="openai")
+ if compressor.last_result:
+ logger.info(
+ f"Image compression: {compressor.last_result.technique.value} "
+ f"({compressor.last_result.savings_percent:.0f}% saved, "
+ f"{compressor.last_result.original_tokens} -> "
+ f"{compressor.last_result.compressed_tokens} tokens)"
+ )
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ tags = self._extract_tags(headers)
+
+ # Rate limiting
+ if self.rate_limiter:
+ rate_key = headers.get("authorization", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited()
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Check cache
+ if self.cache and not stream:
+ cached = await self.cache.get(messages, model)
+ if cached:
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0, # Savings already counted when response was cached
+ latency_ms=(time.time() - start_time) * 1000,
+ cached=True,
+ )
+
+ # Remove compression headers from cached response
+ response_headers = dict(cached.response_headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(content=cached.response_body, headers=response_headers)
+
+ # Token counting
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Hook: pre_compress
+ _hook_biases = None
+ if self.config.hooks:
+ from headroom.hooks import CompressContext
+
+ _hook_ctx = CompressContext(model=model, provider="openai")
+ try:
+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)
+ _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
+ except Exception as e:
+ logger.debug(f"[{request_id}] Hook error: {e}")
+
+ # Optimization
+ transforms_applied = []
+ pipeline_timing: dict[str, float] = {}
+ waste_signals_dict: dict[str, int] | None = None
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ # Get prefix cache tracker for this session
+ openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
+ openai_prefix_tracker = self.session_tracker_store.get_or_create(
+ openai_session_id, "openai"
+ )
+ openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
+
+ _compression_failed = False
+ original_messages = messages # Preserve for 400-retry fallback
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+ if self.config.optimize and messages and not _bypass and _license_ok:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+
+ if self.config.mode == "token_headroom":
+ comp_cache = self._get_compression_cache(openai_session_id)
+
+ # Zone 1: Swap cached compressed versions
+ working_messages = comp_cache.apply_cached(messages)
+
+ # Re-freeze boundary
+ openai_frozen_count = comp_cache.compute_frozen_count(messages)
+
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=working_messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(working_messages),
+ frozen_message_count=openai_frozen_count,
+ biases=_hook_biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != working_messages:
+ comp_cache.update_from_result(messages, result.messages)
+
+ # Always use pipeline result in token_headroom mode
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)
+ # so tokens_saved captures both Zone 1 + Zone 2 savings.
+ optimized_tokens = result.tokens_after
+ else:
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ frozen_message_count=openai_frozen_count,
+ biases=_hook_biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+
+ if result.waste_signals:
+ waste_signals_dict = result.waste_signals.to_dict()
+ except Exception as e:
+ logger.warning(f"Optimization failed: {e}")
+ # Flag compression failure for observability
+ _compression_failed = True
+
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Hook: post_compress
+ if self.config.hooks and tokens_saved > 0:
+ from headroom.hooks import CompressEvent
+
+ try:
+ self.config.hooks.post_compress(
+ CompressEvent(
+ tokens_before=original_tokens,
+ tokens_after=optimized_tokens,
+ tokens_saved=tokens_saved,
+ compression_ratio=tokens_saved / original_tokens
+ if original_tokens > 0
+ else 0,
+ transforms_applied=transforms_applied,
+ model=model,
+ provider="openai",
+ )
+ )
+ except Exception as e:
+ logger.debug(f"[{request_id}] post_compress hook error: {e}")
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = body.get("tools")
+ _original_tools = tools # Preserve for diagnostic / future retry
+ if (
+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
+ ) and not _bypass:
+ injector = CCRToolInjector(
+ provider="openai",
+ inject_tool=self.config.ccr_inject_tool,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ optimized_messages, tools, was_injected = injector.process_request(
+ optimized_messages, tools
+ )
+
+ if injector.has_compressed_content:
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
+ )
+ else:
+ logger.debug(
+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
+ )
+
+ # Query Echo: disabled — hurts prefix caching in long conversations.
+
+ body["messages"] = optimized_messages
+ if tools is not None:
+ body["tools"] = tools
+
+ # Route through LiteLLM/any-llm backend if configured
+ if self.anthropic_backend is not None:
+ try:
+ if stream:
+ # Streaming: use stream_openai_message() → SSE events
+ return await self._stream_openai_via_backend(
+ body,
+ headers,
+ model,
+ request_id,
+ start_time,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+ else:
+ # Non-streaming: use send_openai_message() → JSON
+ backend_response = await self.anthropic_backend.send_openai_message(
+ body, headers
+ )
+
+ if backend_response.error:
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+
+ # Track metrics
+ total_latency = (time.time() - start_time) * 1000
+ usage = backend_response.body.get("usage", {})
+ output_tokens = usage.get("completion_tokens", 0)
+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+
+ await self.metrics.record_request(
+ provider=self.anthropic_backend.name,
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ cached=False,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"
+ )
+
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+ except Exception as e:
+ logger.error(f"[{request_id}] Backend error: {e}")
+ return JSONResponse(
+ status_code=500,
+ content={
+ "error": {
+ "message": str(e),
+ "type": "api_error",
+ "code": "backend_error",
+ }
+ },
+ )
+
+ # Direct OpenAI API (no backend configured)
+ url = f"{self.OPENAI_API_URL}/v1/chat/completions"
+
+ try:
+ if stream:
+ # Inject stream_options to get usage stats in streaming response
+ # This allows accurate token counting instead of byte-based estimation
+ if "stream_options" not in body:
+ body["stream_options"] = {"include_usage": True}
+ elif isinstance(body.get("stream_options"), dict):
+ body["stream_options"]["include_usage"] = True
+
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "openai",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ prefix_tracker=openai_prefix_tracker,
+ )
+ else:
+ response = await self._retry_request("POST", url, headers, body)
+
+ # Full diagnostic dump on upstream errors (OpenAI handler)
+ if response.status_code >= 400:
+ try:
+ err_body = response.json()
+ err_msg = err_body.get("error", {}).get("message", "")
+ err_type = err_body.get("error", {}).get("type", "")
+ except Exception:
+ err_body = {"raw": response.text[:2000]}
+ err_msg = str(response.text[:500])
+ err_type = "parse_error"
+
+ logger.warning(
+ f"[{request_id}] UPSTREAM_ERROR "
+ f"status={response.status_code} "
+ f"error_type={err_type} "
+ f"error_msg={err_msg!r} "
+ f"model={model} "
+ f"compressed={'yes' if transforms_applied else 'no'} "
+ f"transforms={transforms_applied} "
+ f"original_tokens={original_tokens} "
+ f"optimized_tokens={optimized_tokens} "
+ f"message_count={len(body.get('messages', []))} "
+ f"stream={stream}"
+ )
+
+ try:
+ debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"
+ debug_dir.mkdir(parents=True, exist_ok=True)
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+ debug_file = debug_dir / f"{ts}_{request_id}.json"
+
+ safe_headers = {}
+ for k, v in headers.items():
+ if k.lower() in ("x-api-key", "authorization"):
+ safe_headers[k] = v[:12] + "..." if v else ""
+ else:
+ safe_headers[k] = v
+
+ debug_payload = {
+ "request_id": request_id,
+ "timestamp": datetime.now().isoformat(),
+ "status_code": response.status_code,
+ "error_response": err_body,
+ "model": model,
+ "stream": stream,
+ "headers": safe_headers,
+ "compression": {
+ "was_compressed": bool(transforms_applied),
+ "transforms": transforms_applied,
+ "original_tokens": original_tokens,
+ "optimized_tokens": optimized_tokens,
+ "tokens_saved": tokens_saved,
+ "compression_failed": _compression_failed,
+ },
+ "tools_sent": body.get("tools"),
+ "tool_count": len(body.get("tools") or []),
+ "original_tool_count": len(_original_tools or []),
+ "messages_sent": body.get("messages"),
+ "message_count": len(body.get("messages", [])),
+ "original_messages": (
+ original_messages
+ if original_messages is not body.get("messages")
+ else "__same_as_sent__"
+ ),
+ "original_message_count": len(original_messages),
+ "system_prompt": body.get("system"),
+ }
+
+ with open(debug_file, "w") as f:
+ json.dump(debug_payload, f, indent=2, default=str)
+
+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
+ except Exception as dump_err:
+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
+
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = optimized_tokens # fallback
+ output_tokens = 0
+ cache_read_tokens = 0
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usage", {})
+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+ output_tokens = usage.get("completion_tokens", 0)
+ # OpenAI returns cached_tokens in prompt_tokens_details
+ # These are charged at 50% of the input price
+ prompt_details = usage.get("prompt_tokens_details", {})
+ cache_read_tokens = prompt_details.get("cached_tokens", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
+ )
+
+ # Update prefix cache tracker for next turn
+ openai_prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=0, # OpenAI doesn't report write tokens
+ messages=optimized_messages,
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(
+ model,
+ tokens_saved,
+ optimized_tokens,
+ cache_read_tokens=cache_read_tokens,
+ )
+
+ # Cache
+ if self.cache and response.status_code == 200:
+ await self.cache.set(
+ messages, model, response.content, dict(response.headers), tokens_saved
+ )
+
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ waste_signals=waste_signals_dict,
+ cache_read_tokens=cache_read_tokens,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens)"
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Inject Headroom compression metrics (for SaaS metering)
+ response_headers["x-headroom-tokens-before"] = str(original_tokens)
+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
+ response_headers["x-headroom-model"] = model
+ if transforms_applied:
+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
+ if cache_read_tokens > 0:
+ response_headers["x-headroom-cached"] = "true"
+ if _compression_failed:
+ response_headers["x-headroom-compression-failed"] = "true"
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed()
+ # Log full error details internally for debugging
+ logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
+ # Return sanitized error message to client (don't expose internal details)
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "type": "server_error",
+ "code": "proxy_error",
+ }
+ },
+ )
+
+ async def handle_openai_responses(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle OpenAI /v1/responses endpoint (new Responses API).
+
+ The Responses API differs from /v1/chat/completions:
+ - Input: `input` (string or array) instead of `messages`
+ - System: `instructions` instead of system message
+ - Output: `output[]` array instead of `choices[].message`
+ - State: `previous_response_id` for multi-turn
+ - Built-in tools: web_search, file_search, code_interpreter
+ """
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS,
+ MAX_REQUEST_BODY_SIZE,
+ _read_request_json,
+ )
+ from headroom.proxy.responses_converter import (
+ messages_to_responses_items,
+ responses_items_to_messages,
+ )
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+
+ model = body.get("model", "unknown")
+ stream = body.get("stream", False)
+
+ # Convert Responses API input to messages format for optimization.
+ # The Responses API uses a different item model (function_call,
+ # function_call_output, reasoning as top-level items) — we convert to
+ # Chat Completions messages for the pipeline, then convert back.
+
+ input_data = body.get("input", "")
+ instructions = body.get("instructions")
+ previous_response_id = body.get("previous_response_id")
+
+ messages: list[dict[str, Any]] = []
+ original_items: list[dict[str, Any]] | None = None
+ preserved_indices: list[int] = []
+
+ if instructions:
+ messages.append({"role": "system", "content": instructions})
+
+ if isinstance(input_data, str):
+ messages.append({"role": "user", "content": input_data})
+ elif isinstance(input_data, list):
+ original_items = input_data
+ converted, preserved_indices = responses_items_to_messages(input_data)
+ messages.extend(converted)
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ tags = self._extract_tags(headers)
+
+ # Rate limiting
+ if self.rate_limiter:
+ rate_key = headers.get("authorization", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited()
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Token counting on converted messages
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Optimize: convert items → compress → convert back
+ tokens_saved = 0
+ transforms_applied: list[str] = []
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ _should_compress = (
+ self.config.optimize
+ and original_items is not None
+ and not previous_response_id
+ and not _bypass
+ and len(messages) > 1
+ )
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+
+ if _should_compress and _license_ok:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+ except Exception as e:
+ logger.warning(f"[{request_id}] Responses API optimization failed: {e}")
+
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Convert compressed messages back to Responses API items
+ if optimized_messages is not messages and original_items is not None:
+ opt_msgs = optimized_messages
+ # Strip system message (instructions) — it's separate in Responses API
+ if instructions and opt_msgs and opt_msgs[0].get("role") == "system":
+ body["instructions"] = opt_msgs[0]["content"]
+ opt_msgs = opt_msgs[1:]
+
+ body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)
+
+ # /v1/responses is OpenAI-specific (Codex) — always routes direct.
+ # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.
+ if self.anthropic_backend is not None:
+ logger.debug(
+ f"[{request_id}] /v1/responses always routes to OpenAI direct "
+ f"(backend '{self.anthropic_backend.name}' not used for Responses API)"
+ )
+
+ url = f"{self.OPENAI_API_URL}/v1/responses"
+
+ try:
+ if stream:
+ # Streaming for Responses API uses semantic events
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "openai",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ )
+ else:
+ response = await self._retry_request("POST", url, headers, body)
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = original_tokens # fallback
+ output_tokens = 0
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usage", {})
+ total_input_tokens = usage.get("input_tokens", original_tokens)
+ output_tokens = usage.get("output_tokens", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)
+
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ )
+
+ logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed()
+ logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "type": "server_error",
+ "code": "proxy_error",
+ }
+ },
+ )
+
+ async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:
+ """WebSocket proxy for /v1/responses (Codex gpt-5.4+).
+
+ Newer Codex versions use WebSocket instead of HTTP POST for the
+ Responses API. This handler:
+ 1. Accepts the client WebSocket
+ 2. Receives the first message (``response.create`` request)
+ 3. Compresses the ``input`` array using the existing pipeline
+ 4. Opens an upstream WebSocket to OpenAI
+ 5. Sends the compressed request upstream
+ 6. Relays all subsequent messages bidirectionally
+ """
+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ try:
+ import websockets
+ except ImportError:
+ await websocket.accept()
+ await websocket.close(
+ code=1011,
+ reason="websockets package not installed. pip install websockets",
+ )
+ return
+
+ request_id = await self._next_request_id()
+
+ # Forward client headers to upstream, adding required OpenAI-Beta header
+ ws_headers = dict(websocket.headers)
+
+ # Extract subprotocol from client — this is an application-level negotiation
+ # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
+ # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.
+ client_subprotocols: list[str] = []
+ raw_protocol = ws_headers.get("sec-websocket-protocol", "")
+ if raw_protocol:
+ client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]
+
+ # Accept client connection with the requested subprotocol
+ if client_subprotocols:
+ await websocket.accept(subprotocol=client_subprotocols[0])
+ else:
+ await websocket.accept()
+
+ # Build upstream WebSocket URL (http→ws, https→wss)
+ base = self.OPENAI_API_URL
+ ws_base = base.replace("https://", "wss://").replace("http://", "ws://")
+ upstream_url = f"{ws_base}/v1/responses"
+
+ # Forward all client headers except hop-by-hop / per-connection headers.
+ # These are WebSocket handshake mechanics that the `websockets` library
+ # generates fresh for the upstream connection — forwarding them would conflict.
+ # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.
+ _skip_headers = frozenset(
+ {
+ "host", # must match upstream, not local proxy
+ "connection", # hop-by-hop
+ "upgrade", # hop-by-hop
+ "sec-websocket-key", # per-connection cryptographic nonce
+ "sec-websocket-version", # protocol version (websockets lib sets this)
+ "sec-websocket-extensions", # per-connection negotiation
+ "sec-websocket-accept", # server-side only
+ "sec-websocket-protocol", # handled via subprotocols param below
+ "content-length", # hop-by-hop
+ "transfer-encoding", # hop-by-hop
+ }
+ )
+ upstream_headers: dict[str, str] = {}
+ for k, v in ws_headers.items():
+ if k.lower() not in _skip_headers:
+ upstream_headers[k] = v
+
+ # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.
+ # Safety net for clients that don't forward auth headers via WebSocket upgrade.
+ _has_auth = "authorization" in {k.lower() for k in upstream_headers}
+ if not _has_auth:
+ api_key = os.environ.get("OPENAI_API_KEY")
+ if api_key:
+ upstream_headers["Authorization"] = f"Bearer {api_key}"
+ logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")
+ else:
+ logger.warning(
+ f"[{request_id}] WS: no Authorization header from client and "
+ f"OPENAI_API_KEY not set — upstream will likely reject"
+ )
+
+ # Ensure the required beta header is present — OpenAI returns 500 without it
+ if "openai-beta" not in {k.lower() for k in upstream_headers}:
+ upstream_headers["OpenAI-Beta"] = "responses-api=v1"
+
+ logger.debug(
+ f"[{request_id}] WS upstream headers: "
+ f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "
+ f"subprotocols={client_subprotocols}"
+ )
+
+ try:
+ # Receive the first message from client (the response.create request)
+ first_msg_raw = await websocket.receive_text()
+
+ # --- Optional: compress the input in the first message ---
+ try:
+ body = json.loads(first_msg_raw)
+ input_data = body.get("input")
+ tokens_saved = 0
+
+ should_compress = (
+ self.config.optimize
+ and isinstance(input_data, list)
+ and len(input_data) > 1
+ and not body.get("previous_response_id")
+ )
+ if should_compress:
+ try:
+ from headroom.proxy.responses_converter import (
+ messages_to_responses_items,
+ responses_items_to_messages,
+ )
+
+ model = body.get("model", "gpt-4o")
+ converted, preserved = responses_items_to_messages(input_data)
+
+ messages: list[dict[str, Any]] = []
+ instructions = body.get("instructions")
+ if instructions:
+ messages.append({"role": "system", "content": instructions})
+ messages.extend(converted)
+
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ opt = result.messages
+ if instructions and opt and opt[0].get("role") == "system":
+ body["instructions"] = opt[0]["content"]
+ opt = opt[1:]
+ body["input"] = messages_to_responses_items(opt, input_data, preserved)
+ tokens_saved = max(0, original_tokens - result.tokens_after)
+ first_msg_raw = json.dumps(body)
+ logger.info(
+ f"[{request_id}] WS /v1/responses compressed: "
+ f"saved {tokens_saved} tokens"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] WS compression failed: {e}")
+
+ except json.JSONDecodeError:
+ # Not JSON — pass through as-is
+ tokens_saved = 0
+
+ # --- Connect to upstream OpenAI WebSocket ---
+ logger.debug(f"[{request_id}] WS connecting to {upstream_url}")
+
+ # Use ssl=True to let the websockets library handle SSL natively.
+ # Manual ssl.create_default_context() + certifi doesn't load the
+ # Windows system cert store, causing HTTP 500 on wss:// connections.
+ use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
+
+ async with websockets.connect(
+ upstream_url,
+ additional_headers=upstream_headers,
+ subprotocols=(
+ [websockets.Subprotocol(p) for p in client_subprotocols]
+ if client_subprotocols and hasattr(websockets, "Subprotocol")
+ else client_subprotocols or None
+ ),
+ ssl=use_ssl,
+ ) as upstream:
+ # Send (potentially compressed) first message
+ await upstream.send(first_msg_raw)
+
+ # Bidirectional relay
+ async def _client_to_upstream() -> None:
+ try:
+ while True:
+ msg = await websocket.receive_text()
+ await upstream.send(msg)
+ except Exception as relay_err:
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ logger.debug(
+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"
+ )
+ with contextlib.suppress(Exception):
+ await upstream.close()
+
+ async def _upstream_to_client() -> None:
+ try:
+ async for msg in upstream:
+ if isinstance(msg, str):
+ await websocket.send_text(msg)
+ elif isinstance(msg, bytes):
+ await websocket.send_bytes(msg)
+ else:
+ await websocket.send_text(str(msg))
+ except Exception as relay_err:
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ logger.debug(
+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"
+ )
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close()
+
+ await asyncio.gather(
+ _client_to_upstream(),
+ _upstream_to_client(),
+ return_exceptions=True,
+ )
+
+ # Record metrics
+ if tokens_saved > 0:
+ model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"
+ await self.metrics.record_request(
+ provider="openai",
+ model=model_name,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=tokens_saved,
+ latency_ms=0,
+ )
+
+ except Exception as e:
+ if "WebSocketDisconnect" not in type(e).__name__:
+ # Extract response body from websockets InvalidStatus for better debugging
+ error_detail = str(e)
+ if hasattr(e, "response"):
+ try:
+ resp = e.response
+ body_bytes = getattr(resp, "body", None) or b""
+ if body_bytes:
+ error_detail += (
+ f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"
+ )
+ except Exception:
+ pass
+ logger.error(f"[{request_id}] WS proxy error: {error_detail}")
+ with contextlib.suppress(Exception):
+ await websocket.close(code=1011, reason=str(e)[:120])
+
+ async def handle_compress(self, request: Request) -> JSONResponse:
+ """Compress messages without calling an LLM.
+
+ POST /v1/compress
+ Body: {"messages": [...], "model": "...", "config": {}}
+ Returns compressed messages + metrics.
+ """
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+
+ # Check bypass header
+ if request.headers.get("x-headroom-bypass", "").lower() == "true":
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={"error": f"Invalid request body: {e!s}"},
+ )
+ messages = body.get("messages", [])
+ return JSONResponse(
+ {
+ "messages": messages,
+ "tokens_before": 0,
+ "tokens_after": 0,
+ "tokens_saved": 0,
+ "compression_ratio": 1.0,
+ "transforms_applied": [],
+ "ccr_hashes": [],
+ }
+ )
+
+ try:
+ body = await _read_request_json(request)
+ except Exception:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Invalid JSON in request body.",
+ }
+ },
+ )
+
+ messages = body.get("messages")
+ model = body.get("model")
+
+ if messages is None:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Missing required field: messages",
+ }
+ },
+ )
+
+ if model is None:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Missing required field: model",
+ }
+ },
+ )
+
+ if not messages:
+ return JSONResponse(
+ {
+ "messages": [],
+ "tokens_before": 0,
+ "tokens_after": 0,
+ "tokens_saved": 0,
+ "compression_ratio": 1.0,
+ "transforms_applied": [],
+ "ccr_hashes": [],
+ }
+ )
+
+ try:
+ # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)
+ # Allow optional token_budget to override model's context limit
+ # (used by OpenClaw compact() and other callers that need tighter budgets)
+ token_budget = body.get("token_budget")
+ context_limit = (
+ token_budget
+ if token_budget and isinstance(token_budget, int)
+ else self.openai_provider.get_context_limit(model)
+ )
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ )
+
+ return JSONResponse(
+ {
+ "messages": result.messages,
+ "tokens_before": result.tokens_before,
+ "tokens_after": result.tokens_after,
+ "tokens_saved": result.tokens_before - result.tokens_after,
+ "compression_ratio": (
+ result.tokens_after / result.tokens_before
+ if result.tokens_before > 0
+ else 1.0
+ ),
+ "transforms_applied": result.transforms_applied,
+ "transforms_summary": result.transforms_summary,
+ "ccr_hashes": result.markers_inserted,
+ }
+ )
+ except Exception as e:
+ logger.exception("Compression failed: %s", e)
+ return JSONResponse(
+ status_code=503,
+ content={
+ "error": {
+ "type": "compression_error",
+ "message": str(e),
+ }
+ },
+ )
+
+ async def handle_passthrough(
+ self,
+ request: Request,
+ base_url: str,
+ endpoint_name: str | None = None,
+ provider: str | None = None,
+ ) -> Response:
+ """Pass through request unchanged.
+
+ Args:
+ request: The incoming request
+ base_url: The upstream API base URL
+ endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")
+ provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")
+ """
+ from fastapi.responses import Response
+
+ start_time = time.time()
+ path = request.url.path
+ url = f"{base_url}{path}"
+
+ # Preserve query string parameters
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+
+ body = await request.body()
+
+ response = await self.http_client.request( # type: ignore[union-attr]
+ method=request.method,
+ url=url,
+ headers=headers,
+ content=body,
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Track stats for passthrough requests
+ if endpoint_name and provider:
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider=provider,
+ model=f"passthrough:{endpoint_name}",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ cached=False,
+ )
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def handle_databricks_invocations(
+ self,
+ request: Request,
+ model: str,
+ ) -> Response | StreamingResponse:
+ """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.
+
+ This enables using the Databricks CLI directly with Headroom:
+ databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'
+
+ The request/response format is identical to OpenAI chat completions,
+ so we inject the model from the path and delegate to handle_openai_chat.
+ """
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+
+ request_id = await self._next_request_id()
+
+ try:
+ body = await _read_request_json(request)
+ except Exception as e:
+ logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {"message": f"Invalid JSON: {e}", "type": "invalid_request_error"}
+ },
+ )
+
+ # Inject model from path into body (Databricks CLI passes model in URL, not body)
+ body["model"] = model
+
+ logger.info(f"[{request_id}] Databricks invocation: model={model}")
+
+ # Create a new request with the modified body
+ # We reuse the OpenAI chat handler since the format is identical
+ from starlette.requests import Request as StarletteRequest
+
+ # Build new scope with the body already parsed
+ scope = dict(request.scope)
+
+ # Create a simple receive function that returns our modified body
+ body_bytes = json.dumps(body).encode()
+
+ async def receive():
+ return {"type": "http.request", "body": body_bytes}
+
+ modified_request = StarletteRequest(scope, receive)
+
+ # Delegate to the OpenAI chat handler (same format)
+ return await self.handle_openai_chat(modified_request)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
new file mode 100644
index 000000000..ad9f7ed0a
--- /dev/null
+++ b/headroom/proxy/handlers/streaming.py
@@ -0,0 +1,901 @@
+"""Streaming handler mixin for HeadroomProxy.
+
+Contains SSE parsing, streaming response generation, and related utilities.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import time
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from fastapi.responses import StreamingResponse
+
+
+import httpx
+
+logger = logging.getLogger("headroom.proxy")
+
+
+class StreamingMixin:
+ """Mixin providing streaming response methods for HeadroomProxy."""
+
+ def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
+ """Parse usage information from SSE chunk.
+
+ For Anthropic: Looks for message_start (input tokens) and message_delta (output tokens)
+ For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)
+ For Gemini: Looks for usageMetadata in each chunk
+
+ Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens
+ Returns None if no usage found in this chunk.
+ """
+ try:
+ text = chunk.decode("utf-8", errors="ignore")
+ # SSE format: "data: {...}\n\n" or "event: ...\ndata: {...}\n\n"
+ for line in text.split("\n"):
+ if not line.startswith("data: "):
+ continue
+ data_str = line[6:].strip()
+ if not data_str or data_str == "[DONE]":
+ continue
+
+ try:
+ data = json.loads(data_str)
+ except json.JSONDecodeError:
+ continue
+
+ usage = {}
+
+ if provider == "anthropic":
+ # Anthropic sends message_start with input tokens
+ # and message_delta with output tokens
+ event_type = data.get("type", "")
+
+ if event_type == "message_start":
+ msg = data.get("message", {})
+ msg_usage = msg.get("usage", {})
+ if msg_usage:
+ usage["input_tokens"] = msg_usage.get("input_tokens", 0)
+ usage["cache_read_input_tokens"] = msg_usage.get(
+ "cache_read_input_tokens", 0
+ )
+ usage["cache_creation_input_tokens"] = msg_usage.get(
+ "cache_creation_input_tokens", 0
+ )
+
+ elif event_type == "message_delta":
+ delta_usage = data.get("usage", {})
+ if delta_usage:
+ usage["output_tokens"] = delta_usage.get("output_tokens", 0)
+
+ elif provider == "openai":
+ # OpenAI sends usage in final chunk (when stream_options.include_usage=true)
+ chunk_usage = data.get("usage")
+ if chunk_usage:
+ usage["input_tokens"] = chunk_usage.get("prompt_tokens", 0)
+ usage["output_tokens"] = chunk_usage.get("completion_tokens", 0)
+ # OpenAI has cached tokens in prompt_tokens_details
+ details = chunk_usage.get("prompt_tokens_details", {})
+ usage["cache_read_input_tokens"] = details.get("cached_tokens", 0)
+
+ elif provider == "gemini":
+ # Gemini sends usageMetadata in each streaming chunk
+ # Format: {"usageMetadata": {"promptTokenCount": N, "candidatesTokenCount": M}}
+ usage_meta = data.get("usageMetadata")
+ if usage_meta:
+ usage["input_tokens"] = usage_meta.get("promptTokenCount", 0)
+ usage["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)
+ # Gemini also has cachedContentTokenCount for context caching
+ usage["cache_read_input_tokens"] = usage_meta.get(
+ "cachedContentTokenCount", 0
+ )
+
+ if usage:
+ return usage
+
+ except (UnicodeDecodeError, KeyError, TypeError) as e:
+ # Don't fail streaming on parse errors
+ logger.debug(f"SSE usage parsing error for {provider}: {e}")
+
+ return None
+
+ def _parse_sse_usage_from_buffer(
+ self, stream_state: dict[str, Any], provider: str
+ ) -> dict[str, int] | None:
+ """Parse usage from buffered SSE data, handling split chunks.
+
+ Processes complete SSE events (ending with double newline) from the buffer
+ and removes them from the buffer. Incomplete events are kept in the buffer
+ for the next chunk.
+ """
+ buffer = stream_state["sse_buffer"]
+ usage_found: dict[str, int] = {}
+
+ # Process complete SSE events (separated by double newlines)
+ while "\n\n" in buffer:
+ event_end = buffer.index("\n\n")
+ event_text = buffer[: event_end + 2]
+ buffer = buffer[event_end + 2 :]
+
+ # Parse this complete event
+ for line in event_text.split("\n"):
+ if not line.startswith("data: "):
+ continue
+ data_str = line[6:].strip()
+ if not data_str or data_str == "[DONE]":
+ continue
+
+ try:
+ data = json.loads(data_str)
+ except json.JSONDecodeError:
+ continue
+
+ if provider == "anthropic":
+ event_type = data.get("type", "")
+ if event_type == "message_start":
+ msg = data.get("message", {})
+ msg_usage = msg.get("usage", {})
+ if msg_usage:
+ usage_found["input_tokens"] = msg_usage.get("input_tokens", 0)
+ usage_found["cache_read_input_tokens"] = msg_usage.get(
+ "cache_read_input_tokens", 0
+ )
+ usage_found["cache_creation_input_tokens"] = msg_usage.get(
+ "cache_creation_input_tokens", 0
+ )
+ # INFO logging for cache token tracking (temporary for debugging)
+ logger.info(
+ f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "
+ f"cache_read={usage_found.get('cache_read_input_tokens')}, "
+ f"cache_write={usage_found.get('cache_creation_input_tokens')}"
+ )
+ elif event_type == "message_delta":
+ delta_usage = data.get("usage", {})
+ if delta_usage:
+ usage_found["output_tokens"] = delta_usage.get("output_tokens", 0)
+
+ elif provider == "openai":
+ chunk_usage = data.get("usage")
+ if chunk_usage:
+ usage_found["input_tokens"] = chunk_usage.get("prompt_tokens", 0)
+ usage_found["output_tokens"] = chunk_usage.get("completion_tokens", 0)
+ details = chunk_usage.get("prompt_tokens_details", {})
+ usage_found["cache_read_input_tokens"] = details.get("cached_tokens", 0)
+
+ elif provider == "gemini":
+ usage_meta = data.get("usageMetadata")
+ if usage_meta:
+ usage_found["input_tokens"] = usage_meta.get("promptTokenCount", 0)
+ usage_found["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)
+ usage_found["cache_read_input_tokens"] = usage_meta.get(
+ "cachedContentTokenCount", 0
+ )
+
+ # Update buffer with remaining incomplete data
+ stream_state["sse_buffer"] = buffer
+
+ return usage_found if usage_found else None
+
+ def _parse_sse_to_response(self, sse_data: str, provider: str) -> dict[str, Any] | None:
+ """Parse SSE data to reconstruct the API response JSON.
+
+ Args:
+ sse_data: Raw SSE data string.
+ provider: Provider type for parsing.
+
+ Returns:
+ Reconstructed response dict or None if parsing fails.
+ """
+ if provider != "anthropic":
+ return None # Only implemented for Anthropic
+
+ response: dict[str, Any] = {"content": [], "usage": {}}
+ current_block: dict[str, Any] | None = None
+
+ for line in sse_data.split("\n"):
+ if not line.startswith("data: "):
+ continue
+ data_str = line[6:].strip()
+ if not data_str or data_str == "[DONE]":
+ continue
+
+ try:
+ data = json.loads(data_str)
+ except json.JSONDecodeError:
+ continue
+
+ event_type = data.get("type", "")
+
+ if event_type == "message_start":
+ msg = data.get("message", {})
+ response["id"] = msg.get("id")
+ response["model"] = msg.get("model")
+ response["role"] = msg.get("role", "assistant")
+ response["stop_reason"] = msg.get("stop_reason")
+ if msg.get("usage"):
+ response["usage"].update(msg["usage"])
+
+ elif event_type == "content_block_start":
+ block = data.get("content_block", {})
+ current_block = {
+ "type": block.get("type"),
+ "index": data.get("index", len(response["content"])),
+ }
+ if block.get("type") == "text":
+ current_block["text"] = block.get("text", "")
+ elif block.get("type") == "tool_use":
+ current_block["id"] = block.get("id")
+ current_block["name"] = block.get("name")
+ current_block["input"] = {}
+
+ elif event_type == "content_block_delta":
+ if current_block:
+ delta = data.get("delta", {})
+ if delta.get("type") == "text_delta":
+ current_block["text"] = current_block.get("text", "") + delta.get(
+ "text", ""
+ )
+ elif delta.get("type") == "input_json_delta":
+ # Accumulate partial JSON for tool input
+ partial = delta.get("partial_json", "")
+ current_block["_partial_json"] = (
+ current_block.get("_partial_json", "") + partial
+ )
+
+ elif event_type == "content_block_stop":
+ if current_block:
+ # Parse accumulated JSON for tool_use blocks
+ if current_block.get("type") == "tool_use" and "_partial_json" in current_block:
+ try:
+ current_block["input"] = json.loads(current_block["_partial_json"])
+ except json.JSONDecodeError:
+ current_block["input"] = {}
+ del current_block["_partial_json"]
+ response["content"].append(current_block)
+ current_block = None
+
+ elif event_type == "message_delta":
+ delta = data.get("delta", {})
+ if delta.get("stop_reason"):
+ response["stop_reason"] = delta["stop_reason"]
+ if data.get("usage"):
+ response["usage"].update(data["usage"])
+
+ return response if response.get("content") else None
+
+ def _response_to_sse(self, response: dict[str, Any], provider: str) -> list[bytes]:
+ """Convert a response dict back to SSE format.
+
+ Args:
+ response: API response dict.
+ provider: Provider type for formatting.
+
+ Returns:
+ List of SSE event bytes.
+ """
+ if provider != "anthropic":
+ return []
+
+ events: list[bytes] = []
+
+ # message_start
+ msg_start = {
+ "type": "message_start",
+ "message": {
+ "id": response.get("id", "msg_generated"),
+ "type": "message",
+ "role": response.get("role", "assistant"),
+ "model": response.get("model", "unknown"),
+ "content": [],
+ "stop_reason": None,
+ "usage": response.get("usage", {}),
+ },
+ }
+ events.append(f"event: message_start\ndata: {json.dumps(msg_start)}\n\n".encode())
+
+ # Content blocks
+ for idx, block in enumerate(response.get("content", [])):
+ # content_block_start
+ if block.get("type") == "text":
+ block_start = {
+ "type": "content_block_start",
+ "index": idx,
+ "content_block": {"type": "text", "text": ""},
+ }
+ elif block.get("type") == "tool_use":
+ block_start = {
+ "type": "content_block_start",
+ "index": idx,
+ "content_block": {
+ "type": "tool_use",
+ "id": block.get("id", f"toolu_{idx}"),
+ "name": block.get("name", ""),
+ "input": {},
+ },
+ }
+ else:
+ continue
+
+ events.append(
+ f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n".encode()
+ )
+
+ # content_block_delta(s)
+ if block.get("type") == "text" and block.get("text"):
+ delta = {
+ "type": "content_block_delta",
+ "index": idx,
+ "delta": {"type": "text_delta", "text": block["text"]},
+ }
+ events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())
+ elif block.get("type") == "tool_use" and block.get("input"):
+ delta = {
+ "type": "content_block_delta",
+ "index": idx,
+ "delta": {
+ "type": "input_json_delta",
+ "partial_json": json.dumps(block["input"]),
+ },
+ }
+ events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())
+
+ # content_block_stop
+ block_stop = {"type": "content_block_stop", "index": idx}
+ events.append(f"event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n".encode())
+
+ # message_delta
+ msg_delta = {
+ "type": "message_delta",
+ "delta": {"stop_reason": response.get("stop_reason", "end_turn")},
+ "usage": {"output_tokens": response.get("usage", {}).get("output_tokens", 0)},
+ }
+ events.append(f"event: message_delta\ndata: {json.dumps(msg_delta)}\n\n".encode())
+
+ # message_stop
+ events.append(b'event: message_stop\ndata: {"type": "message_stop"}\n\n')
+
+ return events
+
+ def _record_ccr_feedback_from_response(
+ self, response: dict, provider: str, request_id: str
+ ) -> None:
+ """Extract headroom_retrieve tool calls from a response and record feedback.
+
+ This closes the TOIN feedback loop for streaming responses where
+ the proxy can't intercept and handle retrieval calls inline.
+ """
+ from headroom.cache.compression_store import get_compression_store
+
+ content = response.get("content", [])
+ if not isinstance(content, list):
+ return
+
+ store = get_compression_store()
+
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") != "tool_use":
+ continue
+ if block.get("name") != "headroom_retrieve":
+ continue
+
+ input_data = block.get("input", {})
+ hash_key = input_data.get("hash")
+ query = input_data.get("query")
+
+ if not hash_key:
+ continue
+
+ logger.info(
+ f"[{request_id}] CCR Feedback: Recording retrieval "
+ f"hash={hash_key[:8]}... query={query!r}"
+ )
+
+ # Call store.retrieve()/search() for the side effect of triggering
+ # the feedback chain: _log_retrieval -> process_pending_feedback
+ # -> toin.record_retrieval(). We discard the returned content.
+ try:
+ if query:
+ store.search(hash_key, query)
+ else:
+ store.retrieve(hash_key, query=None)
+ except Exception as e:
+ logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")
+
+ async def _stream_response(
+ self,
+ url: str,
+ headers: dict,
+ body: dict,
+ provider: str,
+ model: str,
+ request_id: str,
+ original_tokens: int,
+ optimized_tokens: int,
+ tokens_saved: int,
+ transforms_applied: list[str],
+ tags: dict[str, str],
+ optimization_latency: float,
+ memory_user_id: str | None = None,
+ pipeline_timing: dict[str, float] | None = None,
+ prefix_tracker: Any | None = None,
+ ) -> StreamingResponse:
+ """Stream response with metrics tracking and memory tool handling.
+
+ Parses SSE events to extract actual usage information from the API response
+ for accurate token counting and cost calculation.
+
+ When memory is enabled (memory_user_id provided), this method:
+ 1. Buffers the response to detect memory tool calls
+ 2. Executes memory tools if found
+ 3. Makes continuation requests until no memory tools remain
+ 4. Streams the final response to the client
+ """
+ from fastapi.responses import StreamingResponse
+
+ from headroom.proxy.cost import _summarize_transforms
+ from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE
+
+ start_time = time.time()
+
+ # Mutable state for the generator to update
+ stream_state: dict[str, Any] = {
+ "input_tokens": None,
+ "output_tokens": None,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ "total_bytes": 0,
+ "sse_buffer": "", # Buffer for incomplete SSE events
+ "ttfb_ms": None, # Time to first byte from upstream
+ }
+
+ # Track if we need to handle memory tools
+ memory_enabled = (
+ memory_user_id is not None
+ and self.memory_handler is not None
+ and provider == "anthropic"
+ )
+
+ # Open connection before generator to capture upstream response headers
+ # (needed to forward ratelimit headers to the client via StreamingResponse)
+ assert self.http_client is not None, "http_client must be initialized before streaming"
+ try:
+ _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)
+ upstream_response = await self.http_client.send(_upstream_req, stream=True)
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
+ error_msg = str(e)
+ logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")
+
+ async def _error_gen():
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "connection_error",
+ "message": f"Failed to connect to upstream API: {error_msg}",
+ },
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
+
+ return StreamingResponse(_error_gen(), media_type="text/event-stream")
+
+ # Forward upstream ratelimit headers to the client
+ forwarded_headers = {
+ k: v for k, v in upstream_response.headers.items() if "ratelimit" in k.lower()
+ }
+
+ async def generate():
+ nonlocal body, memory_enabled # May need to modify for continuation requests
+
+ # For memory mode, we buffer the response to check for tool calls
+ buffered_chunks: list[bytes] = []
+ full_sse_data = ""
+
+ try:
+ async with contextlib.aclosing(upstream_response) as response:
+ async for chunk in response.aiter_bytes():
+ # Record TTFB on first chunk
+ if stream_state["ttfb_ms"] is None:
+ stream_state["ttfb_ms"] = (time.time() - start_time) * 1000
+
+ stream_state["total_bytes"] += len(chunk)
+
+ # Buffer SSE data to handle chunks split across calls
+ chunk_str = chunk.decode("utf-8", errors="ignore")
+ stream_state["sse_buffer"] += chunk_str
+
+ # Safety: prevent unbounded buffer growth
+ if len(stream_state["sse_buffer"]) > MAX_SSE_BUFFER_SIZE:
+ logger.error(
+ "SSE buffer exceeded maximum size (%d bytes), "
+ "truncating to prevent memory exhaustion",
+ MAX_SSE_BUFFER_SIZE,
+ )
+ stream_state["sse_buffer"] = stream_state["sse_buffer"][
+ -MAX_SSE_BUFFER_SIZE // 2 :
+ ]
+
+ # Always stream immediately — buffering breaks
+ # real-time clients (LangGraph, LangChain, etc.)
+ yield chunk
+
+ if memory_enabled:
+ # Also buffer for post-stream memory processing
+ buffered_chunks.append(chunk)
+ full_sse_data += chunk_str
+ if len(full_sse_data) > MAX_SSE_BUFFER_SIZE:
+ logger.warning(
+ "Memory-mode SSE buffer exceeded maximum size, "
+ "disabling memory detection for this request"
+ )
+ memory_enabled = False
+
+ # Parse complete SSE events from buffer
+ usage = self._parse_sse_usage_from_buffer(stream_state, provider)
+ if usage:
+ if "input_tokens" in usage:
+ stream_state["input_tokens"] = usage["input_tokens"]
+ if "output_tokens" in usage:
+ stream_state["output_tokens"] = usage["output_tokens"]
+ if "cache_read_input_tokens" in usage:
+ stream_state["cache_read_input_tokens"] = usage[
+ "cache_read_input_tokens"
+ ]
+ if "cache_creation_input_tokens" in usage:
+ stream_state["cache_creation_input_tokens"] = usage[
+ "cache_creation_input_tokens"
+ ]
+
+ # Memory tool handling after stream completes
+ # Chunks were already yielded in real-time above, so we only
+ # do silent background processing here — no yielding.
+ if memory_enabled and full_sse_data:
+ # Check for Claude Code credential error
+ if "only authorized for use with Claude Code" in full_sse_data:
+ logger.warning(
+ f"[{request_id}] Memory: Claude Code subscription credentials "
+ "do not support custom tool injection. Set ANTHROPIC_API_KEY "
+ "environment variable or use --no-memory-tools flag."
+ )
+ return
+
+ # Parse SSE to get response JSON
+ parsed_response = self._parse_sse_to_response(full_sse_data, provider)
+
+ if parsed_response and self.memory_handler.has_memory_tool_calls(
+ parsed_response, provider
+ ):
+ logger.info(
+ f"[{request_id}] Memory: Detected tool calls in streaming response"
+ )
+
+ # Execute memory tool calls silently — response already
+ # streamed so we cannot make a continuation request.
+ tool_results = await self.memory_handler.handle_memory_tool_calls(
+ parsed_response, memory_user_id, provider
+ )
+ if tool_results:
+ logger.info(
+ f"[{request_id}] Memory: Tool calls executed silently "
+ "(streaming mode — no continuation)"
+ )
+
+ # CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.
+ # In streaming mode, the client handles actual retrieval, but we
+ # still need to record the event so TOIN learns which fields matter.
+ if self.config.ccr_inject_tool and full_sse_data:
+ ccr_parsed = (
+ parsed_response
+ if parsed_response
+ else self._parse_sse_to_response(full_sse_data, provider)
+ )
+ if ccr_parsed:
+ self._record_ccr_feedback_from_response(ccr_parsed, provider, request_id)
+
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
+ logger.error(f"[{request_id}] Connection error to upstream API: {e}")
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "connection_error",
+ "message": f"Failed to connect to upstream API: {e}",
+ },
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
+ except httpx.HTTPStatusError as e:
+ logger.error(f"[{request_id}] HTTP error from upstream API: {e}")
+ # Forward the upstream error response
+ yield e.response.content
+ except Exception as e:
+ logger.error(f"[{request_id}] Unexpected streaming error: {e}")
+ error_event = {
+ "type": "error",
+ "error": {"type": "api_error", "message": str(e)},
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
+ finally:
+ # Record metrics after stream completes
+ total_latency = (time.time() - start_time) * 1000
+
+ # Use actual output tokens from API if available, otherwise estimate
+ output_tokens = stream_state["output_tokens"]
+ if output_tokens is None:
+ # Fallback: estimate from bytes (but this is inaccurate for SSE)
+ # Use a more conservative estimate - SSE overhead is ~10-20x
+ output_tokens = stream_state["total_bytes"] // 40
+ logger.debug(
+ f"[{request_id}] No usage in stream, estimated {output_tokens} output tokens"
+ )
+
+ # Use optimized_tokens for dashboard metrics (what we actually sent).
+ # API's input_tokens is the non-cached portion only, which is
+ # misleading for aggregation (often just 1 with prompt caching).
+ cache_read_tokens = stream_state["cache_read_input_tokens"]
+ cache_write_tokens = stream_state["cache_creation_input_tokens"]
+ uncached_input_tokens = stream_state.get("input_tokens") or 0
+
+ # Structured perf log line for `headroom perf` analysis
+ num_msgs = len(body.get("messages", []))
+ cache_hit_pct = (
+ round(cache_read_tokens / (cache_read_tokens + cache_write_tokens) * 100)
+ if (cache_read_tokens + cache_write_tokens) > 0
+ else 0
+ )
+ logger.info(
+ f"[{request_id}] PERF "
+ f"model={model} msgs={num_msgs} "
+ f"tok_before={original_tokens} tok_after={optimized_tokens} "
+ f"tok_saved={tokens_saved} "
+ f"cache_read={cache_read_tokens} cache_write={cache_write_tokens} "
+ f"cache_hit_pct={cache_hit_pct} "
+ f"opt_ms={optimization_latency:.0f} "
+ f"transforms={_summarize_transforms(transforms_applied)}"
+ )
+
+ # Update prefix cache tracker for next turn (streaming path)
+ if prefix_tracker is not None:
+ prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ messages=body.get("messages", []),
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(
+ model,
+ tokens_saved,
+ optimized_tokens,
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ uncached_tokens=uncached_input_tokens,
+ )
+
+ await self.metrics.record_request(
+ provider=provider,
+ model=model,
+ input_tokens=optimized_tokens, # What we sent, not API's non-cached count
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ ttfb_ms=stream_state["ttfb_ms"] or 0,
+ pipeline_timing=pipeline_timing,
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=cache_write_tokens,
+ uncached_input_tokens=uncached_input_tokens,
+ )
+
+ return StreamingResponse(
+ generate(),
+ media_type="text/event-stream",
+ headers=forwarded_headers,
+ )
+
+ async def _stream_response_bedrock(
+ self,
+ body: dict,
+ headers: dict,
+ provider: str,
+ model: str,
+ request_id: str,
+ original_tokens: int,
+ optimized_tokens: int,
+ tokens_saved: int,
+ transforms_applied: list[str],
+ tags: dict[str, str],
+ optimization_latency: float,
+ pipeline_timing: dict[str, float] | None = None,
+ ) -> StreamingResponse:
+ """Stream response from Bedrock backend with metrics tracking.
+
+ Translates Bedrock streaming events to Anthropic SSE format.
+ """
+ from fastapi.responses import StreamingResponse
+
+ from headroom.proxy.cost import _summarize_transforms
+ from headroom.proxy.models import RequestLog
+
+ start_time = time.time()
+
+ # Mutable state for the generator
+ stream_state: dict[str, Any] = {
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "ttfb_ms": None,
+ }
+
+ async def generate():
+ try:
+ assert self.anthropic_backend is not None
+
+ async for event in self.anthropic_backend.stream_message(body, headers):
+ # Record TTFB on first event
+ if stream_state["ttfb_ms"] is None:
+ stream_state["ttfb_ms"] = (time.time() - start_time) * 1000
+
+ # Format as SSE
+ if event.raw_sse:
+ yield event.raw_sse.encode()
+ else:
+ sse_line = f"event: {event.event_type}\ndata: {json.dumps(event.data)}\n\n"
+ yield sse_line.encode()
+
+ # Track usage from message_start event
+ if event.event_type == "message_start":
+ msg = event.data.get("message", {})
+ usage = msg.get("usage", {})
+ if "input_tokens" in usage:
+ stream_state["input_tokens"] = usage["input_tokens"]
+
+ # Track output tokens from message_delta
+ if event.event_type == "message_delta":
+ usage = event.data.get("usage", {})
+ if "output_tokens" in usage:
+ stream_state["output_tokens"] = usage["output_tokens"]
+
+ # Handle errors
+ if event.event_type == "error":
+ logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
+
+ except Exception as e:
+ logger.error(f"[{request_id}] Bedrock streaming error: {e}")
+ error_event = {
+ "type": "error",
+ "error": {"type": "api_error", "message": str(e)},
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
+
+ finally:
+ # Record metrics
+ total_latency = (time.time() - start_time) * 1000
+ output_tokens = stream_state["output_tokens"]
+
+ _backend_name = (
+ self.anthropic_backend.name if self.anthropic_backend else "anthropic"
+ )
+ await self.metrics.record_request(
+ provider=_backend_name,
+ model=model,
+ input_tokens=optimized_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ cached=False,
+ overhead_ms=optimization_latency,
+ ttfb_ms=stream_state["ttfb_ms"] or 0,
+ pipeline_timing=pipeline_timing,
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
+
+ # Log request
+ if self.logger:
+ self.logger.log(
+ RequestLog(
+ request_id=request_id,
+ timestamp=datetime.now().isoformat(),
+ provider=_backend_name,
+ model=model,
+ input_tokens_original=original_tokens,
+ input_tokens_optimized=optimized_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ savings_percent=(tokens_saved / original_tokens * 100)
+ if original_tokens > 0
+ else 0,
+ optimization_latency_ms=optimization_latency,
+ total_latency_ms=total_latency,
+ tags=tags,
+ cache_hit=False,
+ transforms_applied=transforms_applied,
+ request_messages=body.get("messages")
+ if self.config.log_full_messages
+ else None,
+ )
+ )
+
+ # Structured perf log line for `headroom perf` analysis
+ num_msgs = len(body.get("messages", []))
+ logger.info(
+ f"[{request_id}] PERF "
+ f"model={model} msgs={num_msgs} "
+ f"tok_before={original_tokens} tok_after={optimized_tokens} "
+ f"tok_saved={tokens_saved} "
+ f"cache_read=0 cache_write=0 cache_hit_pct=0 "
+ f"opt_ms={optimization_latency:.0f} "
+ f"transforms={_summarize_transforms(transforms_applied)}"
+ )
+
+ return StreamingResponse(
+ generate(),
+ media_type="text/event-stream",
+ )
+
+ async def _stream_openai_via_backend(
+ self,
+ body: dict,
+ headers: dict,
+ model: str,
+ request_id: str,
+ start_time: float,
+ original_tokens: int,
+ optimized_tokens: int,
+ tokens_saved: int,
+ transforms_applied: list[str],
+ tags: dict[str, str],
+ optimization_latency: float,
+ pipeline_timing: dict[str, float] | None = None,
+ ) -> StreamingResponse:
+ """Stream OpenAI chat completion response from backend.
+
+ Routes stream:true requests through the backend's stream_openai_message(),
+ yielding SSE events to the client.
+ """
+ from fastapi.responses import StreamingResponse
+
+ assert self.anthropic_backend is not None
+
+ async def generate():
+ try:
+ async for sse_chunk in self.anthropic_backend.stream_openai_message(body, headers):
+ yield sse_chunk.encode() if isinstance(sse_chunk, str) else sse_chunk
+ except Exception as e:
+ logger.error(f"[{request_id}] Backend streaming error: {e}")
+ error_data = {
+ "error": {
+ "message": str(e),
+ "type": "api_error",
+ "code": "backend_error",
+ }
+ }
+ yield f"data: {json.dumps(error_data)}\n\n".encode()
+ yield b"data: [DONE]\n\n"
+ finally:
+ total_latency = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider=self.anthropic_backend.name,
+ model=model,
+ input_tokens=optimized_tokens,
+ output_tokens=0, # Unknown in streaming
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ cached=False,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name} [stream]"
+ )
+
+ return StreamingResponse(
+ generate(),
+ media_type="text/event-stream",
+ )
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 6a561a67f..83d5a41c8 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -31,7 +31,6 @@ import os
import random
import sys
import time
-from datetime import datetime
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
@@ -39,7 +38,6 @@ if TYPE_CHECKING:
from ..cache.compression_cache import CompressionCache
from ..memory.tracker import MemoryTracker
-import contextlib
import httpx
@@ -47,7 +45,7 @@ try:
import uvicorn
from fastapi import FastAPI, HTTPException, Request, Response, WebSocket
from fastapi.middleware.cors import CORSMiddleware
- from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse
+ from fastapi.responses import HTMLResponse, PlainTextResponse
FASTAPI_AVAILABLE = True
except ImportError:
@@ -64,15 +62,11 @@ from headroom.cache.compression_store import get_compression_store
from headroom.ccr import (
CCR_TOOL_NAME,
# Batch processing
- BatchContext,
- BatchRequestContext,
- BatchResultProcessor,
CCRResponseHandler,
CCRToolInjector,
ContextTracker,
ContextTrackerConfig,
ResponseHandlerConfig,
- get_batch_context_store,
parse_tool_call,
)
from headroom.config import (
@@ -118,7 +112,6 @@ from headroom.proxy.request_logger import RequestLogger # noqa: F401
from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
from headroom.telemetry import get_telemetry_collector
from headroom.telemetry.toin import get_toin
-from headroom.tokenizers import get_tokenizer
from headroom.transforms import (
CacheAligner,
CodeAwareCompressor,
@@ -132,7 +125,6 @@ from headroom.transforms import (
TransformPipeline,
is_tree_sitter_available,
)
-from headroom.utils import extract_user_query
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
@@ -152,12 +144,22 @@ MAX_RATE_LIMITER_BUCKETS = 1000
# Compression pipeline timeout in seconds
-# =============================================================================
-# Caching
-# =============================================================================
+from headroom.proxy.handlers import ( # noqa: E402
+ AnthropicHandlerMixin,
+ BatchHandlerMixin,
+ GeminiHandlerMixin,
+ OpenAIHandlerMixin,
+ StreamingMixin,
+)
-class HeadroomProxy:
+class HeadroomProxy(
+ StreamingMixin,
+ AnthropicHandlerMixin,
+ OpenAIHandlerMixin,
+ GeminiHandlerMixin,
+ BatchHandlerMixin,
+):
"""Production-ready Headroom optimization proxy."""
ANTHROPIC_API_URL = "https://api.anthropic.com"
@@ -745,5077 +747,6 @@ class HeadroomProxy:
raise last_error # type: ignore[misc]
- async def handle_compress(self, request: Request) -> JSONResponse:
- """Compress messages without calling an LLM.
-
- POST /v1/compress
- Body: {"messages": [...], "model": "...", "config": {}}
- Returns compressed messages + metrics.
- """
- # Check bypass header
- if request.headers.get("x-headroom-bypass", "").lower() == "true":
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={"error": f"Invalid request body: {e!s}"},
- )
- messages = body.get("messages", [])
- return JSONResponse(
- {
- "messages": messages,
- "tokens_before": 0,
- "tokens_after": 0,
- "tokens_saved": 0,
- "compression_ratio": 1.0,
- "transforms_applied": [],
- "ccr_hashes": [],
- }
- )
-
- try:
- body = await _read_request_json(request)
- except Exception:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Invalid JSON in request body.",
- }
- },
- )
-
- messages = body.get("messages")
- model = body.get("model")
-
- if messages is None:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Missing required field: messages",
- }
- },
- )
-
- if model is None:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Missing required field: model",
- }
- },
- )
-
- if not messages:
- return JSONResponse(
- {
- "messages": [],
- "tokens_before": 0,
- "tokens_after": 0,
- "tokens_saved": 0,
- "compression_ratio": 1.0,
- "transforms_applied": [],
- "ccr_hashes": [],
- }
- )
-
- try:
- # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)
- # Allow optional token_budget to override model's context limit
- # (used by OpenClaw compact() and other callers that need tighter budgets)
- token_budget = body.get("token_budget")
- context_limit = (
- token_budget
- if token_budget and isinstance(token_budget, int)
- else self.openai_provider.get_context_limit(model)
- )
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- )
-
- return JSONResponse(
- {
- "messages": result.messages,
- "tokens_before": result.tokens_before,
- "tokens_after": result.tokens_after,
- "tokens_saved": result.tokens_before - result.tokens_after,
- "compression_ratio": (
- result.tokens_after / result.tokens_before
- if result.tokens_before > 0
- else 1.0
- ),
- "transforms_applied": result.transforms_applied,
- "transforms_summary": result.transforms_summary,
- "ccr_hashes": result.markers_inserted,
- }
- )
- except Exception as e:
- logger.exception("Compression failed: %s", e)
- return JSONResponse(
- status_code=503,
- content={
- "error": {
- "type": "compression_error",
- "message": str(e),
- }
- },
- )
-
- async def handle_anthropic_messages(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle Anthropic /v1/messages endpoint."""
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "type": "error",
- "error": {
- "type": "request_too_large",
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- },
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "type": "error",
- "error": {
- "type": "invalid_request_error",
- "message": f"Invalid request body: {e!s}",
- },
- },
- )
- model = body.get("model", "unknown")
- messages = body.get("messages", [])
-
- # Validate message array size
- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
- return JSONResponse(
- status_code=400,
- content={
- "type": "error",
- "error": {
- "type": "invalid_request_error",
- "message": f"Message array too large ({len(messages)} messages). "
- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
- },
- },
- )
-
- stream = body.get("stream", False)
-
- # Bypass: skip ALL compression, TOIN learning, and CCR injection
- # when the caller explicitly opts out via header.
- # Prevents Headroom from corrupting sub-agent API calls
- # (e.g., Claude Code sub-agents that inherit ANTHROPIC_BASE_URL).
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- if _bypass:
- logger.info(f"[{request_id}] Bypass: skipping compression (header)")
-
- # Image compression (before text optimization)
- if self.config.image_optimize and messages and not _bypass:
- compressor = _get_image_compressor()
- if compressor and compressor.has_images(messages):
- messages = compressor.compress(messages, provider="anthropic")
- if compressor.last_result:
- logger.info(
- f"Image compression: {compressor.last_result.technique.value} "
- f"({compressor.last_result.savings_percent:.0f}% saved, "
- f"{compressor.last_result.original_tokens} -> "
- f"{compressor.last_result.compressed_tokens} tokens)"
- )
-
- # Extract headers and tags
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- tags = self._extract_tags(headers)
-
- # Rate limiting
- if self.rate_limiter:
- api_key = headers.get("x-api-key", "")
- client_ip = request.client.host if request.client else "unknown"
- rate_key = f"{api_key[:16]}:{client_ip}" if api_key else client_ip
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited()
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- headers={"Retry-After": str(int(wait_seconds) + 1)},
- )
-
- # Budget check
- if self.cost_tracker:
- allowed, remaining = self.cost_tracker.check_budget()
- if not allowed:
- raise HTTPException(
- status_code=429,
- detail=f"Budget exceeded for {self.config.budget_period} period",
- )
-
- # Memory: Get user ID when memory is enabled (fallback to "default" for simple DevEx)
- memory_user_id: str | None = None
- if self.memory_handler:
- memory_user_id = headers.get("x-headroom-user-id", "default")
-
- # Check cache (non-streaming only)
- cache_hit = False
- if self.cache and not stream:
- cached = await self.cache.get(messages, model)
- if cached:
- cache_hit = True
- optimization_latency = (time.time() - start_time) * 1000
-
- await self.metrics.record_request(
- provider="anthropic",
- model=model,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0, # Savings already counted when response was cached
- latency_ms=optimization_latency,
- cached=True,
- )
-
- # Remove compression headers from cached response
- response_headers = dict(cached.response_headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=cached.response_body,
- headers=response_headers,
- media_type="application/json",
- )
-
- # Count original tokens
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Hook: pre_compress — let hooks modify messages before compression
- from headroom.transforms.query_echo import extract_user_query
-
- if self.config.hooks:
- from headroom.hooks import CompressContext
-
- _hook_ctx = CompressContext(
- model=model,
- user_query=extract_user_query(messages),
- provider="anthropic",
- )
- try:
- messages = self.config.hooks.pre_compress(messages, _hook_ctx)
- except Exception as e:
- logger.debug(f"[{request_id}] pre_compress hook error: {e}")
-
- # Apply optimization
- transforms_applied = []
- pipeline_timing: dict[str, float] = {}
- waste_signals_dict: dict[str, int] | None = None
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- # Get prefix cache tracker for this session
- session_id = self.session_tracker_store.compute_session_id(request, model, messages)
- prefix_tracker = self.session_tracker_store.get_or_create(session_id, "anthropic")
- frozen_message_count = prefix_tracker.get_frozen_message_count()
-
- _compression_failed = False
- original_messages = messages # Preserve for 400-retry fallback
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
- if self.config.optimize and messages and not _bypass and _license_ok:
- try:
- context_limit = self.anthropic_provider.get_context_limit(model)
- biases = (
- self.config.hooks.compute_biases(messages, _hook_ctx)
- if self.config.hooks
- else None
- )
-
- if self.config.mode == "token_headroom":
- comp_cache = self._get_compression_cache(session_id)
-
- # Zone 1: Swap cached compressed versions into working copy
- working_messages = comp_cache.apply_cached(messages)
-
- # Re-freeze boundary: consecutive stable messages from start
- frozen_message_count = comp_cache.compute_frozen_count(messages)
-
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.anthropic_pipeline.apply(
- messages=working_messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(working_messages),
- frozen_message_count=frozen_message_count,
- biases=biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- # Cache newly compressed messages (index-aligned diff)
- if result.messages != working_messages:
- comp_cache.update_from_result(messages, result.messages)
-
- # Always use pipeline result — Zone 1 swaps are already applied
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- # Keep original_tokens as the REAL original (pre-Zone-1-swap)
- # so tokens_saved captures both Zone 1 + Zone 2 savings.
- # original_tokens was set at line ~2183 from uncompressed messages.
- optimized_tokens = result.tokens_after
- else:
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.anthropic_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- frozen_message_count=frozen_message_count,
- biases=biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
-
- if result.waste_signals:
- waste_signals_dict = result.waste_signals.to_dict()
- except Exception as e:
- logger.warning(f"Optimization failed: {e}")
- # Flag compression failure for observability
- _compression_failed = True
-
- tokens_saved = max(0, original_tokens - optimized_tokens)
- optimization_latency = (time.time() - start_time) * 1000
-
- # Hook: post_compress — let hooks observe compression results
- if self.config.hooks and tokens_saved > 0:
- from headroom.hooks import CompressEvent
-
- try:
- self.config.hooks.post_compress(
- CompressEvent(
- tokens_before=original_tokens,
- tokens_after=optimized_tokens,
- tokens_saved=tokens_saved,
- compression_ratio=tokens_saved / original_tokens
- if original_tokens > 0
- else 0,
- transforms_applied=transforms_applied,
- model=model,
- user_query=_hook_ctx.user_query if self.config.hooks else "",
- provider="anthropic",
- )
- )
- except Exception as e:
- logger.debug(f"[{request_id}] post_compress hook error: {e}")
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = body.get("tools")
- _original_tools = tools # Preserve for diagnostic / future retry
- if (
- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
- ) and not _bypass:
- # Create fresh injector to avoid state leakage between requests
- injector = CCRToolInjector(
- provider="anthropic",
- inject_tool=self.config.ccr_inject_tool,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- optimized_messages, tools, was_injected = injector.process_request(
- optimized_messages, tools
- )
-
- if injector.has_compressed_content:
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
- )
- else:
- logger.debug(
- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
- )
-
- # Track compression in context tracker for multi-turn awareness
- if self.ccr_context_tracker:
- self._turn_counter += 1
- for hash_key in injector.detected_hashes:
- # Get compression metadata from store
- store = get_compression_store()
- entry = store.get_metadata(hash_key)
- if entry:
- self.ccr_context_tracker.track_compression(
- hash_key=hash_key,
- turn_number=self._turn_counter,
- tool_name=entry.get("tool_name"),
- original_count=entry.get("original_item_count", 0),
- compressed_count=entry.get("compressed_item_count", 0),
- query_context=entry.get("query_context", ""),
- sample_content=entry.get("compressed_content", "")[:500],
- )
-
- # CCR Proactive Expansion: Check if current query needs expanded context
- if self.ccr_context_tracker and self.config.ccr_proactive_expansion:
- # Extract user query from messages
- user_query = ""
- for msg in reversed(messages):
- if msg.get("role") == "user":
- content = msg.get("content", "")
- if isinstance(content, str):
- user_query = content
- elif isinstance(content, list):
- for block in content:
- if isinstance(block, dict) and block.get("type") == "text":
- user_query = block.get("text", "")
- break
- break
-
- if user_query:
- recommendations = self.ccr_context_tracker.analyze_query(
- user_query, self._turn_counter
- )
- if recommendations:
- expansions = self.ccr_context_tracker.execute_expansions(recommendations)
- if expansions:
- # Add expanded context to the system message or as additional context
- expansion_text = self.ccr_context_tracker.format_expansions_for_context(
- expansions
- )
- logger.info(
- f"[{request_id}] CCR: Proactively expanded {len(expansions)} context(s) "
- f"based on query relevance"
- )
- # Append to the last user message
- if optimized_messages and optimized_messages[-1].get("role") == "user":
- last_msg = optimized_messages[-1]
- content = last_msg.get("content", "")
- if isinstance(content, str):
- optimized_messages[-1] = {
- **last_msg,
- "content": content + "\n\n" + expansion_text,
- }
-
- # Traffic Learner: Extract patterns from inbound tool results
- if self.traffic_learner:
- try:
- # Wire backend on first use (lazy init after memory handler is ready)
- if (
- self.traffic_learner._backend is None
- and self.memory_handler
- and self.memory_handler.initialized
- and self.memory_handler.backend
- ):
- self.traffic_learner.set_backend(self.memory_handler.backend)
-
- # Extract tool results from messages and learn from them
- tool_results = self.traffic_learner.extract_tool_results_from_messages(
- optimized_messages
- )
- for tr in tool_results[-5:]: # Only recent results
- await self.traffic_learner.on_tool_result(
- tool_name=tr["tool_name"],
- tool_input=tr["input"],
- tool_output=tr["output"],
- is_error=tr["is_error"],
- )
-
- # Also extract preference signals from user messages
- await self.traffic_learner.on_messages(optimized_messages)
- except Exception as e:
- logger.debug(f"[{request_id}] Traffic learner: {e}")
-
- # Memory: Inject context and tools
- if self.memory_handler and memory_user_id:
- # Search and inject memory context
- if self.memory_handler.config.inject_context:
- try:
- memory_context = await self.memory_handler.search_and_format_context(
- memory_user_id, optimized_messages
- )
- if memory_context:
- optimized_messages = self._inject_system_context(
- optimized_messages, memory_context, body=body
- )
- logger.info(
- f"[{request_id}] Memory: Injected {len(memory_context)} chars of context"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Memory: Context injection failed: {e}")
-
- # Inject memory tools
- if self.memory_handler.config.inject_tools:
- tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "anthropic")
- if mem_tools_injected:
- tool_names = [
- t.get("name") or t.get("type", "")
- for t in tools
- if t.get("name", "").startswith("memory")
- or t.get("type", "").startswith("memory")
- ]
- logger.info(f"[{request_id}] Memory: Injected tools: {tool_names}")
-
- # Add beta headers for native memory tool
- beta_headers = self.memory_handler.get_beta_headers()
- if beta_headers:
- for key, value in beta_headers.items():
- # Merge with existing beta header if present
- existing = headers.get(key, "")
- if existing and value not in existing:
- headers[key] = f"{existing},{value}"
- else:
- headers[key] = value
- logger.info(
- f"[{request_id}] Memory: Added beta header: {key}={headers[key]}"
- )
-
- # Query Echo: disabled — hurts prefix caching in long conversations.
- # The echo changes every turn, invalidating the cached prefix.
- # To re-enable, uncomment and set query_echo_enabled on ProxyConfig.
-
- # Update body
- body["messages"] = optimized_messages
- if tools is not None:
- body["tools"] = tools
-
- # Forward request - use Bedrock backend if configured, otherwise direct API
- if self.anthropic_backend is not None:
- # Route through Bedrock backend
- try:
- if stream:
- return await self._stream_response_bedrock(
- body,
- headers,
- "anthropic",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- )
- else:
- backend_response = await self.anthropic_backend.send_message(body, headers)
-
- if backend_response.error:
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
-
- # Track metrics
- total_latency = (time.time() - start_time) * 1000
- usage = backend_response.body.get("usage", {})
- output_tokens = usage.get("output_tokens", 0)
-
- _backend_name = (
- self.anthropic_backend.name if self.anthropic_backend else "anthropic"
- )
- await self.metrics.record_request(
- provider=_backend_name,
- model=model,
- input_tokens=optimized_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- cached=False,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
-
- # Log request
- if self.logger:
- self.logger.log(
- RequestLog(
- request_id=request_id,
- timestamp=datetime.now().isoformat(),
- provider=_backend_name,
- model=model,
- input_tokens_original=original_tokens,
- input_tokens_optimized=optimized_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- savings_percent=(tokens_saved / original_tokens * 100)
- if original_tokens > 0
- else 0,
- optimization_latency_ms=optimization_latency,
- total_latency_ms=total_latency,
- tags=tags,
- cache_hit=False,
- transforms_applied=transforms_applied,
- request_messages=body.get("messages")
- if self.config.log_full_messages
- else None,
- )
- )
-
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
- except Exception as e:
- logger.error(f"[{request_id}] Bedrock backend error: {e}")
- return JSONResponse(
- status_code=500,
- content={
- "type": "error",
- "error": {"type": "api_error", "message": str(e)},
- },
- )
-
- # Direct Anthropic API
- url = f"{self.ANTHROPIC_API_URL}/v1/messages"
-
- try:
- if stream:
- return await self._stream_response(
- url,
- headers,
- body,
- "anthropic",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- memory_user_id=memory_user_id,
- pipeline_timing=pipeline_timing,
- prefix_tracker=prefix_tracker,
- )
- else:
- response = await self._retry_request("POST", url, headers, body)
-
- # Full diagnostic dump on upstream errors.
- # Writes pre/post compression messages, tools, and error
- # to ~/.headroom/logs/debug_400/ for offline analysis.
- if response.status_code >= 400:
- try:
- err_body = response.json()
- err_msg = err_body.get("error", {}).get("message", "")
- err_type = err_body.get("error", {}).get("type", "")
- except Exception:
- err_body = {"raw": response.text[:2000]}
- err_msg = str(response.text[:500])
- err_type = "parse_error"
-
- logger.warning(
- f"[{request_id}] UPSTREAM_ERROR "
- f"status={response.status_code} "
- f"error_type={err_type} "
- f"error_msg={err_msg!r} "
- f"model={model} "
- f"compressed={'yes' if transforms_applied else 'no'} "
- f"transforms={transforms_applied} "
- f"original_tokens={original_tokens} "
- f"optimized_tokens={optimized_tokens} "
- f"message_count={len(body.get('messages', []))} "
- f"stream={stream}"
- )
-
- # Dump full request details to debug file
- try:
- debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"
- debug_dir.mkdir(parents=True, exist_ok=True)
- ts = datetime.now().strftime("%Y%m%d_%H%M%S")
- debug_file = debug_dir / f"{ts}_{request_id}.json"
-
- # Sanitize headers (redact API keys)
- safe_headers = {}
- for k, v in headers.items():
- if k.lower() in ("x-api-key", "authorization"):
- safe_headers[k] = v[:12] + "..." if v else ""
- else:
- safe_headers[k] = v
-
- debug_payload = {
- "request_id": request_id,
- "timestamp": datetime.now().isoformat(),
- "status_code": response.status_code,
- "error_response": err_body,
- "model": model,
- "stream": stream,
- "headers": safe_headers,
- "compression": {
- "was_compressed": bool(transforms_applied),
- "transforms": transforms_applied,
- "original_tokens": original_tokens,
- "optimized_tokens": optimized_tokens,
- "tokens_saved": tokens_saved,
- "compression_failed": _compression_failed,
- },
- "tools_sent": body.get("tools"),
- "tool_count": len(body.get("tools") or []),
- "original_tool_count": len(_original_tools or []),
- "messages_sent": body.get("messages"),
- "message_count": len(body.get("messages", [])),
- "original_messages": (
- original_messages
- if original_messages is not body.get("messages")
- else "__same_as_sent__"
- ),
- "original_message_count": len(original_messages),
- "system_prompt": body.get("system"),
- }
-
- with open(debug_file, "w") as f:
- json.dump(debug_payload, f, indent=2, default=str)
-
- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
- except Exception as dump_err:
- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
-
- # Parse response for CCR handling
- resp_json = None
- try:
- resp_json = response.json()
- except (json.JSONDecodeError, ValueError) as e:
- logger.debug(
- f"[{request_id}] Failed to parse response JSON for CCR handling: {e}"
- )
-
- # CCR Response Handling: Handle headroom_retrieve tool calls automatically
- if (
- self.ccr_response_handler
- and resp_json
- and response.status_code == 200
- and self.ccr_response_handler.has_ccr_tool_calls(resp_json, "anthropic")
- ):
- logger.info(f"[{request_id}] CCR: Detected retrieval tool call, handling...")
-
- # Create API call function for continuation
- # Use a fresh client to avoid potential decompression state issues
- async def api_call_fn(
- msgs: list[dict], tls: list[dict] | None
- ) -> dict[str, Any]:
- continuation_body = {
- **body,
- "messages": msgs,
- }
- if tls is not None:
- continuation_body["tools"] = tls
-
- # Use clean headers for continuation
- continuation_headers = {
- k: v
- for k, v in headers.items()
- if k.lower()
- not in (
- "content-encoding",
- "transfer-encoding",
- "accept-encoding",
- "content-length",
- )
- }
-
- # Reuse main client for CCR continuations (connection pooling)
- logger.info(f"CCR: Making continuation request with {len(msgs)} messages")
- assert self.http_client is not None, "HTTP client not initialized"
- try:
- cont_response = await self.http_client.post(
- url,
- json=continuation_body,
- headers=continuation_headers,
- timeout=httpx.Timeout(120.0), # Override timeout for CCR
- )
- logger.info(
- f"CCR: Got response status={cont_response.status_code}, "
- f"content-encoding={cont_response.headers.get('content-encoding')}"
- )
- result: dict[str, Any] = cont_response.json()
- logger.info("CCR: Parsed JSON successfully")
- return result
- except Exception as e:
- resp_headers: str | dict[str, str] = "N/A"
- try:
- resp_headers = dict(cont_response.headers)
- except Exception:
- pass
- logger.error(
- f"CCR: API call failed: {e}, response headers: {resp_headers}"
- )
- raise
-
- # Handle CCR tool calls
- try:
- final_resp_json = await self.ccr_response_handler.handle_response(
- resp_json,
- optimized_messages,
- tools,
- api_call_fn,
- provider="anthropic",
- )
- # Update response content with final response
- resp_json = final_resp_json
- # Remove encoding headers since content is now uncompressed JSON
- ccr_response_headers = {
- k: v
- for k, v in response.headers.items()
- if k.lower() not in ("content-encoding", "content-length")
- }
- try:
- ccr_content = json.dumps(final_resp_json).encode()
- except (TypeError, ValueError) as json_err:
- logger.warning(
- f"[{request_id}] CCR: JSON serialization failed: {json_err}"
- )
- ccr_content = json.dumps(resp_json).encode()
- response = httpx.Response(
- status_code=200,
- content=ccr_content,
- headers=ccr_response_headers,
- )
- logger.info(f"[{request_id}] CCR: Retrieval handled successfully")
- except Exception as e:
- import traceback
-
- logger.warning(
- f"[{request_id}] CCR: Response handling failed: {e}\n"
- f"Traceback: {traceback.format_exc()}"
- )
- # Continue with original response
-
- # Memory: Handle memory tool calls in response
- if (
- self.memory_handler
- and memory_user_id
- and resp_json
- and response.status_code == 200
- and self.memory_handler.has_memory_tool_calls(resp_json, "anthropic")
- ):
- logger.info(f"[{request_id}] Memory: Detected memory tool call, handling...")
-
- try:
- # Execute memory tool calls
- tool_results = await self.memory_handler.handle_memory_tool_calls(
- resp_json, memory_user_id, "anthropic"
- )
-
- if tool_results:
- # Create continuation messages
- assistant_msg = {
- "role": "assistant",
- "content": resp_json.get("content", []),
- }
- user_msg = {
- "role": "user",
- "content": tool_results,
- }
-
- continuation_messages = optimized_messages + [assistant_msg, user_msg]
-
- # Make continuation API call
- continuation_body = {**body, "messages": continuation_messages}
- if tools:
- continuation_body["tools"] = tools
-
- cont_response = await self._retry_request(
- "POST", url, headers, continuation_body
- )
-
- # Update response with continuation
- resp_json = cont_response.json()
- response = cont_response
- logger.info(
- f"[{request_id}] Memory: Tool calls handled, continuation complete"
- )
-
- except Exception as e:
- logger.warning(f"[{request_id}] Memory: Tool call handling failed: {e}")
- # Continue with original response
-
- total_latency = (time.time() - start_time) * 1000
-
- # Parse response for output token count and cache metrics
- output_tokens = 0
- cr_tokens = 0
- cw_tokens = 0
- uncached_input_tokens = 0
- if resp_json:
- usage = resp_json.get("usage", {})
- output_tokens = usage.get("output_tokens", 0)
- cr_tokens = usage.get("cache_read_input_tokens", 0)
- cw_tokens = usage.get("cache_creation_input_tokens", 0)
- uncached_input_tokens = usage.get("input_tokens", 0)
-
- # Update prefix cache tracker for next turn
- prefix_tracker.update_from_response(
- cache_read_tokens=cr_tokens,
- cache_write_tokens=cw_tokens,
- messages=optimized_messages,
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(
- model,
- tokens_saved,
- optimized_tokens,
- cache_read_tokens=cr_tokens,
- cache_write_tokens=cw_tokens,
- uncached_tokens=uncached_input_tokens,
- )
-
- # Cache response
- if self.cache and response.status_code == 200:
- await self.cache.set(
- messages,
- model,
- response.content,
- dict(response.headers),
- tokens_saved=tokens_saved,
- )
-
- # Record metrics — use optimized_tokens (what we sent), not API's
- # input_tokens which is just the non-cached portion with prompt caching
- await self.metrics.record_request(
- provider="anthropic",
- model=model,
- input_tokens=optimized_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- waste_signals=waste_signals_dict,
- cache_read_tokens=cr_tokens,
- cache_write_tokens=cw_tokens,
- uncached_input_tokens=uncached_input_tokens,
- )
-
- # Log request
- if self.logger:
- self.logger.log(
- RequestLog(
- request_id=request_id,
- timestamp=datetime.now().isoformat(),
- provider="anthropic",
- model=model,
- input_tokens_original=original_tokens,
- input_tokens_optimized=optimized_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- savings_percent=(tokens_saved / original_tokens * 100)
- if original_tokens > 0
- else 0,
- optimization_latency_ms=optimization_latency,
- total_latency_ms=total_latency,
- tags=tags,
- cache_hit=cache_hit,
- transforms_applied=transforms_applied,
- waste_signals=waste_signals_dict,
- request_messages=messages if self.config.log_full_messages else None,
- )
- )
-
- # Structured perf log line for `headroom perf` analysis
- num_msgs = len(messages)
- resp_usage = resp_json.get("usage", {}) if resp_json else {}
- cr = resp_usage.get("cache_read_input_tokens", 0)
- cw = resp_usage.get("cache_creation_input_tokens", 0)
- chp = round(cr / (cr + cw) * 100) if (cr + cw) > 0 else 0
- timing_str = (
- " ".join(f"{k}={v:.0f}ms" for k, v in pipeline_timing.items())
- if pipeline_timing
- else ""
- )
- logger.info(
- f"[{request_id}] PERF "
- f"model={model} msgs={num_msgs} "
- f"tok_before={original_tokens} tok_after={optimized_tokens} "
- f"tok_saved={tokens_saved} "
- f"cache_read={cr} cache_write={cw} cache_hit_pct={chp} "
- f"opt_ms={optimization_latency:.0f} "
- f"transforms={_summarize_transforms(transforms_applied)}"
- f"{' timing=' + timing_str if timing_str else ''}"
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Inject Headroom compression metrics (for SaaS metering)
- response_headers["x-headroom-tokens-before"] = str(original_tokens)
- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
- response_headers["x-headroom-model"] = model
- if transforms_applied:
- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
- if cache_hit:
- response_headers["x-headroom-cached"] = "true"
- if _compression_failed:
- response_headers["x-headroom-compression-failed"] = "true"
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- except Exception as e:
- await self.metrics.record_failed()
- # Log full error details internally for debugging
- logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")
-
- # Try fallback if enabled
- if self.config.fallback_enabled and self.config.fallback_provider == "openai":
- logger.info(f"[{request_id}] Attempting fallback to OpenAI")
- # Convert to OpenAI format and retry
- # (simplified - would need message format conversion)
-
- # Return sanitized error message to client (don't expose internal details)
- return JSONResponse(
- status_code=502,
- content={
- "type": "error",
- "error": {
- "type": "api_error",
- "message": "An error occurred while processing your request. Please try again.",
- },
- },
- )
-
- async def handle_anthropic_batch_create(
- self,
- request: Request,
- ) -> Response:
- """Handle Anthropic POST /v1/messages/batches endpoint with compression.
-
- Anthropic batch format:
- {
- "requests": [
- {
- "custom_id": "req-1",
- "params": {
- "model": "claude-sonnet-4-20250514",
- "max_tokens": 1024,
- "messages": [{"role": "user", "content": "Hello"}]
- }
- },
- ...
- ]
- }
-
- This method applies compression to each request's messages before forwarding.
- """
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "type": "error",
- "error": {
- "type": "request_too_large",
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- },
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "type": "error",
- "error": {
- "type": "invalid_request_error",
- "message": f"Invalid request body: {e!s}",
- },
- },
- )
-
- requests_list = body.get("requests", [])
- if not requests_list:
- return JSONResponse(
- status_code=400,
- content={
- "type": "error",
- "error": {
- "type": "invalid_request_error",
- "message": "Missing or empty 'requests' field in batch request",
- },
- },
- )
-
- # Extract headers
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
-
- # Track compression stats across all batch requests
- total_original_tokens = 0
- total_optimized_tokens = 0
- total_tokens_saved = 0
- compressed_requests = []
- pipeline_timing: dict[str, float] = {}
-
- # Apply compression to each request in the batch
- for batch_req in requests_list:
- custom_id = batch_req.get("custom_id", "")
- params = batch_req.get("params", {})
- messages = params.get("messages", [])
- model = params.get("model", "unknown")
-
- if not messages or not self.config.optimize:
- # No messages or optimization disabled - pass through unchanged
- compressed_requests.append(batch_req)
- continue
-
- # Apply optimization
- try:
- context_limit = self.anthropic_provider.get_context_limit(model)
- result = self.anthropic_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
-
- optimized_messages = result.messages
- for k, v in result.timing.items():
- pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v
- # Use pipeline's token counts for consistency with pipeline logs
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
- total_original_tokens += original_tokens
- total_optimized_tokens += optimized_tokens
- tokens_saved = max(0, original_tokens - optimized_tokens)
- total_tokens_saved += tokens_saved
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = params.get("tools")
- if self.config.ccr_inject_tool and tokens_saved > 0:
- injector = CCRToolInjector(
- provider="anthropic",
- inject_tool=True,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- optimized_messages, tools, was_injected = injector.process_request(
- optimized_messages, tools
- )
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for batch request '{custom_id}'"
- )
-
- # Create compressed batch request
- compressed_params = {**params, "messages": optimized_messages}
- if tools is not None:
- compressed_params["tools"] = tools
- compressed_requests.append(
- {
- "custom_id": custom_id,
- "params": compressed_params,
- }
- )
-
- if tokens_saved > 0:
- logger.debug(
- f"[{request_id}] Batch request '{custom_id}': "
- f"{original_tokens:,} -> {optimized_tokens:,} tokens "
- f"(saved {tokens_saved:,})"
- )
-
- except Exception as e:
- logger.warning(
- f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}"
- )
- # Pass through unchanged on failure
- compressed_requests.append(batch_req)
- total_optimized_tokens += original_tokens
-
- # Update body with compressed requests
- body["requests"] = compressed_requests
-
- optimization_latency = (time.time() - start_time) * 1000
-
- # Forward request to Anthropic
- url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches"
-
- try:
- response = await self._retry_request("POST", url, headers, body)
-
- # Record metrics
- await self.metrics.record_request(
- provider="anthropic",
- model="batch",
- input_tokens=total_optimized_tokens,
- output_tokens=0,
- tokens_saved=total_tokens_saved,
- latency_ms=optimization_latency,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
-
- # Log compression stats
- if total_tokens_saved > 0:
- savings_percent = (
- (total_tokens_saved / total_original_tokens * 100)
- if total_original_tokens > 0
- else 0
- )
- logger.info(
- f"[{request_id}] Batch ({len(compressed_requests)} requests): "
- f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "
- f"(saved {total_tokens_saved:,}, {savings_percent:.1f}%)"
- )
-
- # Store batch context for CCR result processing
- if response.status_code == 200 and self.config.ccr_inject_tool:
- try:
- response_data = response.json()
- batch_id = response_data.get("id")
- if batch_id:
- await self._store_anthropic_batch_context(
- batch_id,
- requests_list,
- headers.get("x-api-key"),
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Failed to store batch context: {e}")
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- except Exception as e:
- await self.metrics.record_failed()
- logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")
- return JSONResponse(
- status_code=502,
- content={
- "type": "error",
- "error": {
- "type": "api_error",
- "message": "An error occurred while processing your batch request. Please try again.",
- },
- },
- )
-
- async def handle_anthropic_batch_passthrough(
- self,
- request: Request,
- batch_id: str | None = None,
- ) -> Response:
- """Handle Anthropic batch passthrough endpoints.
-
- Used for:
- - GET /v1/messages/batches - List batches
- - GET /v1/messages/batches/{batch_id} - Get batch
- - GET /v1/messages/batches/{batch_id}/results - Get batch results
- - POST /v1/messages/batches/{batch_id}/cancel - Cancel batch
- """
- start_time = time.time()
- path = request.url.path
- url = f"{self.ANTHROPIC_API_URL}{path}"
-
- # Preserve query string parameters (e.g., limit, after_id for list endpoint)
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
-
- body = await request.body()
-
- response = await self.http_client.request( # type: ignore[union-attr]
- method=request.method,
- url=url,
- headers=headers,
- content=body,
- )
-
- # Track metrics
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider="anthropic",
- model="passthrough:batches",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- )
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- async def _store_anthropic_batch_context(
- self,
- batch_id: str,
- requests_list: list[dict[str, Any]],
- api_key: str | None,
- ) -> None:
- """Store batch context for CCR result processing.
-
- Args:
- batch_id: The batch ID from the API response.
- requests_list: The original batch requests.
- api_key: The API key for continuation calls.
- """
- store = get_batch_context_store()
- context = BatchContext(
- batch_id=batch_id,
- provider="anthropic",
- api_key=api_key,
- api_base_url=self.ANTHROPIC_API_URL,
- )
-
- for batch_req in requests_list:
- custom_id = batch_req.get("custom_id", "")
- params = batch_req.get("params", {})
- context.add_request(
- BatchRequestContext(
- custom_id=custom_id,
- messages=params.get("messages", []),
- tools=params.get("tools"),
- model=params.get("model", ""),
- extras={
- "max_tokens": params.get("max_tokens", 4096),
- "system": params.get("system"),
- },
- )
- )
-
- await store.store(context)
- logger.debug(f"Stored batch context for {batch_id} with {len(requests_list)} requests")
-
- async def handle_anthropic_batch_results(
- self,
- request: Request,
- batch_id: str,
- ) -> Response:
- """Handle Anthropic batch results with CCR post-processing.
-
- This endpoint:
- 1. Fetches raw results from Anthropic
- 2. Detects CCR tool calls in each result
- 3. Executes retrieval and makes continuation calls
- 4. Returns processed results with complete responses
- """
- start_time = time.time()
-
- # Forward request to get raw results
- url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches/{batch_id}/results"
-
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
-
- response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]
-
- if response.status_code != 200:
- # Error - pass through
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Parse results - Anthropic batch results are JSONL format
- raw_content = response.content.decode("utf-8")
- results = []
- for line in raw_content.strip().split("\n"):
- if line.strip():
- try:
- results.append(json.loads(line))
- except json.JSONDecodeError:
- continue
-
- if not results:
- # No results to process
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Check if we have context and CCR processing is enabled
- store = get_batch_context_store()
- batch_context = await store.get(batch_id)
-
- if batch_context is None or not self.config.ccr_inject_tool:
- # No context or CCR disabled - pass through
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Process results with CCR handler
- processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]
- processed = await processor.process_results(batch_id, results, "anthropic")
-
- # Convert back to JSONL format
- processed_lines = []
- for p in processed:
- processed_lines.append(json.dumps(p.result))
- if p.was_processed:
- logger.info(
- f"CCR: Processed batch result {p.custom_id} "
- f"({p.continuation_rounds} continuation rounds)"
- )
-
- processed_content = "\n".join(processed_lines)
-
- # Track metrics
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider="anthropic",
- model="batch:ccr-processed",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- )
-
- return Response(
- content=processed_content.encode("utf-8"),
- status_code=200,
- media_type="application/jsonl",
- )
-
- # =========================================================================
- # Google/Gemini Batch API Handlers
- # =========================================================================
-
- async def handle_google_batch_create(
- self,
- request: Request,
- model: str,
- ) -> Response:
- """Handle Google POST /v1beta/models/{model}:batchGenerateContent endpoint.
-
- Google batch format:
- {
- "batch": {
- "display_name": "my-batch",
- "input_config": {
- "requests": {
- "requests": [
- {
- "request": {"contents": [{"parts": [{"text": "..."}]}]},
- "metadata": {"key": "request-1"}
- }
- ]
- }
- }
- }
- }
-
- This method applies compression to each request's contents before forwarding.
- """
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "code": 413,
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "status": "INVALID_ARGUMENT",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "code": 400,
- "message": f"Invalid request body: {e!s}",
- "status": "INVALID_ARGUMENT",
- }
- },
- )
-
- # Extract batch config
- batch_config = body.get("batch", {})
- input_config = batch_config.get("input_config", {})
- requests_wrapper = input_config.get("requests", {})
- requests_list = requests_wrapper.get("requests", [])
-
- if not requests_list:
- # No inline requests - might be using file input, pass through
- logger.debug(f"[{request_id}] Google batch: No inline requests, passing through")
- return await self._google_batch_passthrough(request, model, body)
-
- # Extract headers
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
-
- # Track compression stats
- total_original_tokens = 0
- total_optimized_tokens = 0
- total_tokens_saved = 0
- compressed_requests = []
- pipeline_timing: dict[str, float] = {}
-
- # Apply compression to each request in the batch
- for idx, batch_req in enumerate(requests_list):
- req_content = batch_req.get("request", {})
- metadata = batch_req.get("metadata", {})
- contents = req_content.get("contents", [])
-
- if not contents or not self.config.optimize:
- # No contents or optimization disabled - pass through unchanged
- compressed_requests.append(batch_req)
- continue
-
- # Convert Google format to messages for compression
- system_instruction = req_content.get("systemInstruction")
- messages, preserved_indices = self._gemini_contents_to_messages(
- contents, system_instruction
- )
-
- # Store original content entries that have non-text parts before compression
- preserved_contents = {idx: contents[idx] for idx in preserved_indices}
-
- # Early exit if ALL content has non-text parts (nothing to compress)
- if len(preserved_indices) == len(contents):
- # All content has non-text parts, skip compression
- compressed_requests.append(batch_req)
- continue
-
- # Apply optimization
- try:
- # Default context limit for most models
- context_limit = 128000
-
- # Use OpenAI pipeline (similar message format after conversion)
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
-
- optimized_messages = result.messages
- for k, v in result.timing.items():
- pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v
- # Use pipeline's token counts for consistency with pipeline logs
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
- total_original_tokens += original_tokens
- total_optimized_tokens += optimized_tokens
- tokens_saved = max(0, original_tokens - optimized_tokens)
- total_tokens_saved += tokens_saved
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = req_content.get("tools")
- # Extract existing function declarations if present
- existing_funcs = None
- if tools:
- for tool in tools:
- if "functionDeclarations" in tool:
- existing_funcs = tool["functionDeclarations"]
- break
-
- if self.config.ccr_inject_tool and tokens_saved > 0:
- injector = CCRToolInjector(
- provider="google",
- inject_tool=True,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- optimized_messages, injected_funcs, was_injected = injector.process_request(
- optimized_messages, existing_funcs
- )
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for Google batch request {idx}"
- )
- existing_funcs = injected_funcs
-
- # Convert back to Google contents format
- optimized_contents, optimized_sys_inst = self._messages_to_gemini_contents(
- optimized_messages
- )
-
- # Restore preserved content entries that had non-text parts
- for orig_idx, original_content in preserved_contents.items():
- if orig_idx < len(optimized_contents):
- optimized_contents[orig_idx] = original_content
-
- # Create compressed batch request
- compressed_req_content = {**req_content, "contents": optimized_contents}
- if optimized_sys_inst:
- compressed_req_content["systemInstruction"] = optimized_sys_inst
- if existing_funcs is not None:
- compressed_req_content["tools"] = [{"functionDeclarations": existing_funcs}]
-
- compressed_req = {
- "request": compressed_req_content,
- "metadata": metadata,
- }
-
- compressed_requests.append(compressed_req)
-
- if tokens_saved > 0:
- logger.debug(
- f"[{request_id}] Google batch request {idx}: "
- f"{original_tokens:,} -> {optimized_tokens:,} tokens "
- f"(saved {tokens_saved:,})"
- )
-
- except Exception as e:
- logger.warning(
- f"[{request_id}] Optimization failed for Google batch request {idx}: {e}"
- )
- # Pass through unchanged on failure
- compressed_requests.append(batch_req)
- total_optimized_tokens += original_tokens
-
- # Update body with compressed requests
- body["batch"]["input_config"]["requests"]["requests"] = compressed_requests
-
- optimization_latency = (time.time() - start_time) * 1000
-
- # Forward request to Google
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"
-
- # Add API key to URL if present in headers
- api_key = headers.pop("x-goog-api-key", None)
- if api_key:
- url = f"{url}?key={api_key}"
-
- try:
- response = await self._retry_request("POST", url, headers, body)
-
- # Record metrics
- await self.metrics.record_request(
- provider="google",
- model=f"batch:{model}",
- input_tokens=total_optimized_tokens,
- output_tokens=0,
- tokens_saved=total_tokens_saved,
- latency_ms=optimization_latency,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
-
- # Log compression stats
- if total_tokens_saved > 0:
- savings_percent = (
- (total_tokens_saved / total_original_tokens * 100)
- if total_original_tokens > 0
- else 0
- )
- logger.info(
- f"[{request_id}] Google batch compression: "
- f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "
- f"({savings_percent:.1f}% saved across {len(requests_list)} requests)"
- )
-
- # Store batch context for CCR result processing
- if response.status_code == 200 and self.config.ccr_inject_tool:
- try:
- response_data = response.json()
- batch_name = response_data.get("name")
- if batch_name:
- await self._store_google_batch_context(
- batch_name,
- requests_list,
- model,
- api_key,
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Failed to store Google batch context: {e}")
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- except Exception as e:
- logger.error(f"[{request_id}] Google batch request failed: {e}")
- return JSONResponse(
- status_code=500,
- content={
- "error": {
- "code": 500,
- "message": f"Failed to forward batch request: {e!s}",
- "status": "INTERNAL",
- }
- },
- )
-
- async def _google_batch_passthrough(
- self,
- request: Request,
- model: str,
- body: dict | None = None,
- ) -> Response:
- """Pass through Google batch request without modification."""
- start_time = time.time()
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
-
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"
-
- # Add API key to URL if present in headers
- api_key = headers.pop("x-goog-api-key", None)
- if api_key:
- url = f"{url}?key={api_key}"
-
- if body is None:
- body_content = await request.body()
- else:
- body_content = json.dumps(body).encode()
-
- response = await self.http_client.post( # type: ignore[union-attr]
- url,
- headers=headers,
- content=body_content,
- )
-
- # Track metrics
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider="google",
- model=f"passthrough:batch:{model}",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- )
-
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- async def handle_google_batch_passthrough(
- self,
- request: Request,
- batch_name: str | None = None,
- ) -> Response:
- """Handle Google batch passthrough endpoints.
-
- Used for:
- - GET /v1beta/batches/{batch_name} - Get batch status
- - POST /v1beta/batches/{batch_name}:cancel - Cancel batch
- - DELETE /v1beta/batches/{batch_name} - Delete batch
- """
- start_time = time.time()
- path = request.url.path
- url = f"{self.GEMINI_API_URL}{path}"
-
- # Preserve query string parameters
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
-
- # Handle API key
- api_key = headers.pop("x-goog-api-key", None)
- if api_key:
- if "?" in url:
- url = f"{url}&key={api_key}"
- else:
- url = f"{url}?key={api_key}"
-
- body = await request.body()
-
- response = await self.http_client.request( # type: ignore[union-attr]
- method=request.method,
- url=url,
- headers=headers,
- content=body,
- )
-
- # Track metrics
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider="google",
- model="passthrough:batches",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- )
-
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- async def _store_google_batch_context(
- self,
- batch_name: str,
- requests_list: list[dict[str, Any]],
- model: str,
- api_key: str | None,
- ) -> None:
- """Store Google batch context for CCR result processing.
-
- Args:
- batch_name: The batch name from the API response.
- requests_list: The original batch requests.
- model: The model used for the batch.
- api_key: The API key for continuation calls.
- """
- store = get_batch_context_store()
- context = BatchContext(
- batch_id=batch_name,
- provider="google",
- api_key=api_key,
- api_base_url=self.GEMINI_API_URL,
- )
-
- for batch_req in requests_list:
- metadata = batch_req.get("metadata", {})
- custom_id = metadata.get("key", "")
- req_content = batch_req.get("request", {})
- contents = req_content.get("contents", [])
- system_instruction = req_content.get("systemInstruction")
-
- # Convert contents to messages format for CCR handler
- messages, _ = self._gemini_contents_to_messages(contents, system_instruction)
-
- # Extract system instruction text if present
- sys_text = None
- if system_instruction:
- parts = system_instruction.get("parts", [])
- if parts and isinstance(parts[0], dict):
- sys_text = parts[0].get("text")
-
- context.add_request(
- BatchRequestContext(
- custom_id=custom_id,
- messages=messages,
- tools=req_content.get("tools"),
- model=model,
- system_instruction=sys_text,
- )
- )
-
- await store.store(context)
- logger.debug(
- f"Stored Google batch context for {batch_name} with {len(requests_list)} requests"
- )
-
- async def handle_google_batch_results(
- self,
- request: Request,
- batch_name: str,
- ) -> Response:
- """Handle Google batch results with CCR post-processing.
-
- Google batch results endpoint returns the batch operation status.
- When status is SUCCEEDED, results are embedded in the response.
- This handler processes CCR tool calls in those results.
- """
- start_time = time.time()
-
- # Forward request to get batch status/results
- url = f"{self.GEMINI_API_URL}/v1beta/{batch_name}"
-
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
-
- # Handle API key
- api_key = headers.pop("x-goog-api-key", None)
- if api_key:
- if "?" in url:
- url = f"{url}&key={api_key}"
- else:
- url = f"{url}?key={api_key}"
-
- response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]
-
- if response.status_code != 200:
- # Error - pass through
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Parse response
- try:
- response_data = response.json()
- except json.JSONDecodeError:
- # Not JSON - pass through
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Check if batch has results (state must be SUCCEEDED)
- metadata = response_data.get("metadata", {})
- state = metadata.get("state")
-
- if state != "SUCCEEDED":
- # Batch not complete - pass through
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Extract results from response
- # Google embeds results in the batch response
- results = response_data.get("response", {}).get("responses", [])
-
- if not results:
- # No results to process
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Check if we have context and CCR processing is enabled
- store = get_batch_context_store()
- batch_context = await store.get(batch_name)
-
- if batch_context is None or not self.config.ccr_inject_tool:
- # No context or CCR disabled - pass through
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Process results with CCR handler
- processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]
- processed = await processor.process_results(batch_name, results, "google")
-
- # Update response with processed results
- processed_results = [p.result for p in processed]
- response_data["response"]["responses"] = processed_results
-
- for p in processed:
- if p.was_processed:
- logger.info(
- f"CCR: Processed Google batch result {p.custom_id} "
- f"({p.continuation_rounds} continuation rounds)"
- )
-
- # Track metrics
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider="google",
- model="batch:ccr-processed",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- )
-
- return JSONResponse(content=response_data, status_code=200)
-
- def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:
- """Parse usage information from SSE chunk.
-
- For Anthropic: Looks for message_start (input tokens) and message_delta (output tokens)
- For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)
- For Gemini: Looks for usageMetadata in each chunk
-
- Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens
- Returns None if no usage found in this chunk.
- """
- try:
- text = chunk.decode("utf-8", errors="ignore")
- # SSE format: "data: {...}\n\n" or "event: ...\ndata: {...}\n\n"
- for line in text.split("\n"):
- if not line.startswith("data: "):
- continue
- data_str = line[6:].strip()
- if not data_str or data_str == "[DONE]":
- continue
-
- try:
- data = json.loads(data_str)
- except json.JSONDecodeError:
- continue
-
- usage = {}
-
- if provider == "anthropic":
- # Anthropic sends message_start with input tokens
- # and message_delta with output tokens
- event_type = data.get("type", "")
-
- if event_type == "message_start":
- msg = data.get("message", {})
- msg_usage = msg.get("usage", {})
- if msg_usage:
- usage["input_tokens"] = msg_usage.get("input_tokens", 0)
- usage["cache_read_input_tokens"] = msg_usage.get(
- "cache_read_input_tokens", 0
- )
- usage["cache_creation_input_tokens"] = msg_usage.get(
- "cache_creation_input_tokens", 0
- )
-
- elif event_type == "message_delta":
- delta_usage = data.get("usage", {})
- if delta_usage:
- usage["output_tokens"] = delta_usage.get("output_tokens", 0)
-
- elif provider == "openai":
- # OpenAI sends usage in final chunk (when stream_options.include_usage=true)
- chunk_usage = data.get("usage")
- if chunk_usage:
- usage["input_tokens"] = chunk_usage.get("prompt_tokens", 0)
- usage["output_tokens"] = chunk_usage.get("completion_tokens", 0)
- # OpenAI has cached tokens in prompt_tokens_details
- details = chunk_usage.get("prompt_tokens_details", {})
- usage["cache_read_input_tokens"] = details.get("cached_tokens", 0)
-
- elif provider == "gemini":
- # Gemini sends usageMetadata in each streaming chunk
- # Format: {"usageMetadata": {"promptTokenCount": N, "candidatesTokenCount": M}}
- usage_meta = data.get("usageMetadata")
- if usage_meta:
- usage["input_tokens"] = usage_meta.get("promptTokenCount", 0)
- usage["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)
- # Gemini also has cachedContentTokenCount for context caching
- usage["cache_read_input_tokens"] = usage_meta.get(
- "cachedContentTokenCount", 0
- )
-
- if usage:
- return usage
-
- except (UnicodeDecodeError, KeyError, TypeError) as e:
- # Don't fail streaming on parse errors
- logger.debug(f"SSE usage parsing error for {provider}: {e}")
-
- return None
-
- def _parse_sse_usage_from_buffer(
- self, stream_state: dict[str, Any], provider: str
- ) -> dict[str, int] | None:
- """Parse usage from buffered SSE data, handling split chunks.
-
- Processes complete SSE events (ending with double newline) from the buffer
- and removes them from the buffer. Incomplete events are kept in the buffer
- for the next chunk.
- """
- buffer = stream_state["sse_buffer"]
- usage_found: dict[str, int] = {}
-
- # Process complete SSE events (separated by double newlines)
- while "\n\n" in buffer:
- event_end = buffer.index("\n\n")
- event_text = buffer[: event_end + 2]
- buffer = buffer[event_end + 2 :]
-
- # Parse this complete event
- for line in event_text.split("\n"):
- if not line.startswith("data: "):
- continue
- data_str = line[6:].strip()
- if not data_str or data_str == "[DONE]":
- continue
-
- try:
- data = json.loads(data_str)
- except json.JSONDecodeError:
- continue
-
- if provider == "anthropic":
- event_type = data.get("type", "")
- if event_type == "message_start":
- msg = data.get("message", {})
- msg_usage = msg.get("usage", {})
- if msg_usage:
- usage_found["input_tokens"] = msg_usage.get("input_tokens", 0)
- usage_found["cache_read_input_tokens"] = msg_usage.get(
- "cache_read_input_tokens", 0
- )
- usage_found["cache_creation_input_tokens"] = msg_usage.get(
- "cache_creation_input_tokens", 0
- )
- # INFO logging for cache token tracking (temporary for debugging)
- logger.info(
- f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "
- f"cache_read={usage_found.get('cache_read_input_tokens')}, "
- f"cache_write={usage_found.get('cache_creation_input_tokens')}"
- )
- elif event_type == "message_delta":
- delta_usage = data.get("usage", {})
- if delta_usage:
- usage_found["output_tokens"] = delta_usage.get("output_tokens", 0)
-
- elif provider == "openai":
- chunk_usage = data.get("usage")
- if chunk_usage:
- usage_found["input_tokens"] = chunk_usage.get("prompt_tokens", 0)
- usage_found["output_tokens"] = chunk_usage.get("completion_tokens", 0)
- details = chunk_usage.get("prompt_tokens_details", {})
- usage_found["cache_read_input_tokens"] = details.get("cached_tokens", 0)
-
- elif provider == "gemini":
- usage_meta = data.get("usageMetadata")
- if usage_meta:
- usage_found["input_tokens"] = usage_meta.get("promptTokenCount", 0)
- usage_found["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)
- usage_found["cache_read_input_tokens"] = usage_meta.get(
- "cachedContentTokenCount", 0
- )
-
- # Update buffer with remaining incomplete data
- stream_state["sse_buffer"] = buffer
-
- return usage_found if usage_found else None
-
- def _parse_sse_to_response(self, sse_data: str, provider: str) -> dict[str, Any] | None:
- """Parse SSE data to reconstruct the API response JSON.
-
- Args:
- sse_data: Raw SSE data string.
- provider: Provider type for parsing.
-
- Returns:
- Reconstructed response dict or None if parsing fails.
- """
- if provider != "anthropic":
- return None # Only implemented for Anthropic
-
- response: dict[str, Any] = {"content": [], "usage": {}}
- current_block: dict[str, Any] | None = None
-
- for line in sse_data.split("\n"):
- if not line.startswith("data: "):
- continue
- data_str = line[6:].strip()
- if not data_str or data_str == "[DONE]":
- continue
-
- try:
- data = json.loads(data_str)
- except json.JSONDecodeError:
- continue
-
- event_type = data.get("type", "")
-
- if event_type == "message_start":
- msg = data.get("message", {})
- response["id"] = msg.get("id")
- response["model"] = msg.get("model")
- response["role"] = msg.get("role", "assistant")
- response["stop_reason"] = msg.get("stop_reason")
- if msg.get("usage"):
- response["usage"].update(msg["usage"])
-
- elif event_type == "content_block_start":
- block = data.get("content_block", {})
- current_block = {
- "type": block.get("type"),
- "index": data.get("index", len(response["content"])),
- }
- if block.get("type") == "text":
- current_block["text"] = block.get("text", "")
- elif block.get("type") == "tool_use":
- current_block["id"] = block.get("id")
- current_block["name"] = block.get("name")
- current_block["input"] = {}
-
- elif event_type == "content_block_delta":
- if current_block:
- delta = data.get("delta", {})
- if delta.get("type") == "text_delta":
- current_block["text"] = current_block.get("text", "") + delta.get(
- "text", ""
- )
- elif delta.get("type") == "input_json_delta":
- # Accumulate partial JSON for tool input
- partial = delta.get("partial_json", "")
- current_block["_partial_json"] = (
- current_block.get("_partial_json", "") + partial
- )
-
- elif event_type == "content_block_stop":
- if current_block:
- # Parse accumulated JSON for tool_use blocks
- if current_block.get("type") == "tool_use" and "_partial_json" in current_block:
- try:
- current_block["input"] = json.loads(current_block["_partial_json"])
- except json.JSONDecodeError:
- current_block["input"] = {}
- del current_block["_partial_json"]
- response["content"].append(current_block)
- current_block = None
-
- elif event_type == "message_delta":
- delta = data.get("delta", {})
- if delta.get("stop_reason"):
- response["stop_reason"] = delta["stop_reason"]
- if data.get("usage"):
- response["usage"].update(data["usage"])
-
- return response if response.get("content") else None
-
- def _response_to_sse(self, response: dict[str, Any], provider: str) -> list[bytes]:
- """Convert a response dict back to SSE format.
-
- Args:
- response: API response dict.
- provider: Provider type for formatting.
-
- Returns:
- List of SSE event bytes.
- """
- if provider != "anthropic":
- return []
-
- events: list[bytes] = []
-
- # message_start
- msg_start = {
- "type": "message_start",
- "message": {
- "id": response.get("id", "msg_generated"),
- "type": "message",
- "role": response.get("role", "assistant"),
- "model": response.get("model", "unknown"),
- "content": [],
- "stop_reason": None,
- "usage": response.get("usage", {}),
- },
- }
- events.append(f"event: message_start\ndata: {json.dumps(msg_start)}\n\n".encode())
-
- # Content blocks
- for idx, block in enumerate(response.get("content", [])):
- # content_block_start
- if block.get("type") == "text":
- block_start = {
- "type": "content_block_start",
- "index": idx,
- "content_block": {"type": "text", "text": ""},
- }
- elif block.get("type") == "tool_use":
- block_start = {
- "type": "content_block_start",
- "index": idx,
- "content_block": {
- "type": "tool_use",
- "id": block.get("id", f"toolu_{idx}"),
- "name": block.get("name", ""),
- "input": {},
- },
- }
- else:
- continue
-
- events.append(
- f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n".encode()
- )
-
- # content_block_delta(s)
- if block.get("type") == "text" and block.get("text"):
- delta = {
- "type": "content_block_delta",
- "index": idx,
- "delta": {"type": "text_delta", "text": block["text"]},
- }
- events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())
- elif block.get("type") == "tool_use" and block.get("input"):
- delta = {
- "type": "content_block_delta",
- "index": idx,
- "delta": {
- "type": "input_json_delta",
- "partial_json": json.dumps(block["input"]),
- },
- }
- events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())
-
- # content_block_stop
- block_stop = {"type": "content_block_stop", "index": idx}
- events.append(f"event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n".encode())
-
- # message_delta
- msg_delta = {
- "type": "message_delta",
- "delta": {"stop_reason": response.get("stop_reason", "end_turn")},
- "usage": {"output_tokens": response.get("usage", {}).get("output_tokens", 0)},
- }
- events.append(f"event: message_delta\ndata: {json.dumps(msg_delta)}\n\n".encode())
-
- # message_stop
- events.append(b'event: message_stop\ndata: {"type": "message_stop"}\n\n')
-
- return events
-
- def _record_ccr_feedback_from_response(
- self, response: dict, provider: str, request_id: str
- ) -> None:
- """Extract headroom_retrieve tool calls from a response and record feedback.
-
- This closes the TOIN feedback loop for streaming responses where
- the proxy can't intercept and handle retrieval calls inline.
- """
- from headroom.cache.compression_store import get_compression_store
-
- content = response.get("content", [])
- if not isinstance(content, list):
- return
-
- store = get_compression_store()
-
- for block in content:
- if not isinstance(block, dict):
- continue
- if block.get("type") != "tool_use":
- continue
- if block.get("name") != "headroom_retrieve":
- continue
-
- input_data = block.get("input", {})
- hash_key = input_data.get("hash")
- query = input_data.get("query")
-
- if not hash_key:
- continue
-
- logger.info(
- f"[{request_id}] CCR Feedback: Recording retrieval "
- f"hash={hash_key[:8]}... query={query!r}"
- )
-
- # Call store.retrieve()/search() for the side effect of triggering
- # the feedback chain: _log_retrieval -> process_pending_feedback
- # -> toin.record_retrieval(). We discard the returned content.
- try:
- if query:
- store.search(hash_key, query)
- else:
- store.retrieve(hash_key, query=None)
- except Exception as e:
- logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")
-
- async def _stream_response(
- self,
- url: str,
- headers: dict,
- body: dict,
- provider: str,
- model: str,
- request_id: str,
- original_tokens: int,
- optimized_tokens: int,
- tokens_saved: int,
- transforms_applied: list[str],
- tags: dict[str, str],
- optimization_latency: float,
- memory_user_id: str | None = None,
- pipeline_timing: dict[str, float] | None = None,
- prefix_tracker: Any | None = None,
- ) -> StreamingResponse:
- """Stream response with metrics tracking and memory tool handling.
-
- Parses SSE events to extract actual usage information from the API response
- for accurate token counting and cost calculation.
-
- When memory is enabled (memory_user_id provided), this method:
- 1. Buffers the response to detect memory tool calls
- 2. Executes memory tools if found
- 3. Makes continuation requests until no memory tools remain
- 4. Streams the final response to the client
- """
- start_time = time.time()
-
- # Mutable state for the generator to update
- stream_state: dict[str, Any] = {
- "input_tokens": None,
- "output_tokens": None,
- "cache_read_input_tokens": 0,
- "cache_creation_input_tokens": 0,
- "total_bytes": 0,
- "sse_buffer": "", # Buffer for incomplete SSE events
- "ttfb_ms": None, # Time to first byte from upstream
- }
-
- # Track if we need to handle memory tools
- memory_enabled = (
- memory_user_id is not None
- and self.memory_handler is not None
- and provider == "anthropic"
- )
-
- # Open connection before generator to capture upstream response headers
- # (needed to forward ratelimit headers to the client via StreamingResponse)
- assert self.http_client is not None, "http_client must be initialized before streaming"
- try:
- _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)
- upstream_response = await self.http_client.send(_upstream_req, stream=True)
- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
- error_msg = str(e)
- logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")
-
- async def _error_gen():
- error_event = {
- "type": "error",
- "error": {
- "type": "connection_error",
- "message": f"Failed to connect to upstream API: {error_msg}",
- },
- }
- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
-
- return StreamingResponse(_error_gen(), media_type="text/event-stream")
-
- # Forward upstream ratelimit headers to the client
- forwarded_headers = {
- k: v for k, v in upstream_response.headers.items() if "ratelimit" in k.lower()
- }
-
- async def generate():
- nonlocal body, memory_enabled # May need to modify for continuation requests
-
- # For memory mode, we buffer the response to check for tool calls
- buffered_chunks: list[bytes] = []
- full_sse_data = ""
-
- try:
- async with contextlib.aclosing(upstream_response) as response:
- async for chunk in response.aiter_bytes():
- # Record TTFB on first chunk
- if stream_state["ttfb_ms"] is None:
- stream_state["ttfb_ms"] = (time.time() - start_time) * 1000
-
- stream_state["total_bytes"] += len(chunk)
-
- # Buffer SSE data to handle chunks split across calls
- chunk_str = chunk.decode("utf-8", errors="ignore")
- stream_state["sse_buffer"] += chunk_str
-
- # Safety: prevent unbounded buffer growth
- if len(stream_state["sse_buffer"]) > MAX_SSE_BUFFER_SIZE:
- logger.error(
- "SSE buffer exceeded maximum size (%d bytes), "
- "truncating to prevent memory exhaustion",
- MAX_SSE_BUFFER_SIZE,
- )
- stream_state["sse_buffer"] = stream_state["sse_buffer"][
- -MAX_SSE_BUFFER_SIZE // 2 :
- ]
-
- # Always stream immediately — buffering breaks
- # real-time clients (LangGraph, LangChain, etc.)
- yield chunk
-
- if memory_enabled:
- # Also buffer for post-stream memory processing
- buffered_chunks.append(chunk)
- full_sse_data += chunk_str
- if len(full_sse_data) > MAX_SSE_BUFFER_SIZE:
- logger.warning(
- "Memory-mode SSE buffer exceeded maximum size, "
- "disabling memory detection for this request"
- )
- memory_enabled = False
-
- # Parse complete SSE events from buffer
- usage = self._parse_sse_usage_from_buffer(stream_state, provider)
- if usage:
- if "input_tokens" in usage:
- stream_state["input_tokens"] = usage["input_tokens"]
- if "output_tokens" in usage:
- stream_state["output_tokens"] = usage["output_tokens"]
- if "cache_read_input_tokens" in usage:
- stream_state["cache_read_input_tokens"] = usage[
- "cache_read_input_tokens"
- ]
- if "cache_creation_input_tokens" in usage:
- stream_state["cache_creation_input_tokens"] = usage[
- "cache_creation_input_tokens"
- ]
-
- # Memory tool handling after stream completes
- # Chunks were already yielded in real-time above, so we only
- # do silent background processing here — no yielding.
- if memory_enabled and full_sse_data:
- # Check for Claude Code credential error
- if "only authorized for use with Claude Code" in full_sse_data:
- logger.warning(
- f"[{request_id}] Memory: Claude Code subscription credentials "
- "do not support custom tool injection. Set ANTHROPIC_API_KEY "
- "environment variable or use --no-memory-tools flag."
- )
- return
-
- # Parse SSE to get response JSON
- parsed_response = self._parse_sse_to_response(full_sse_data, provider)
-
- if parsed_response and self.memory_handler.has_memory_tool_calls(
- parsed_response, provider
- ):
- logger.info(
- f"[{request_id}] Memory: Detected tool calls in streaming response"
- )
-
- # Execute memory tool calls silently — response already
- # streamed so we cannot make a continuation request.
- tool_results = await self.memory_handler.handle_memory_tool_calls(
- parsed_response, memory_user_id, provider
- )
- if tool_results:
- logger.info(
- f"[{request_id}] Memory: Tool calls executed silently "
- "(streaming mode — no continuation)"
- )
-
- # CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.
- # In streaming mode, the client handles actual retrieval, but we
- # still need to record the event so TOIN learns which fields matter.
- if self.config.ccr_inject_tool and full_sse_data:
- ccr_parsed = (
- parsed_response
- if parsed_response
- else self._parse_sse_to_response(full_sse_data, provider)
- )
- if ccr_parsed:
- self._record_ccr_feedback_from_response(ccr_parsed, provider, request_id)
-
- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
- logger.error(f"[{request_id}] Connection error to upstream API: {e}")
- error_event = {
- "type": "error",
- "error": {
- "type": "connection_error",
- "message": f"Failed to connect to upstream API: {e}",
- },
- }
- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
- except httpx.HTTPStatusError as e:
- logger.error(f"[{request_id}] HTTP error from upstream API: {e}")
- # Forward the upstream error response
- yield e.response.content
- except Exception as e:
- logger.error(f"[{request_id}] Unexpected streaming error: {e}")
- error_event = {
- "type": "error",
- "error": {"type": "api_error", "message": str(e)},
- }
- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
- finally:
- # Record metrics after stream completes
- total_latency = (time.time() - start_time) * 1000
-
- # Use actual output tokens from API if available, otherwise estimate
- output_tokens = stream_state["output_tokens"]
- if output_tokens is None:
- # Fallback: estimate from bytes (but this is inaccurate for SSE)
- # Use a more conservative estimate - SSE overhead is ~10-20x
- output_tokens = stream_state["total_bytes"] // 40
- logger.debug(
- f"[{request_id}] No usage in stream, estimated {output_tokens} output tokens"
- )
-
- # Use optimized_tokens for dashboard metrics (what we actually sent).
- # API's input_tokens is the non-cached portion only, which is
- # misleading for aggregation (often just 1 with prompt caching).
- cache_read_tokens = stream_state["cache_read_input_tokens"]
- cache_write_tokens = stream_state["cache_creation_input_tokens"]
- uncached_input_tokens = stream_state.get("input_tokens") or 0
-
- # Structured perf log line for `headroom perf` analysis
- num_msgs = len(body.get("messages", []))
- cache_hit_pct = (
- round(cache_read_tokens / (cache_read_tokens + cache_write_tokens) * 100)
- if (cache_read_tokens + cache_write_tokens) > 0
- else 0
- )
- logger.info(
- f"[{request_id}] PERF "
- f"model={model} msgs={num_msgs} "
- f"tok_before={original_tokens} tok_after={optimized_tokens} "
- f"tok_saved={tokens_saved} "
- f"cache_read={cache_read_tokens} cache_write={cache_write_tokens} "
- f"cache_hit_pct={cache_hit_pct} "
- f"opt_ms={optimization_latency:.0f} "
- f"transforms={_summarize_transforms(transforms_applied)}"
- )
-
- # Update prefix cache tracker for next turn (streaming path)
- if prefix_tracker is not None:
- prefix_tracker.update_from_response(
- cache_read_tokens=cache_read_tokens,
- cache_write_tokens=cache_write_tokens,
- messages=body.get("messages", []),
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(
- model,
- tokens_saved,
- optimized_tokens,
- cache_read_tokens=cache_read_tokens,
- cache_write_tokens=cache_write_tokens,
- uncached_tokens=uncached_input_tokens,
- )
-
- await self.metrics.record_request(
- provider=provider,
- model=model,
- input_tokens=optimized_tokens, # What we sent, not API's non-cached count
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- ttfb_ms=stream_state["ttfb_ms"] or 0,
- pipeline_timing=pipeline_timing,
- cache_read_tokens=cache_read_tokens,
- cache_write_tokens=cache_write_tokens,
- uncached_input_tokens=uncached_input_tokens,
- )
-
- return StreamingResponse(
- generate(),
- media_type="text/event-stream",
- headers=forwarded_headers,
- )
-
- async def _stream_response_bedrock(
- self,
- body: dict,
- headers: dict,
- provider: str,
- model: str,
- request_id: str,
- original_tokens: int,
- optimized_tokens: int,
- tokens_saved: int,
- transforms_applied: list[str],
- tags: dict[str, str],
- optimization_latency: float,
- pipeline_timing: dict[str, float] | None = None,
- ) -> StreamingResponse:
- """Stream response from Bedrock backend with metrics tracking.
-
- Translates Bedrock streaming events to Anthropic SSE format.
- """
- start_time = time.time()
-
- # Mutable state for the generator
- stream_state: dict[str, Any] = {
- "input_tokens": 0,
- "output_tokens": 0,
- "ttfb_ms": None,
- }
-
- async def generate():
- try:
- assert self.anthropic_backend is not None
-
- async for event in self.anthropic_backend.stream_message(body, headers):
- # Record TTFB on first event
- if stream_state["ttfb_ms"] is None:
- stream_state["ttfb_ms"] = (time.time() - start_time) * 1000
-
- # Format as SSE
- if event.raw_sse:
- yield event.raw_sse.encode()
- else:
- sse_line = f"event: {event.event_type}\ndata: {json.dumps(event.data)}\n\n"
- yield sse_line.encode()
-
- # Track usage from message_start event
- if event.event_type == "message_start":
- msg = event.data.get("message", {})
- usage = msg.get("usage", {})
- if "input_tokens" in usage:
- stream_state["input_tokens"] = usage["input_tokens"]
-
- # Track output tokens from message_delta
- if event.event_type == "message_delta":
- usage = event.data.get("usage", {})
- if "output_tokens" in usage:
- stream_state["output_tokens"] = usage["output_tokens"]
-
- # Handle errors
- if event.event_type == "error":
- logger.error(f"[{request_id}] Bedrock stream error: {event.data}")
-
- except Exception as e:
- logger.error(f"[{request_id}] Bedrock streaming error: {e}")
- error_event = {
- "type": "error",
- "error": {"type": "api_error", "message": str(e)},
- }
- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
-
- finally:
- # Record metrics
- total_latency = (time.time() - start_time) * 1000
- output_tokens = stream_state["output_tokens"]
-
- _backend_name = (
- self.anthropic_backend.name if self.anthropic_backend else "anthropic"
- )
- await self.metrics.record_request(
- provider=_backend_name,
- model=model,
- input_tokens=optimized_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- cached=False,
- overhead_ms=optimization_latency,
- ttfb_ms=stream_state["ttfb_ms"] or 0,
- pipeline_timing=pipeline_timing,
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)
-
- # Log request
- if self.logger:
- self.logger.log(
- RequestLog(
- request_id=request_id,
- timestamp=datetime.now().isoformat(),
- provider=_backend_name,
- model=model,
- input_tokens_original=original_tokens,
- input_tokens_optimized=optimized_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- savings_percent=(tokens_saved / original_tokens * 100)
- if original_tokens > 0
- else 0,
- optimization_latency_ms=optimization_latency,
- total_latency_ms=total_latency,
- tags=tags,
- cache_hit=False,
- transforms_applied=transforms_applied,
- request_messages=body.get("messages")
- if self.config.log_full_messages
- else None,
- )
- )
-
- # Structured perf log line for `headroom perf` analysis
- num_msgs = len(body.get("messages", []))
- logger.info(
- f"[{request_id}] PERF "
- f"model={model} msgs={num_msgs} "
- f"tok_before={original_tokens} tok_after={optimized_tokens} "
- f"tok_saved={tokens_saved} "
- f"cache_read=0 cache_write=0 cache_hit_pct=0 "
- f"opt_ms={optimization_latency:.0f} "
- f"transforms={_summarize_transforms(transforms_applied)}"
- )
-
- return StreamingResponse(
- generate(),
- media_type="text/event-stream",
- )
-
- async def _stream_openai_via_backend(
- self,
- body: dict,
- headers: dict,
- model: str,
- request_id: str,
- start_time: float,
- original_tokens: int,
- optimized_tokens: int,
- tokens_saved: int,
- transforms_applied: list[str],
- tags: dict[str, str],
- optimization_latency: float,
- pipeline_timing: dict[str, float] | None = None,
- ) -> StreamingResponse:
- """Stream OpenAI chat completion response from backend.
-
- Routes stream:true requests through the backend's stream_openai_message(),
- yielding SSE events to the client.
- """
- assert self.anthropic_backend is not None
-
- async def generate():
- try:
- async for sse_chunk in self.anthropic_backend.stream_openai_message(body, headers):
- yield sse_chunk.encode() if isinstance(sse_chunk, str) else sse_chunk
- except Exception as e:
- logger.error(f"[{request_id}] Backend streaming error: {e}")
- error_data = {
- "error": {
- "message": str(e),
- "type": "api_error",
- "code": "backend_error",
- }
- }
- yield f"data: {json.dumps(error_data)}\n\n".encode()
- yield b"data: [DONE]\n\n"
- finally:
- total_latency = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider=self.anthropic_backend.name,
- model=model,
- input_tokens=optimized_tokens,
- output_tokens=0, # Unknown in streaming
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- cached=False,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name} [stream]"
- )
-
- return StreamingResponse(
- generate(),
- media_type="text/event-stream",
- )
-
- async def handle_openai_chat(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle OpenAI /v1/chat/completions endpoint."""
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "type": "invalid_request_error",
- "code": "request_too_large",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
- model = body.get("model", "unknown")
- messages = body.get("messages", [])
-
- # Validate message array size
- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Message array too large ({len(messages)} messages). "
- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
- "type": "invalid_request_error",
- "code": "invalid_request",
- }
- },
- )
-
- stream = body.get("stream", False)
-
- # Bypass: skip ALL compression for explicit opt-out
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- if _bypass:
- logger.info(f"[{request_id}] Bypass: skipping compression (header)")
-
- # Image compression (before text optimization)
- if self.config.image_optimize and messages and not _bypass:
- compressor = _get_image_compressor()
- if compressor and compressor.has_images(messages):
- messages = compressor.compress(messages, provider="openai")
- if compressor.last_result:
- logger.info(
- f"Image compression: {compressor.last_result.technique.value} "
- f"({compressor.last_result.savings_percent:.0f}% saved, "
- f"{compressor.last_result.original_tokens} -> "
- f"{compressor.last_result.compressed_tokens} tokens)"
- )
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- tags = self._extract_tags(headers)
-
- # Rate limiting
- if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited()
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Check cache
- if self.cache and not stream:
- cached = await self.cache.get(messages, model)
- if cached:
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0, # Savings already counted when response was cached
- latency_ms=(time.time() - start_time) * 1000,
- cached=True,
- )
-
- # Remove compression headers from cached response
- response_headers = dict(cached.response_headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(content=cached.response_body, headers=response_headers)
-
- # Token counting
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Hook: pre_compress
- _hook_biases = None
- if self.config.hooks:
- from headroom.hooks import CompressContext
-
- _hook_ctx = CompressContext(model=model, provider="openai")
- try:
- messages = self.config.hooks.pre_compress(messages, _hook_ctx)
- _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
- except Exception as e:
- logger.debug(f"[{request_id}] Hook error: {e}")
-
- # Optimization
- transforms_applied = []
- pipeline_timing: dict[str, float] = {}
- waste_signals_dict: dict[str, int] | None = None
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- # Get prefix cache tracker for this session
- openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
- openai_prefix_tracker = self.session_tracker_store.get_or_create(
- openai_session_id, "openai"
- )
- openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
-
- _compression_failed = False
- original_messages = messages # Preserve for 400-retry fallback
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
- if self.config.optimize and messages and not _bypass and _license_ok:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
-
- if self.config.mode == "token_headroom":
- comp_cache = self._get_compression_cache(openai_session_id)
-
- # Zone 1: Swap cached compressed versions
- working_messages = comp_cache.apply_cached(messages)
-
- # Re-freeze boundary
- openai_frozen_count = comp_cache.compute_frozen_count(messages)
-
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=working_messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(working_messages),
- frozen_message_count=openai_frozen_count,
- biases=_hook_biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != working_messages:
- comp_cache.update_from_result(messages, result.messages)
-
- # Always use pipeline result in token_headroom mode
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- # Keep original_tokens as the REAL original (pre-Zone-1-swap)
- # so tokens_saved captures both Zone 1 + Zone 2 savings.
- optimized_tokens = result.tokens_after
- else:
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- frozen_message_count=openai_frozen_count,
- biases=_hook_biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
-
- if result.waste_signals:
- waste_signals_dict = result.waste_signals.to_dict()
- except Exception as e:
- logger.warning(f"Optimization failed: {e}")
- # Flag compression failure for observability
- _compression_failed = True
-
- tokens_saved = max(0, original_tokens - optimized_tokens)
- optimization_latency = (time.time() - start_time) * 1000
-
- # Hook: post_compress
- if self.config.hooks and tokens_saved > 0:
- from headroom.hooks import CompressEvent
-
- try:
- self.config.hooks.post_compress(
- CompressEvent(
- tokens_before=original_tokens,
- tokens_after=optimized_tokens,
- tokens_saved=tokens_saved,
- compression_ratio=tokens_saved / original_tokens
- if original_tokens > 0
- else 0,
- transforms_applied=transforms_applied,
- model=model,
- provider="openai",
- )
- )
- except Exception as e:
- logger.debug(f"[{request_id}] post_compress hook error: {e}")
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = body.get("tools")
- _original_tools = tools # Preserve for diagnostic / future retry
- if (
- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
- ) and not _bypass:
- injector = CCRToolInjector(
- provider="openai",
- inject_tool=self.config.ccr_inject_tool,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- optimized_messages, tools, was_injected = injector.process_request(
- optimized_messages, tools
- )
-
- if injector.has_compressed_content:
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
- )
- else:
- logger.debug(
- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
- )
-
- # Query Echo: disabled — hurts prefix caching in long conversations.
-
- body["messages"] = optimized_messages
- if tools is not None:
- body["tools"] = tools
-
- # Route through LiteLLM/any-llm backend if configured
- if self.anthropic_backend is not None:
- try:
- if stream:
- # Streaming: use stream_openai_message() → SSE events
- return await self._stream_openai_via_backend(
- body,
- headers,
- model,
- request_id,
- start_time,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- )
- else:
- # Non-streaming: use send_openai_message() → JSON
- backend_response = await self.anthropic_backend.send_openai_message(
- body, headers
- )
-
- if backend_response.error:
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
-
- # Track metrics
- total_latency = (time.time() - start_time) * 1000
- usage = backend_response.body.get("usage", {})
- output_tokens = usage.get("completion_tokens", 0)
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
-
- await self.metrics.record_request(
- provider=self.anthropic_backend.name,
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- cached=False,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"
- )
-
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
- except Exception as e:
- logger.error(f"[{request_id}] Backend error: {e}")
- return JSONResponse(
- status_code=500,
- content={
- "error": {
- "message": str(e),
- "type": "api_error",
- "code": "backend_error",
- }
- },
- )
-
- # Direct OpenAI API (no backend configured)
- url = f"{self.OPENAI_API_URL}/v1/chat/completions"
-
- try:
- if stream:
- # Inject stream_options to get usage stats in streaming response
- # This allows accurate token counting instead of byte-based estimation
- if "stream_options" not in body:
- body["stream_options"] = {"include_usage": True}
- elif isinstance(body.get("stream_options"), dict):
- body["stream_options"]["include_usage"] = True
-
- return await self._stream_response(
- url,
- headers,
- body,
- "openai",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- prefix_tracker=openai_prefix_tracker,
- )
- else:
- response = await self._retry_request("POST", url, headers, body)
-
- # Full diagnostic dump on upstream errors (OpenAI handler)
- if response.status_code >= 400:
- try:
- err_body = response.json()
- err_msg = err_body.get("error", {}).get("message", "")
- err_type = err_body.get("error", {}).get("type", "")
- except Exception:
- err_body = {"raw": response.text[:2000]}
- err_msg = str(response.text[:500])
- err_type = "parse_error"
-
- logger.warning(
- f"[{request_id}] UPSTREAM_ERROR "
- f"status={response.status_code} "
- f"error_type={err_type} "
- f"error_msg={err_msg!r} "
- f"model={model} "
- f"compressed={'yes' if transforms_applied else 'no'} "
- f"transforms={transforms_applied} "
- f"original_tokens={original_tokens} "
- f"optimized_tokens={optimized_tokens} "
- f"message_count={len(body.get('messages', []))} "
- f"stream={stream}"
- )
-
- try:
- debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"
- debug_dir.mkdir(parents=True, exist_ok=True)
- ts = datetime.now().strftime("%Y%m%d_%H%M%S")
- debug_file = debug_dir / f"{ts}_{request_id}.json"
-
- safe_headers = {}
- for k, v in headers.items():
- if k.lower() in ("x-api-key", "authorization"):
- safe_headers[k] = v[:12] + "..." if v else ""
- else:
- safe_headers[k] = v
-
- debug_payload = {
- "request_id": request_id,
- "timestamp": datetime.now().isoformat(),
- "status_code": response.status_code,
- "error_response": err_body,
- "model": model,
- "stream": stream,
- "headers": safe_headers,
- "compression": {
- "was_compressed": bool(transforms_applied),
- "transforms": transforms_applied,
- "original_tokens": original_tokens,
- "optimized_tokens": optimized_tokens,
- "tokens_saved": tokens_saved,
- "compression_failed": _compression_failed,
- },
- "tools_sent": body.get("tools"),
- "tool_count": len(body.get("tools") or []),
- "original_tool_count": len(_original_tools or []),
- "messages_sent": body.get("messages"),
- "message_count": len(body.get("messages", [])),
- "original_messages": (
- original_messages
- if original_messages is not body.get("messages")
- else "__same_as_sent__"
- ),
- "original_message_count": len(original_messages),
- "system_prompt": body.get("system"),
- }
-
- with open(debug_file, "w") as f:
- json.dump(debug_payload, f, indent=2, default=str)
-
- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
- except Exception as dump_err:
- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
-
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = optimized_tokens # fallback
- output_tokens = 0
- cache_read_tokens = 0
- try:
- resp_json = response.json()
- usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
- output_tokens = usage.get("completion_tokens", 0)
- # OpenAI returns cached_tokens in prompt_tokens_details
- # These are charged at 50% of the input price
- prompt_details = usage.get("prompt_tokens_details", {})
- cache_read_tokens = prompt_details.get("cached_tokens", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
- )
-
- # Update prefix cache tracker for next turn
- openai_prefix_tracker.update_from_response(
- cache_read_tokens=cache_read_tokens,
- cache_write_tokens=0, # OpenAI doesn't report write tokens
- messages=optimized_messages,
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(
- model,
- tokens_saved,
- optimized_tokens,
- cache_read_tokens=cache_read_tokens,
- )
-
- # Cache
- if self.cache and response.status_code == 200:
- await self.cache.set(
- messages, model, response.content, dict(response.headers), tokens_saved
- )
-
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- waste_signals=waste_signals_dict,
- cache_read_tokens=cache_read_tokens,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens)"
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Inject Headroom compression metrics (for SaaS metering)
- response_headers["x-headroom-tokens-before"] = str(original_tokens)
- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
- response_headers["x-headroom-model"] = model
- if transforms_applied:
- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
- if cache_read_tokens > 0:
- response_headers["x-headroom-cached"] = "true"
- if _compression_failed:
- response_headers["x-headroom-compression-failed"] = "true"
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed()
- # Log full error details internally for debugging
- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
- # Return sanitized error message to client (don't expose internal details)
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "type": "server_error",
- "code": "proxy_error",
- }
- },
- )
-
- async def handle_passthrough(
- self,
- request: Request,
- base_url: str,
- endpoint_name: str | None = None,
- provider: str | None = None,
- ) -> Response:
- """Pass through request unchanged.
-
- Args:
- request: The incoming request
- base_url: The upstream API base URL
- endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")
- provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")
- """
- start_time = time.time()
- path = request.url.path
- url = f"{base_url}{path}"
-
- # Preserve query string parameters
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
-
- body = await request.body()
-
- response = await self.http_client.request( # type: ignore[union-attr]
- method=request.method,
- url=url,
- headers=headers,
- content=body,
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Track stats for passthrough requests
- if endpoint_name and provider:
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider=provider,
- model=f"passthrough:{endpoint_name}",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- cached=False,
- )
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # =========================================================================
- # Databricks Native API
- # =========================================================================
-
- async def handle_databricks_invocations(
- self,
- request: Request,
- model: str,
- ) -> Response | StreamingResponse:
- """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.
-
- This enables using the Databricks CLI directly with Headroom:
- databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'
-
- The request/response format is identical to OpenAI chat completions,
- so we inject the model from the path and delegate to handle_openai_chat.
- """
- request_id = await self._next_request_id()
-
- try:
- body = await _read_request_json(request)
- except Exception as e:
- logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")
- return JSONResponse(
- status_code=400,
- content={
- "error": {"message": f"Invalid JSON: {e}", "type": "invalid_request_error"}
- },
- )
-
- # Inject model from path into body (Databricks CLI passes model in URL, not body)
- body["model"] = model
-
- logger.info(f"[{request_id}] Databricks invocation: model={model}")
-
- # Create a new request with the modified body
- # We reuse the OpenAI chat handler since the format is identical
- from starlette.requests import Request as StarletteRequest
-
- # Build new scope with the body already parsed
- scope = dict(request.scope)
-
- # Create a simple receive function that returns our modified body
- body_bytes = json.dumps(body).encode()
-
- async def receive():
- return {"type": "http.request", "body": body_bytes}
-
- modified_request = StarletteRequest(scope, receive)
-
- # Delegate to the OpenAI chat handler (same format)
- return await self.handle_openai_chat(modified_request)
-
- # =========================================================================
- # OpenAI Batch API with Compression
- # =========================================================================
-
- async def handle_batch_create(self, request: Request) -> Response:
- """Handle POST /v1/batches - Create a batch with compression.
-
- Flow:
- 1. Parse request to get input_file_id
- 2. Download the JSONL file content from OpenAI
- 3. Parse each line and compress the messages
- 4. Create a new compressed JSONL file
- 5. Upload compressed file to OpenAI
- 6. Create batch with the new compressed file_id
- 7. Return batch object with compression stats in metadata
- """
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
-
- input_file_id = body.get("input_file_id")
- endpoint = body.get("endpoint")
- completion_window = body.get("completion_window", "24h")
- metadata = body.get("metadata", {})
-
- if not input_file_id:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": "input_file_id is required",
- "type": "invalid_request_error",
- "code": "missing_parameter",
- }
- },
- )
-
- if not endpoint:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": "endpoint is required",
- "type": "invalid_request_error",
- "code": "missing_parameter",
- }
- },
- )
-
- # Only compress chat completions endpoint
- if endpoint != "/v1/chat/completions":
- # Pass through for other endpoints
- return await self._batch_passthrough(request, body)
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
-
- try:
- # Step 1: Download the input file from OpenAI
- logger.info(f"[{request_id}] Batch: Downloading input file {input_file_id}")
- file_content = await self._download_openai_file(input_file_id, headers)
-
- if file_content is None:
- return JSONResponse(
- status_code=404,
- content={
- "error": {
- "message": f"Failed to download file {input_file_id}",
- "type": "invalid_request_error",
- "code": "file_not_found",
- }
- },
- )
-
- # Step 2: Parse and compress each line
- logger.info(f"[{request_id}] Batch: Compressing JSONL content")
- compressed_lines, stats = await self._compress_batch_jsonl(file_content, request_id)
-
- if stats["total_requests"] == 0:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": "No valid requests found in input file",
- "type": "invalid_request_error",
- "code": "empty_file",
- }
- },
- )
-
- # Step 3: Create compressed JSONL content
- compressed_content = "\n".join(compressed_lines)
-
- # Step 4: Upload compressed file to OpenAI
- logger.info(f"[{request_id}] Batch: Uploading compressed file")
- new_file_id = await self._upload_openai_file(
- compressed_content, f"compressed_{input_file_id}.jsonl", headers
- )
-
- if new_file_id is None:
- return JSONResponse(
- status_code=500,
- content={
- "error": {
- "message": "Failed to upload compressed file",
- "type": "server_error",
- "code": "upload_failed",
- }
- },
- )
-
- # Step 5: Create batch with compressed file
- logger.info(f"[{request_id}] Batch: Creating batch with compressed file {new_file_id}")
-
- # Add compression stats to metadata
- compression_metadata = {
- **metadata,
- "headroom_compressed": "true",
- "headroom_original_file_id": input_file_id,
- "headroom_total_requests": str(stats["total_requests"]),
- "headroom_tokens_saved": str(stats["total_tokens_saved"]),
- "headroom_original_tokens": str(stats["total_original_tokens"]),
- "headroom_compressed_tokens": str(stats["total_compressed_tokens"]),
- "headroom_savings_percent": f"{stats['savings_percent']:.1f}",
- }
-
- batch_body = {
- "input_file_id": new_file_id,
- "endpoint": endpoint,
- "completion_window": completion_window,
- "metadata": compression_metadata,
- }
-
- url = f"{self.OPENAI_API_URL}/v1/batches"
- response = await self.http_client.post(url, json=batch_body, headers=headers) # type: ignore[union-attr]
-
- total_latency = (time.time() - start_time) * 1000
-
- # Log compression stats
- logger.info(
- f"[{request_id}] Batch created: {stats['total_requests']} requests, "
- f"{stats['total_original_tokens']:,} -> {stats['total_compressed_tokens']:,} tokens "
- f"(saved {stats['total_tokens_saved']:,} tokens, {stats['savings_percent']:.1f}%) "
- f"in {total_latency:.0f}ms"
- )
-
- # Record metrics
- await self.metrics.record_request(
- provider="openai",
- model="batch",
- input_tokens=stats["total_compressed_tokens"],
- output_tokens=0,
- tokens_saved=stats["total_tokens_saved"],
- latency_ms=total_latency,
- )
-
- # Return response with compression info in headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- response_headers["x-headroom-tokens-saved"] = str(stats["total_tokens_saved"])
- response_headers["x-headroom-savings-percent"] = f"{stats['savings_percent']:.1f}"
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- except Exception as e:
- logger.error(f"[{request_id}] Batch creation failed: {type(e).__name__}: {e}")
- await self.metrics.record_failed()
- return JSONResponse(
- status_code=500,
- content={
- "error": {
- "message": "An error occurred while processing the batch request",
- "type": "server_error",
- "code": "batch_processing_error",
- }
- },
- )
-
- async def _download_openai_file(self, file_id: str, headers: dict) -> str | None:
- """Download file content from OpenAI."""
- url = f"{self.OPENAI_API_URL}/v1/files/{file_id}/content"
- try:
- response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]
- if response.status_code == 200:
- return str(response.text)
- logger.error(f"Failed to download file {file_id}: {response.status_code}")
- return None
- except Exception as e:
- logger.error(f"Error downloading file {file_id}: {e}")
- return None
-
- async def _upload_openai_file(self, content: str, filename: str, headers: dict) -> str | None:
- """Upload a file to OpenAI for batch processing."""
- url = f"{self.OPENAI_API_URL}/v1/files"
-
- # Prepare multipart form data
- # We need to use httpx's files parameter for multipart upload
- files = {
- "file": (filename, content.encode("utf-8"), "application/jsonl"),
- }
- data = {
- "purpose": "batch",
- }
-
- # Remove content-type from headers (httpx will set it for multipart)
- upload_headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}
-
- try:
- response = await self.http_client.post( # type: ignore[union-attr]
- url, files=files, data=data, headers=upload_headers
- )
- if response.status_code == 200:
- result = response.json()
- file_id: str | None = result.get("id")
- return file_id
- logger.error(f"Failed to upload file: {response.status_code} - {response.text}")
- return None
- except Exception as e:
- logger.error(f"Error uploading file: {e}")
- return None
-
- async def _compress_batch_jsonl(self, content: str, request_id: str) -> tuple[list[str], dict]:
- """Compress messages in each line of a batch JSONL file.
-
- Returns:
- Tuple of (compressed_lines, stats_dict)
- """
- lines = content.strip().split("\n")
- compressed_lines = []
- total_original_tokens = 0
- total_compressed_tokens = 0
- total_requests = 0
- errors = 0
-
- tokenizer = get_tokenizer("gpt-4") # Use gpt-4 tokenizer for batch
-
- for i, line in enumerate(lines):
- if not line.strip():
- continue
-
- try:
- request_obj = json.loads(line)
- body = request_obj.get("body", {})
- messages = body.get("messages", [])
- model = body.get("model", "gpt-4")
-
- if not messages:
- # No messages to compress, pass through
- compressed_lines.append(line)
- total_requests += 1
- continue
-
- # Compress messages using the OpenAI pipeline
- if self.config.optimize:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- compressed_messages = result.messages
- # Use pipeline's token counts for consistency with pipeline logs
- original_tokens = result.tokens_before
- compressed_tokens = result.tokens_after
- except Exception as e:
- logger.warning(f"[{request_id}] Compression failed for line {i}: {e}")
- compressed_messages = messages
- original_tokens = tokenizer.count_messages(messages)
- compressed_tokens = original_tokens
- else:
- compressed_messages = messages
- original_tokens = tokenizer.count_messages(messages)
- compressed_tokens = original_tokens
-
- total_original_tokens += original_tokens
- total_compressed_tokens += compressed_tokens
- tokens_saved = original_tokens - compressed_tokens
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = body.get("tools")
- if self.config.ccr_inject_tool and tokens_saved > 0:
- injector = CCRToolInjector(
- provider="openai",
- inject_tool=True,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- compressed_messages, tools, was_injected = injector.process_request(
- compressed_messages, tools
- )
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for batch line {i}"
- )
-
- # Update body with compressed messages
- body["messages"] = compressed_messages
- if tools is not None:
- body["tools"] = tools
- request_obj["body"] = body
-
- compressed_lines.append(json.dumps(request_obj))
- total_requests += 1
-
- except json.JSONDecodeError as e:
- logger.warning(f"[{request_id}] Invalid JSON on line {i}: {e}")
- errors += 1
- # Keep original line on error
- compressed_lines.append(line)
- total_requests += 1
-
- total_tokens_saved = total_original_tokens - total_compressed_tokens
- savings_percent = (
- (total_tokens_saved / total_original_tokens * 100) if total_original_tokens > 0 else 0
- )
-
- stats = {
- "total_requests": total_requests,
- "total_original_tokens": total_original_tokens,
- "total_compressed_tokens": total_compressed_tokens,
- "total_tokens_saved": total_tokens_saved,
- "savings_percent": savings_percent,
- "errors": errors,
- }
-
- return compressed_lines, stats
-
- async def _batch_passthrough(self, request: Request, body: dict) -> Response:
- """Pass through batch request to OpenAI without compression."""
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
-
- url = f"{self.OPENAI_API_URL}/v1/batches"
- response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]
-
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- async def handle_batch_list(self, request: Request) -> Response:
- """Handle GET /v1/batches - List batches (passthrough)."""
- return await self.handle_passthrough(request, self.OPENAI_API_URL)
-
- async def handle_batch_get(self, request: Request, batch_id: str) -> Response:
- """Handle GET /v1/batches/{batch_id} - Get batch (passthrough)."""
- return await self.handle_passthrough(request, self.OPENAI_API_URL)
-
- async def handle_batch_cancel(self, request: Request, batch_id: str) -> Response:
- """Handle POST /v1/batches/{batch_id}/cancel - Cancel batch (passthrough)."""
- return await self.handle_passthrough(request, self.OPENAI_API_URL)
-
- def _has_non_text_parts(self, content: dict) -> bool:
- """Check if a Gemini content entry has non-text parts.
-
- Non-text parts include:
- - inlineData: Base64-encoded images/media
- - fileData: File references (URI + MIME type)
- - functionCall: Function calls from model
- - functionResponse: Responses to function calls
-
- Args:
- content: A single Gemini content entry with 'parts' list.
-
- Returns:
- True if any part contains non-text data.
- """
- parts = content.get("parts", [])
- for part in parts:
- if any(
- key in part
- for key in ("inlineData", "fileData", "functionCall", "functionResponse")
- ):
- return True
- return False
-
- def _gemini_contents_to_messages(
- self, contents: list[dict], system_instruction: dict | None = None
- ) -> tuple[list[dict], set[int]]:
- """Convert Gemini contents[] format to OpenAI messages[] format for optimization.
-
- Gemini format:
- contents: [{"role": "user", "parts": [{"text": "..."}]}]
- systemInstruction: {"parts": [{"text": "..."}]}
-
- OpenAI format:
- messages: [{"role": "user", "content": "..."}]
-
- Returns:
- Tuple of (messages, preserved_indices) where preserved_indices contains
- the indices of content entries that have non-text parts (images, function
- calls, etc.) and should not be compressed.
- """
- messages = []
- preserved_indices: set[int] = set()
-
- # Add system instruction as system message
- if system_instruction:
- parts = system_instruction.get("parts", [])
- text_parts = [p.get("text", "") for p in parts if "text" in p]
- if text_parts:
- messages.append({"role": "system", "content": "\n".join(text_parts)})
-
- # Convert contents to messages
- for idx, content in enumerate(contents):
- # Track content entries with non-text parts
- if self._has_non_text_parts(content):
- preserved_indices.add(idx)
-
- role = content.get("role", "user")
- # Map Gemini roles to OpenAI roles
- if role == "model":
- role = "assistant"
-
- parts = content.get("parts", [])
- text_parts = [p.get("text", "") for p in parts if "text" in p]
-
- if text_parts:
- messages.append({"role": role, "content": "\n".join(text_parts)})
-
- return messages, preserved_indices
-
- def _messages_to_gemini_contents(self, messages: list[dict]) -> tuple[list[dict], dict | None]:
- """Convert OpenAI messages[] format back to Gemini contents[] format.
-
- Returns:
- (contents, system_instruction) tuple
- """
- contents = []
- system_instruction = None
-
- for msg in messages:
- role = msg.get("role", "user")
- content = msg.get("content", "")
-
- if role == "system":
- # Extract as systemInstruction
- system_instruction = {"parts": [{"text": content}]}
- else:
- # Map OpenAI roles to Gemini roles
- gemini_role = "model" if role == "assistant" else "user"
- contents.append({"role": gemini_role, "parts": [{"text": content}]})
-
- return contents, system_instruction
-
- async def handle_openai_responses(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle OpenAI /v1/responses endpoint (new Responses API).
-
- The Responses API differs from /v1/chat/completions:
- - Input: `input` (string or array) instead of `messages`
- - System: `instructions` instead of system message
- - Output: `output[]` array instead of `choices[].message`
- - State: `previous_response_id` for multi-turn
- - Built-in tools: web_search, file_search, code_interpreter
- """
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "type": "invalid_request_error",
- "code": "request_too_large",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
-
- model = body.get("model", "unknown")
- stream = body.get("stream", False)
-
- # Convert Responses API input to messages format for optimization.
- # The Responses API uses a different item model (function_call,
- # function_call_output, reasoning as top-level items) — we convert to
- # Chat Completions messages for the pipeline, then convert back.
- from headroom.proxy.responses_converter import (
- messages_to_responses_items,
- responses_items_to_messages,
- )
-
- input_data = body.get("input", "")
- instructions = body.get("instructions")
- previous_response_id = body.get("previous_response_id")
-
- messages: list[dict[str, Any]] = []
- original_items: list[dict[str, Any]] | None = None
- preserved_indices: list[int] = []
-
- if instructions:
- messages.append({"role": "system", "content": instructions})
-
- if isinstance(input_data, str):
- messages.append({"role": "user", "content": input_data})
- elif isinstance(input_data, list):
- original_items = input_data
- converted, preserved_indices = responses_items_to_messages(input_data)
- messages.extend(converted)
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- tags = self._extract_tags(headers)
-
- # Rate limiting
- if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited()
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Token counting on converted messages
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Optimize: convert items → compress → convert back
- tokens_saved = 0
- transforms_applied: list[str] = []
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- _should_compress = (
- self.config.optimize
- and original_items is not None
- and not previous_response_id
- and not _bypass
- and len(messages) > 1
- )
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
-
- if _should_compress and _license_ok:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
- except Exception as e:
- logger.warning(f"[{request_id}] Responses API optimization failed: {e}")
-
- tokens_saved = max(0, original_tokens - optimized_tokens)
- optimization_latency = (time.time() - start_time) * 1000
-
- # Convert compressed messages back to Responses API items
- if optimized_messages is not messages and original_items is not None:
- opt_msgs = optimized_messages
- # Strip system message (instructions) — it's separate in Responses API
- if instructions and opt_msgs and opt_msgs[0].get("role") == "system":
- body["instructions"] = opt_msgs[0]["content"]
- opt_msgs = opt_msgs[1:]
-
- body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)
-
- # /v1/responses is OpenAI-specific (Codex) — always routes direct.
- # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.
- if self.anthropic_backend is not None:
- logger.debug(
- f"[{request_id}] /v1/responses always routes to OpenAI direct "
- f"(backend '{self.anthropic_backend.name}' not used for Responses API)"
- )
-
- url = f"{self.OPENAI_API_URL}/v1/responses"
-
- try:
- if stream:
- # Streaming for Responses API uses semantic events
- return await self._stream_response(
- url,
- headers,
- body,
- "openai",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- )
- else:
- response = await self._retry_request("POST", url, headers, body)
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = original_tokens # fallback
- output_tokens = 0
- try:
- resp_json = response.json()
- usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("input_tokens", original_tokens)
- output_tokens = usage.get("output_tokens", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)
-
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- )
-
- logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed()
- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "type": "server_error",
- "code": "proxy_error",
- }
- },
- )
-
- async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:
- """WebSocket proxy for /v1/responses (Codex gpt-5.4+).
-
- Newer Codex versions use WebSocket instead of HTTP POST for the
- Responses API. This handler:
- 1. Accepts the client WebSocket
- 2. Receives the first message (``response.create`` request)
- 3. Compresses the ``input`` array using the existing pipeline
- 4. Opens an upstream WebSocket to OpenAI
- 5. Sends the compressed request upstream
- 6. Relays all subsequent messages bidirectionally
- """
- try:
- import websockets
- except ImportError:
- await websocket.accept()
- await websocket.close(
- code=1011,
- reason="websockets package not installed. pip install websockets",
- )
- return
-
- request_id = await self._next_request_id()
-
- # Forward client headers to upstream, adding required OpenAI-Beta header
- ws_headers = dict(websocket.headers)
-
- # Extract subprotocol from client — this is an application-level negotiation
- # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
- # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.
- client_subprotocols: list[str] = []
- raw_protocol = ws_headers.get("sec-websocket-protocol", "")
- if raw_protocol:
- client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]
-
- # Accept client connection with the requested subprotocol
- if client_subprotocols:
- await websocket.accept(subprotocol=client_subprotocols[0])
- else:
- await websocket.accept()
-
- # Build upstream WebSocket URL (http→ws, https→wss)
- base = self.OPENAI_API_URL
- ws_base = base.replace("https://", "wss://").replace("http://", "ws://")
- upstream_url = f"{ws_base}/v1/responses"
-
- # Forward all client headers except hop-by-hop / per-connection headers.
- # These are WebSocket handshake mechanics that the `websockets` library
- # generates fresh for the upstream connection — forwarding them would conflict.
- # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.
- _skip_headers = frozenset(
- {
- "host", # must match upstream, not local proxy
- "connection", # hop-by-hop
- "upgrade", # hop-by-hop
- "sec-websocket-key", # per-connection cryptographic nonce
- "sec-websocket-version", # protocol version (websockets lib sets this)
- "sec-websocket-extensions", # per-connection negotiation
- "sec-websocket-accept", # server-side only
- "sec-websocket-protocol", # handled via subprotocols param below
- "content-length", # hop-by-hop
- "transfer-encoding", # hop-by-hop
- }
- )
- upstream_headers: dict[str, str] = {}
- for k, v in ws_headers.items():
- if k.lower() not in _skip_headers:
- upstream_headers[k] = v
-
- # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.
- # Safety net for clients that don't forward auth headers via WebSocket upgrade.
- _has_auth = "authorization" in {k.lower() for k in upstream_headers}
- if not _has_auth:
- api_key = os.environ.get("OPENAI_API_KEY")
- if api_key:
- upstream_headers["Authorization"] = f"Bearer {api_key}"
- logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")
- else:
- logger.warning(
- f"[{request_id}] WS: no Authorization header from client and "
- f"OPENAI_API_KEY not set — upstream will likely reject"
- )
-
- # Ensure the required beta header is present — OpenAI returns 500 without it
- if "openai-beta" not in {k.lower() for k in upstream_headers}:
- upstream_headers["OpenAI-Beta"] = "responses-api=v1"
-
- logger.debug(
- f"[{request_id}] WS upstream headers: "
- f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "
- f"subprotocols={client_subprotocols}"
- )
-
- try:
- # Receive the first message from client (the response.create request)
- first_msg_raw = await websocket.receive_text()
-
- # --- Optional: compress the input in the first message ---
- try:
- body = json.loads(first_msg_raw)
- input_data = body.get("input")
- tokens_saved = 0
-
- should_compress = (
- self.config.optimize
- and isinstance(input_data, list)
- and len(input_data) > 1
- and not body.get("previous_response_id")
- )
- if should_compress:
- try:
- from headroom.proxy.responses_converter import (
- messages_to_responses_items,
- responses_items_to_messages,
- )
-
- model = body.get("model", "gpt-4o")
- converted, preserved = responses_items_to_messages(input_data)
-
- messages: list[dict[str, Any]] = []
- instructions = body.get("instructions")
- if instructions:
- messages.append({"role": "system", "content": instructions})
- messages.extend(converted)
-
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- context_limit = self.openai_provider.get_context_limit(model)
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- opt = result.messages
- if instructions and opt and opt[0].get("role") == "system":
- body["instructions"] = opt[0]["content"]
- opt = opt[1:]
- body["input"] = messages_to_responses_items(opt, input_data, preserved)
- tokens_saved = max(0, original_tokens - result.tokens_after)
- first_msg_raw = json.dumps(body)
- logger.info(
- f"[{request_id}] WS /v1/responses compressed: "
- f"saved {tokens_saved} tokens"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] WS compression failed: {e}")
-
- except json.JSONDecodeError:
- # Not JSON — pass through as-is
- tokens_saved = 0
-
- # --- Connect to upstream OpenAI WebSocket ---
- logger.debug(f"[{request_id}] WS connecting to {upstream_url}")
-
- # Use ssl=True to let the websockets library handle SSL natively.
- # Manual ssl.create_default_context() + certifi doesn't load the
- # Windows system cert store, causing HTTP 500 on wss:// connections.
- use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
-
- async with websockets.connect(
- upstream_url,
- additional_headers=upstream_headers,
- subprotocols=(
- [websockets.Subprotocol(p) for p in client_subprotocols]
- if client_subprotocols and hasattr(websockets, "Subprotocol")
- else client_subprotocols or None
- ),
- ssl=use_ssl,
- ) as upstream:
- # Send (potentially compressed) first message
- await upstream.send(first_msg_raw)
-
- # Bidirectional relay
- async def _client_to_upstream() -> None:
- try:
- while True:
- msg = await websocket.receive_text()
- await upstream.send(msg)
- except Exception as relay_err:
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- logger.debug(
- f"[{request_id}] WS client→upstream relay ended: {relay_err}"
- )
- with contextlib.suppress(Exception):
- await upstream.close()
-
- async def _upstream_to_client() -> None:
- try:
- async for msg in upstream:
- if isinstance(msg, str):
- await websocket.send_text(msg)
- elif isinstance(msg, bytes):
- await websocket.send_bytes(msg)
- else:
- await websocket.send_text(str(msg))
- except Exception as relay_err:
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- logger.debug(
- f"[{request_id}] WS upstream→client relay ended: {relay_err}"
- )
- finally:
- with contextlib.suppress(Exception):
- await websocket.close()
-
- await asyncio.gather(
- _client_to_upstream(),
- _upstream_to_client(),
- return_exceptions=True,
- )
-
- # Record metrics
- if tokens_saved > 0:
- model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"
- await self.metrics.record_request(
- provider="openai",
- model=model_name,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=tokens_saved,
- latency_ms=0,
- )
-
- except Exception as e:
- if "WebSocketDisconnect" not in type(e).__name__:
- # Extract response body from websockets InvalidStatus for better debugging
- error_detail = str(e)
- if hasattr(e, "response"):
- try:
- resp = e.response
- body_bytes = getattr(resp, "body", None) or b""
- if body_bytes:
- error_detail += (
- f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"
- )
- except Exception:
- pass
- logger.error(f"[{request_id}] WS proxy error: {error_detail}")
- with contextlib.suppress(Exception):
- await websocket.close(code=1011, reason=str(e)[:120])
-
- async def handle_gemini_generate_content(
- self,
- request: Request,
- model: str,
- ) -> Response | StreamingResponse:
- """Handle Gemini native /v1beta/models/{model}:generateContent endpoint.
-
- Gemini's native API differs from OpenAI:
- - Input: `contents[]` with `parts[]` instead of `messages`
- - System: `systemInstruction` instead of system message
- - Auth: `x-goog-api-key` header instead of `Authorization: Bearer`
- - Output: `candidates[].content.parts[].text`
- """
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "code": 413,
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "code": 400,
- }
- },
- )
-
- contents = body.get("contents", [])
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- tags = self._extract_tags(headers)
-
- # Rate limiting (use Gemini API key)
- if self.rate_limiter:
- rate_key = headers.get("x-goog-api-key", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited()
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Convert Gemini format to messages for optimization
- system_instruction = body.get("systemInstruction")
- messages, preserved_indices = self._gemini_contents_to_messages(
- contents, system_instruction
- )
-
- # Store original content entries that have non-text parts before compression
- preserved_contents = {idx: contents[idx] for idx in preserved_indices}
-
- # Early exit if ALL content has non-text parts (nothing to compress)
- if len(preserved_indices) == len(contents):
- # All content has non-text parts, skip compression entirely
- # Just forward the request as-is
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"
- query_params = dict(request.query_params)
- is_streaming = query_params.get("alt") == "sse"
- if "key" in query_params:
- url += f"?key={query_params['key']}"
-
- if is_streaming:
- stream_url = (
- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
- )
- if "key" in query_params:
- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
- return await self._stream_response(
- stream_url,
- headers,
- body,
- "gemini",
- model,
- request_id,
- 0,
- 0,
- 0,
- [],
- tags,
- 0,
- )
- else:
- response = await self._retry_request("POST", url, headers, body)
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Token counting
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Optimization
- transforms_applied: list[str] = []
- waste_signals_dict: dict[str, int] | None = None
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- _compression_failed = False
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
- if self.config.optimize and messages and _license_ok:
- try:
- # Use OpenAI pipeline (similar message format)
- context_limit = self.openai_provider.get_context_limit(model)
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- # Use pipeline's token counts for consistency with pipeline logs
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
- if result.waste_signals:
- waste_signals_dict = result.waste_signals.to_dict()
- except Exception as e:
- _compression_failed = True
- logger.warning(f"[{request_id}] Gemini optimization failed: {e}")
-
- tokens_saved = max(0, original_tokens - optimized_tokens)
- optimization_latency = (time.time() - start_time) * 1000
-
- # Query Echo: disabled — hurts prefix caching in long conversations.
-
- # Convert back to Gemini format if optimized
- if optimized_messages != messages:
- optimized_contents, optimized_system = self._messages_to_gemini_contents(
- optimized_messages
- )
-
- # Restore preserved content entries that had non-text parts
- for orig_idx, original_content in preserved_contents.items():
- if orig_idx < len(optimized_contents):
- optimized_contents[orig_idx] = original_content
-
- body["contents"] = optimized_contents
- if optimized_system:
- body["systemInstruction"] = optimized_system
- elif "systemInstruction" in body:
- del body["systemInstruction"]
-
- # Build URL - model is extracted from path
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"
-
- # Check if streaming requested via query param
- query_params = dict(request.query_params)
- is_streaming = query_params.get("alt") == "sse"
-
- # Preserve API key in query params if present
- if "key" in query_params:
- url += f"?key={query_params['key']}"
-
- try:
- if is_streaming:
- # For streaming, use streamGenerateContent endpoint
- stream_url = (
- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
- )
- if "key" in query_params:
- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
-
- return await self._stream_response(
- stream_url,
- headers,
- body,
- "gemini",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- )
- else:
- response = await self._retry_request("POST", url, headers, body)
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = optimized_tokens # fallback
- output_tokens = 0
- cache_read_tokens = 0
- try:
- resp_json = response.json()
- usage = resp_json.get("usageMetadata", {})
- total_input_tokens = usage.get("promptTokenCount", optimized_tokens)
- output_tokens = usage.get("candidatesTokenCount", 0)
- # Gemini returns cachedContentTokenCount for context-cached tokens
- # These are charged at 10-25% of the input price depending on model
- cache_read_tokens = usage.get("cachedContentTokenCount", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from Gemini response: {e}"
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(
- model,
- tokens_saved,
- optimized_tokens,
- cache_read_tokens=cache_read_tokens,
- )
-
- await self.metrics.record_request(
- provider="gemini",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- waste_signals=waste_signals_dict,
- cache_read_tokens=cache_read_tokens,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] Gemini {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens)"
- )
- else:
- logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- # Inject Headroom compression metrics (for SaaS metering)
- response_headers["x-headroom-tokens-before"] = str(original_tokens)
- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
- response_headers["x-headroom-model"] = model
- if transforms_applied:
- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
- if cache_read_tokens > 0:
- response_headers["x-headroom-cached"] = "true"
- if _compression_failed:
- response_headers["x-headroom-compression-failed"] = "true"
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed()
- logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "code": 502,
- }
- },
- )
-
- async def handle_gemini_stream_generate_content(
- self,
- request: Request,
- model: str,
- ) -> StreamingResponse | JSONResponse:
- """Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "code": 400,
- }
- },
- )
-
- contents = body.get("contents", [])
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- tags = self._extract_tags(headers)
-
- # Token counting
- tokenizer = get_tokenizer(model)
- original_tokens = 0
- for content in contents:
- parts = content.get("parts", [])
- for part in parts:
- if "text" in part:
- original_tokens += tokenizer.count_text(part["text"])
-
- optimization_latency = (time.time() - start_time) * 1000
-
- # Build URL with SSE param
- query_params = dict(request.query_params)
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"
- if "key" in query_params:
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"
-
- return await self._stream_response(
- url,
- headers,
- body,
- "gemini",
- model,
- request_id,
- original_tokens,
- original_tokens,
- 0, # tokens_saved
- [], # transforms_applied
- tags,
- optimization_latency,
- )
-
- async def handle_gemini_count_tokens(
- self,
- request: Request,
- model: str,
- ) -> Response:
- """Handle Gemini /v1beta/models/{model}:countTokens endpoint with compression.
-
- This endpoint counts tokens AFTER applying compression, so users can see
- how many tokens they'll actually use after optimization.
-
- The request format is the same as generateContent:
- {"contents": [...], "systemInstruction": {...}}
- """
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "code": 400,
- }
- },
- )
-
- contents = body.get("contents", [])
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
-
- # Convert Gemini format to messages for optimization
- system_instruction = body.get("systemInstruction")
- messages, preserved_indices = self._gemini_contents_to_messages(
- contents, system_instruction
- )
-
- # Store original content entries that have non-text parts before compression
- preserved_contents = {idx: contents[idx] for idx in preserved_indices}
-
- # Early exit if ALL content has non-text parts (nothing to compress)
- if len(preserved_indices) == len(contents):
- # All content has non-text parts, skip compression entirely
- # Just forward the countTokens request as-is
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"
- query_params = dict(request.query_params)
- if "key" in query_params:
- url += f"?key={query_params['key']}"
-
- response = await self._retry_request("POST", url, headers, body)
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- # Token counting (original)
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Apply compression using the same pipeline as generateContent
- transforms_applied: list[str] = []
- optimized_messages = messages
-
- if self.config.optimize and messages:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- except Exception as e:
- logger.warning(f"[{request_id}] Gemini countTokens optimization failed: {e}")
-
- # Convert back to Gemini format for the API call
- if optimized_messages != messages:
- optimized_contents, optimized_system = self._messages_to_gemini_contents(
- optimized_messages
- )
-
- # Restore preserved content entries that had non-text parts
- for orig_idx, original_content in preserved_contents.items():
- if orig_idx < len(optimized_contents):
- optimized_contents[orig_idx] = original_content
-
- body["contents"] = optimized_contents
- if optimized_system:
- body["systemInstruction"] = optimized_system
- elif "systemInstruction" in body:
- del body["systemInstruction"]
-
- # Build URL
- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"
-
- # Preserve API key in query params if present
- query_params = dict(request.query_params)
- if "key" in query_params:
- url += f"?key={query_params['key']}"
-
- try:
- response = await self._retry_request("POST", url, headers, body)
- total_latency = (time.time() - start_time) * 1000
-
- # Parse response to get token count
- compressed_tokens = 0
- try:
- resp_json = response.json()
- compressed_tokens = resp_json.get("totalTokens", 0)
- except (json.JSONDecodeError, ValueError) as e:
- logger.debug(f"[{request_id}] Failed to parse Gemini token count response: {e}")
-
- # Track stats
- tokens_saved = (
- max(0, original_tokens - compressed_tokens) if compressed_tokens > 0 else 0
- )
-
- await self.metrics.record_request(
- provider="gemini",
- model=model,
- input_tokens=compressed_tokens,
- output_tokens=0,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] Gemini countTokens {model}: {original_tokens:,} → {compressed_tokens:,} "
- f"(saved {tokens_saved:,} tokens, transforms: {transforms_applied})"
- )
- else:
- logger.info(
- f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens"
- )
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed()
- logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "code": 502,
- }
- },
- )
-
-
-# =============================================================================
-# FastAPI App
-# =============================================================================
-
async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:
"""Background task that logs TOIN stats periodically.
diff --git a/pyproject.toml b/pyproject.toml
index 3f2e27e86..328e42780 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -272,6 +272,12 @@ module = [
disallow_untyped_defs = false
warn_return_any = false
+# Handler mixins use self.* from HeadroomProxy via duck typing — mypy can't resolve these
+[[tool.mypy.overrides]]
+module = ["headroom.proxy.handlers.*"]
+disallow_untyped_defs = false
+ignore_errors = true
+
# Ignore third-party stubs with syntax errors
[[tool.mypy.overrides]]
module = ["mlx.*"]