Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Refactor: split server.py into handler mixins (Steps 5-9)
chopratejase8ab444f09e9
diff --git a/headroom/proxy/handlers/__init__.py b/headroom/proxy/handlers/__init__.pynew file mode 100644index 000000000..d0d1c8ce7--- /dev/null+++ b/headroom/proxy/handlers/__init__.py@@ -0,0 +1,20 @@+"""Handler mixins for HeadroomProxy.++Each mixin class contains methods extracted from HeadroomProxy that handle+requests for a specific provider or concern. The mixins rely on HeadroomProxy's+__init__ for all self.* attributes (duck typing).+"""++from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin+from headroom.proxy.handlers.batch import BatchHandlerMixin+from headroom.proxy.handlers.gemini import GeminiHandlerMixin+from headroom.proxy.handlers.openai import OpenAIHandlerMixin+from headroom.proxy.handlers.streaming import StreamingMixin++__all__ = [+ "AnthropicHandlerMixin",+ "BatchHandlerMixin",+ "GeminiHandlerMixin",+ "OpenAIHandlerMixin",+ "StreamingMixin",+]diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pynew file mode 100644index 000000000..848de1710--- /dev/null+++ b/headroom/proxy/handlers/anthropic.py@@ -0,0 +1,1435 @@+"""Anthropic handler mixin for HeadroomProxy.++Contains all Anthropic Messages API handlers including batch operations.+"""++from __future__ import annotations++import asyncio+import json+import logging+import time+from datetime import datetime+from pathlib import Path+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from fastapi import Request+ from fastapi.responses import Response, StreamingResponse++import httpx++logger = logging.getLogger("headroom.proxy")+++class AnthropicHandlerMixin:+ """Mixin providing Anthropic API handler methods for HeadroomProxy."""++ async def handle_anthropic_messages(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle Anthropic /v1/messages endpoint."""+ from fastapi import HTTPException+ from fastapi.responses import JSONResponse, Response++ from headroom.cache.compression_store import get_compression_store+ from headroom.ccr import CCRToolInjector+ from headroom.proxy.cost import _summarize_transforms+ from headroom.proxy.helpers import (+ MAX_MESSAGE_ARRAY_LENGTH,+ MAX_REQUEST_BODY_SIZE,+ _get_image_compressor,+ _read_request_json,+ )+ from headroom.proxy.models import RequestLog+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "type": "error",+ "error": {+ "type": "request_too_large",+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ },+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": f"Invalid request body: {e!s}",+ },+ },+ )+ model = body.get("model", "unknown")+ messages = body.get("messages", [])++ # Validate message array size+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": f"Message array too large ({len(messages)} messages). "+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",+ },+ },+ )++ stream = body.get("stream", False)++ # Bypass: skip ALL compression, TOIN learning, and CCR injection+ # when the caller explicitly opts out via header.+ # Prevents Headroom from corrupting sub-agent API calls+ # (e.g., Claude Code sub-agents that inherit ANTHROPIC_BASE_URL).+ _bypass = (+ request.headers.get("x-headroom-bypass", "").lower() == "true"+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"+ )+ if _bypass:+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")++ # Image compression (before text optimization)+ if self.config.image_optimize and messages and not _bypass:+ compressor = _get_image_compressor()+ if compressor and compressor.has_images(messages):+ messages = compressor.compress(messages, provider="anthropic")+ if compressor.last_result:+ logger.info(+ f"Image compression: {compressor.last_result.technique.value} "+ f"({compressor.last_result.savings_percent:.0f}% saved, "+ f"{compressor.last_result.original_tokens} -> "+ f"{compressor.last_result.compressed_tokens} tokens)"+ )++ # Extract headers and tags+ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting+ if self.rate_limiter:+ api_key = headers.get("x-api-key", "")+ client_ip = request.client.host if request.client else "unknown"+ rate_key = f"{api_key[:16]}:{client_ip}" if api_key else client_ip+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ headers={"Retry-After": str(int(wait_seconds) + 1)},+ )++ # Budget check+ if self.cost_tracker:+ allowed, remaining = self.cost_tracker.check_budget()+ if not allowed:+ raise HTTPException(+ status_code=429,+ detail=f"Budget exceeded for {self.config.budget_period} period",+ )++ # Memory: Get user ID when memory is enabled (fallback to "default" for simple DevEx)+ memory_user_id: str | None = None+ if self.memory_handler:+ memory_user_id = headers.get("x-headroom-user-id", "default")++ # Check cache (non-streaming only)+ cache_hit = False+ if self.cache and not stream:+ cached = await self.cache.get(messages, model)+ if cached:+ cache_hit = True+ optimization_latency = (time.time() - start_time) * 1000++ await self.metrics.record_request(+ provider="anthropic",+ model=model,+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0, # Savings already counted when response was cached+ latency_ms=optimization_latency,+ cached=True,+ )++ # Remove compression headers from cached response+ response_headers = dict(cached.response_headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=cached.response_body,+ headers=response_headers,+ media_type="application/json",+ )++ # Count original tokens+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Hook: pre_compress — let hooks modify messages before compression++ if self.config.hooks:+ from headroom.hooks import CompressContext++ _hook_ctx = CompressContext(+ model=model,+ user_query=extract_user_query(messages),+ provider="anthropic",+ )+ try:+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)+ except Exception as e:+ logger.debug(f"[{request_id}] pre_compress hook error: {e}")++ # Apply optimization+ transforms_applied = []+ pipeline_timing: dict[str, float] = {}+ waste_signals_dict: dict[str, int] | None = None+ optimized_messages = messages+ optimized_tokens = original_tokens++ # Get prefix cache tracker for this session+ session_id = self.session_tracker_store.compute_session_id(request, model, messages)+ prefix_tracker = self.session_tracker_store.get_or_create(session_id, "anthropic")+ frozen_message_count = prefix_tracker.get_frozen_message_count()++ _compression_failed = False+ original_messages = messages # Preserve for 400-retry fallback+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True+ if self.config.optimize and messages and not _bypass and _license_ok:+ try:+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS++ context_limit = self.anthropic_provider.get_context_limit(model)+ biases = (+ self.config.hooks.compute_biases(messages, _hook_ctx)+ if self.config.hooks+ else None+ )++ if self.config.mode == "token_headroom":+ comp_cache = self._get_compression_cache(session_id)++ # Zone 1: Swap cached compressed versions into working copy+ working_messages = comp_cache.apply_cached(messages)++ # Re-freeze boundary: consecutive stable messages from start+ frozen_message_count = comp_cache.compute_frozen_count(messages)++ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.anthropic_pipeline.apply(+ messages=working_messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(working_messages),+ frozen_message_count=frozen_message_count,+ biases=biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ # Cache newly compressed messages (index-aligned diff)+ if result.messages != working_messages:+ comp_cache.update_from_result(messages, result.messages)++ # Always use pipeline result — Zone 1 swaps are already applied+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)+ # so tokens_saved captures both Zone 1 + Zone 2 savings.+ # original_tokens was set at line ~2183 from uncompressed messages.+ optimized_tokens = result.tokens_after+ else:+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.anthropic_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ frozen_message_count=frozen_message_count,+ biases=biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after++ if result.waste_signals:+ waste_signals_dict = result.waste_signals.to_dict()+ except Exception as e:+ logger.warning(f"Optimization failed: {e}")+ # Flag compression failure for observability+ _compression_failed = True++ tokens_saved = max(0, original_tokens - optimized_tokens)+ optimization_latency = (time.time() - start_time) * 1000++ # Hook: post_compress — let hooks observe compression results+ if self.config.hooks and tokens_saved > 0:+ from headroom.hooks import CompressEvent++ try:+ self.config.hooks.post_compress(+ CompressEvent(+ tokens_before=original_tokens,+ tokens_after=optimized_tokens,+ tokens_saved=tokens_saved,+ compression_ratio=tokens_saved / original_tokens+ if original_tokens > 0+ else 0,+ transforms_applied=transforms_applied,+ model=model,+ user_query=_hook_ctx.user_query if self.config.hooks else "",+ provider="anthropic",+ )+ )+ except Exception as e:+ logger.debug(f"[{request_id}] post_compress hook error: {e}")++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = body.get("tools")+ _original_tools = tools # Preserve for diagnostic / future retry+ if (+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions+ ) and not _bypass:+ # Create fresh injector to avoid state leakage between requests+ injector = CCRToolInjector(+ provider="anthropic",+ inject_tool=self.config.ccr_inject_tool,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, tools, was_injected = injector.process_request(+ optimized_messages, tools+ )++ if injector.has_compressed_content:+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"+ )+ else:+ logger.debug(+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"+ )++ # Track compression in context tracker for multi-turn awareness+ if self.ccr_context_tracker:+ self._turn_counter += 1+ for hash_key in injector.detected_hashes:+ # Get compression metadata from store+ store = get_compression_store()+ entry = store.get_metadata(hash_key)+ if entry:+ self.ccr_context_tracker.track_compression(+ hash_key=hash_key,+ turn_number=self._turn_counter,+ tool_name=entry.get("tool_name"),+ original_count=entry.get("original_item_count", 0),+ compressed_count=entry.get("compressed_item_count", 0),+ query_context=entry.get("query_context", ""),+ sample_content=entry.get("compressed_content", "")[:500],+ )++ # CCR Proactive Expansion: Check if current query needs expanded context+ if self.ccr_context_tracker and self.config.ccr_proactive_expansion:+ # Extract user query from messages+ user_query = ""+ for msg in reversed(messages):+ if msg.get("role") == "user":+ content = msg.get("content", "")+ if isinstance(content, str):+ user_query = content+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ user_query = block.get("text", "")+ break+ break++ if user_query:+ recommendations = self.ccr_context_tracker.analyze_query(+ user_query, self._turn_counter+ )+ if recommendations:+ expansions = self.ccr_context_tracker.execute_expansions(recommendations)+ if expansions:+ # Add expanded context to the system message or as additional context+ expansion_text = self.ccr_context_tracker.format_expansions_for_context(+ expansions+ )+ logger.info(+ f"[{request_id}] CCR: Proactively expanded {len(expansions)} context(s) "+ f"based on query relevance"+ )+ # Append to the last user message+ if optimized_messages and optimized_messages[-1].get("role") == "user":+ last_msg = optimized_messages[-1]+ content = last_msg.get("content", "")+ if isinstance(content, str):+ optimized_messages[-1] = {+ **last_msg,+ "content": content + "\n\n" + expansion_text,+ }++ # Traffic Learner: Extract patterns from inbound tool results+ if self.traffic_learner:+ try:+ # Wire backend on first use (lazy init after memory handler is ready)+ if (+ self.traffic_learner._backend is None+ and self.memory_handler+ and self.memory_handler.initialized+ and self.memory_handler.backend+ ):+ self.traffic_learner.set_backend(self.memory_handler.backend)++ # Extract tool results from messages and learn from them+ tool_results = self.traffic_learner.extract_tool_results_from_messages(+ optimized_messages+ )+ for tr in tool_results[-5:]: # Only recent results+ await self.traffic_learner.on_tool_result(+ tool_name=tr["tool_name"],+ tool_input=tr["input"],+ tool_output=tr["output"],+ is_error=tr["is_error"],+ )++ # Also extract preference signals from user messages+ await self.traffic_learner.on_messages(optimized_messages)+ except Exception as e:+ logger.debug(f"[{request_id}] Traffic learner: {e}")++ # Memory: Inject context and tools+ if self.memory_handler and memory_user_id:+ # Search and inject memory context+ if self.memory_handler.config.inject_context:+ try:+ memory_context = await self.memory_handler.search_and_format_context(+ memory_user_id, optimized_messages+ )+ if memory_context:+ optimized_messages = self._inject_system_context(+ optimized_messages, memory_context, body=body+ )+ logger.info(+ f"[{request_id}] Memory: Injected {len(memory_context)} chars of context"+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Memory: Context injection failed: {e}")++ # Inject memory tools+ if self.memory_handler.config.inject_tools:+ tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "anthropic")+ if mem_tools_injected:+ tool_names = [+ t.get("name") or t.get("type", "")+ for t in tools+ if t.get("name", "").startswith("memory")+ or t.get("type", "").startswith("memory")+ ]+ logger.info(f"[{request_id}] Memory: Injected tools: {tool_names}")++ # Add beta headers for native memory tool+ beta_headers = self.memory_handler.get_beta_headers()+ if beta_headers:+ for key, value in beta_headers.items():+ # Merge with existing beta header if present+ existing = headers.get(key, "")+ if existing and value not in existing:+ headers[key] = f"{existing},{value}"+ else:+ headers[key] = value+ logger.info(+ f"[{request_id}] Memory: Added beta header: {key}={headers[key]}"+ )++ # Query Echo: disabled — hurts prefix caching in long conversations.+ # The echo changes every turn, invalidating the cached prefix.+ # To re-enable, uncomment and set query_echo_enabled on ProxyConfig.++ # Update body+ body["messages"] = optimized_messages+ if tools is not None:+ body["tools"] = tools++ # Forward request - use Bedrock backend if configured, otherwise direct API+ if self.anthropic_backend is not None:+ # Route through Bedrock backend+ try:+ if stream:+ return await self._stream_response_bedrock(+ body,+ headers,+ "anthropic",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ pipeline_timing=pipeline_timing,+ )+ else:+ backend_response = await self.anthropic_backend.send_message(body, headers)++ if backend_response.error:+ return JSONResponse(+ status_code=backend_response.status_code,+ content=backend_response.body,+ )++ # Track metrics+ total_latency = (time.time() - start_time) * 1000+ usage = backend_response.body.get("usage", {})+ output_tokens = usage.get("output_tokens", 0)++ _backend_name = (+ self.anthropic_backend.name if self.anthropic_backend else "anthropic"+ )+ await self.metrics.record_request(+ provider=_backend_name,+ model=model,+ input_tokens=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cached=False,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)++ # Log request+ if self.logger:+ self.logger.log(+ RequestLog(+ request_id=request_id,+ timestamp=datetime.now().isoformat(),+ provider=_backend_name,+ model=model,+ input_tokens_original=original_tokens,+ input_tokens_optimized=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ savings_percent=(tokens_saved / original_tokens * 100)+ if original_tokens > 0+ else 0,+ optimization_latency_ms=optimization_latency,+ total_latency_ms=total_latency,+ tags=tags,+ cache_hit=False,+ transforms_applied=transforms_applied,+ request_messages=body.get("messages")+ if self.config.log_full_messages+ else None,+ )+ )++ return JSONResponse(+ status_code=backend_response.status_code,+ content=backend_response.body,+ )+ except Exception as e:+ logger.error(f"[{request_id}] Bedrock backend error: {e}")+ return JSONResponse(+ status_code=500,+ content={+ "type": "error",+ "error": {"type": "api_error", "message": str(e)},+ },+ )++ # Direct Anthropic API+ url = f"{self.ANTHROPIC_API_URL}/v1/messages"++ try:+ if stream:+ return await self._stream_response(+ url,+ headers,+ body,+ "anthropic",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ memory_user_id=memory_user_id,+ pipeline_timing=pipeline_timing,+ prefix_tracker=prefix_tracker,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)++ # Full diagnostic dump on upstream errors.+ # Writes pre/post compression messages, tools, and error+ # to ~/.headroom/logs/debug_400/ for offline analysis.+ if response.status_code >= 400:+ try:+ err_body = response.json()+ err_msg = err_body.get("error", {}).get("message", "")+ err_type = err_body.get("error", {}).get("type", "")+ except Exception:+ err_body = {"raw": response.text[:2000]}+ err_msg = str(response.text[:500])+ err_type = "parse_error"++ logger.warning(+ f"[{request_id}] UPSTREAM_ERROR "+ f"status={response.status_code} "+ f"error_type={err_type} "+ f"error_msg={err_msg!r} "+ f"model={model} "+ f"compressed={'yes' if transforms_applied else 'no'} "+ f"transforms={transforms_applied} "+ f"original_tokens={original_tokens} "+ f"optimized_tokens={optimized_tokens} "+ f"message_count={len(body.get('messages', []))} "+ f"stream={stream}"+ )++ # Dump full request details to debug file+ try:+ debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"+ debug_dir.mkdir(parents=True, exist_ok=True)+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")+ debug_file = debug_dir / f"{ts}_{request_id}.json"++ # Sanitize headers (redact API keys)+ safe_headers = {}+ for k, v in headers.items():+ if k.lower() in ("x-api-key", "authorization"):+ safe_headers[k] = v[:12] + "..." if v else ""+ else:+ safe_headers[k] = v++ debug_payload = {+ "request_id": request_id,+ "timestamp": datetime.now().isoformat(),+ "status_code": response.status_code,+ "error_response": err_body,+ "model": model,+ "stream": stream,+ "headers": safe_headers,+ "compression": {+ "was_compressed": bool(transforms_applied),+ "transforms": transforms_applied,+ "original_tokens": original_tokens,+ "optimized_tokens": optimized_tokens,+ "tokens_saved": tokens_saved,+ "compression_failed": _compression_failed,+ },+ "tools_sent": body.get("tools"),+ "tool_count": len(body.get("tools") or []),+ "original_tool_count": len(_original_tools or []),+ "messages_sent": body.get("messages"),+ "message_count": len(body.get("messages", [])),+ "original_messages": (+ original_messages+ if original_messages is not body.get("messages")+ else "__same_as_sent__"+ ),+ "original_message_count": len(original_messages),+ "system_prompt": body.get("system"),+ }++ with open(debug_file, "w") as f:+ json.dump(debug_payload, f, indent=2, default=str)++ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")+ except Exception as dump_err:+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")++ # Parse response for CCR handling+ resp_json = None+ try:+ resp_json = response.json()+ except (json.JSONDecodeError, ValueError) as e:+ logger.debug(+ f"[{request_id}] Failed to parse response JSON for CCR handling: {e}"+ )++ # CCR Response Handling: Handle headroom_retrieve tool calls automatically+ if (+ self.ccr_response_handler+ and resp_json+ and response.status_code == 200+ and self.ccr_response_handler.has_ccr_tool_calls(resp_json, "anthropic")+ ):+ logger.info(f"[{request_id}] CCR: Detected retrieval tool call, handling...")++ # Create API call function for continuation+ # Use a fresh client to avoid potential decompression state issues+ async def api_call_fn(+ msgs: list[dict], tls: list[dict] | None+ ) -> dict[str, Any]:+ continuation_body = {+ **body,+ "messages": msgs,+ }+ if tls is not None:+ continuation_body["tools"] = tls++ # Use clean headers for continuation+ continuation_headers = {+ k: v+ for k, v in headers.items()+ if k.lower()+ not in (+ "content-encoding",+ "transfer-encoding",+ "accept-encoding",+ "content-length",+ )+ }++ # Reuse main client for CCR continuations (connection pooling)+ logger.info(f"CCR: Making continuation request with {len(msgs)} messages")+ assert self.http_client is not None, "HTTP client not initialized"+ try:+ cont_response = await self.http_client.post(+ url,+ json=continuation_body,+ headers=continuation_headers,+ timeout=httpx.Timeout(120.0), # Override timeout for CCR+ )+ logger.info(+ f"CCR: Got response status={cont_response.status_code}, "+ f"content-encoding={cont_response.headers.get('content-encoding')}"+ )+ result: dict[str, Any] = cont_response.json()+ logger.info("CCR: Parsed JSON successfully")+ return result+ except Exception as e:+ resp_headers: str | dict[str, str] = "N/A"+ try:+ resp_headers = dict(cont_response.headers)+ except Exception:+ pass+ logger.error(+ f"CCR: API call failed: {e}, response headers: {resp_headers}"+ )+ raise++ # Handle CCR tool calls+ try:+ final_resp_json = await self.ccr_response_handler.handle_response(+ resp_json,+ optimized_messages,+ tools,+ api_call_fn,+ provider="anthropic",+ )+ # Update response content with final response+ resp_json = final_resp_json+ # Remove encoding headers since content is now uncompressed JSON+ ccr_response_headers = {+ k: v+ for k, v in response.headers.items()+ if k.lower() not in ("content-encoding", "content-length")+ }+ try:+ ccr_content = json.dumps(final_resp_json).encode()+ except (TypeError, ValueError) as json_err:+ logger.warning(+ f"[{request_id}] CCR: JSON serialization failed: {json_err}"+ )+ ccr_content = json.dumps(resp_json).encode()+ response = httpx.Response(+ status_code=200,+ content=ccr_content,+ headers=ccr_response_headers,+ )+ logger.info(f"[{request_id}] CCR: Retrieval handled successfully")+ except Exception as e:+ import traceback++ logger.warning(+ f"[{request_id}] CCR: Response handling failed: {e}\n"+ f"Traceback: {traceback.format_exc()}"+ )+ # Continue with original response++ # Memory: Handle memory tool calls in response+ if (+ self.memory_handler+ and memory_user_id+ and resp_json+ and response.status_code == 200+ and self.memory_handler.has_memory_tool_calls(resp_json, "anthropic")+ ):+ logger.info(f"[{request_id}] Memory: Detected memory tool call, handling...")++ try:+ # Execute memory tool calls+ tool_results = await self.memory_handler.handle_memory_tool_calls(+ resp_json, memory_user_id, "anthropic"+ )++ if tool_results:+ # Create continuation messages+ assistant_msg = {+ "role": "assistant",+ "content": resp_json.get("content", []),+ }+ user_msg = {+ "role": "user",+ "content": tool_results,+ }++ continuation_messages = optimized_messages + [assistant_msg, user_msg]++ # Make continuation API call+ continuation_body = {**body, "messages": continuation_messages}+ if tools:+ continuation_body["tools"] = tools++ cont_response = await self._retry_request(+ "POST", url, headers, continuation_body+ )++ # Update response with continuation+ resp_json = cont_response.json()+ response = cont_response+ logger.info(+ f"[{request_id}] Memory: Tool calls handled, continuation complete"+ )++ except Exception as e:+ logger.warning(f"[{request_id}] Memory: Tool call handling failed: {e}")+ # Continue with original response++ total_latency = (time.time() - start_time) * 1000++ # Parse response for output token count and cache metrics+ output_tokens = 0+ cr_tokens = 0+ cw_tokens = 0+ uncached_input_tokens = 0+ if resp_json:+ usage = resp_json.get("usage", {})+ output_tokens = usage.get("output_tokens", 0)+ cr_tokens = usage.get("cache_read_input_tokens", 0)+ cw_tokens = usage.get("cache_creation_input_tokens", 0)+ uncached_input_tokens = usage.get("input_tokens", 0)++ # Update prefix cache tracker for next turn+ prefix_tracker.update_from_response(+ cache_read_tokens=cr_tokens,+ cache_write_tokens=cw_tokens,+ messages=optimized_messages,+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(+ model,+ tokens_saved,+ optimized_tokens,+ cache_read_tokens=cr_tokens,+ cache_write_tokens=cw_tokens,+ uncached_tokens=uncached_input_tokens,+ )++ # Cache response+ if self.cache and response.status_code == 200:+ await self.cache.set(+ messages,+ model,+ response.content,+ dict(response.headers),+ tokens_saved=tokens_saved,+ )++ # Record metrics — use optimized_tokens (what we sent), not API's+ # input_tokens which is just the non-cached portion with prompt caching+ await self.metrics.record_request(+ provider="anthropic",+ model=model,+ input_tokens=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ waste_signals=waste_signals_dict,+ cache_read_tokens=cr_tokens,+ cache_write_tokens=cw_tokens,+ uncached_input_tokens=uncached_input_tokens,+ )++ # Log request+ if self.logger:+ self.logger.log(+ RequestLog(+ request_id=request_id,+ timestamp=datetime.now().isoformat(),+ provider="anthropic",+ model=model,+ input_tokens_original=original_tokens,+ input_tokens_optimized=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ savings_percent=(tokens_saved / original_tokens * 100)+ if original_tokens > 0+ else 0,+ optimization_latency_ms=optimization_latency,+ total_latency_ms=total_latency,+ tags=tags,+ cache_hit=cache_hit,+ transforms_applied=transforms_applied,+ waste_signals=waste_signals_dict,+ request_messages=messages if self.config.log_full_messages else None,+ )+ )++ # Structured perf log line for `headroom perf` analysis+ num_msgs = len(messages)+ resp_usage = resp_json.get("usage", {}) if resp_json else {}+ cr = resp_usage.get("cache_read_input_tokens", 0)+ cw = resp_usage.get("cache_creation_input_tokens", 0)+ chp = round(cr / (cr + cw) * 100) if (cr + cw) > 0 else 0+ timing_str = (+ " ".join(f"{k}={v:.0f}ms" for k, v in pipeline_timing.items())+ if pipeline_timing+ else ""+ )+ logger.info(+ f"[{request_id}] PERF "+ f"model={model} msgs={num_msgs} "+ f"tok_before={original_tokens} tok_after={optimized_tokens} "+ f"tok_saved={tokens_saved} "+ f"cache_read={cr} cache_write={cw} cache_hit_pct={chp} "+ f"opt_ms={optimization_latency:.0f} "+ f"transforms={_summarize_transforms(transforms_applied)}"+ f"{' timing=' + timing_str if timing_str else ''}"+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ # Inject Headroom compression metrics (for SaaS metering)+ response_headers["x-headroom-tokens-before"] = str(original_tokens)+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)+ response_headers["x-headroom-model"] = model+ if transforms_applied:+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)+ if cache_hit:+ response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ await self.metrics.record_failed()+ # Log full error details internally for debugging+ logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")++ # Try fallback if enabled+ if self.config.fallback_enabled and self.config.fallback_provider == "openai":+ logger.info(f"[{request_id}] Attempting fallback to OpenAI")+ # Convert to OpenAI format and retry+ # (simplified - would need message format conversion)++ # Return sanitized error message to client (don't expose internal details)+ return JSONResponse(+ status_code=502,+ content={+ "type": "error",+ "error": {+ "type": "api_error",+ "message": "An error occurred while processing your request. Please try again.",+ },+ },+ )++ async def handle_anthropic_batch_create(+ self,+ request: Request,+ ) -> Response:+ """Handle Anthropic POST /v1/messages/batches endpoint with compression.++ Anthropic batch format:+ {+ "requests": [+ {+ "custom_id": "req-1",+ "params": {+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 1024,+ "messages": [{"role": "user", "content": "Hello"}]+ }+ },+ ...+ ]+ }++ This method applies compression to each request's messages before forwarding.+ """+ from fastapi.responses import JSONResponse, Response++ from headroom.ccr import CCRToolInjector+ from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "type": "error",+ "error": {+ "type": "request_too_large",+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ },+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": f"Invalid request body: {e!s}",+ },+ },+ )++ requests_list = body.get("requests", [])+ if not requests_list:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": "Missing or empty 'requests' field in batch request",+ },+ },+ )++ # Extract headers+ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ # Track compression stats across all batch requests+ total_original_tokens = 0+ total_optimized_tokens = 0+ total_tokens_saved = 0+ compressed_requests = []+ pipeline_timing: dict[str, float] = {}++ # Apply compression to each request in the batch+ for batch_req in requests_list:+ custom_id = batch_req.get("custom_id", "")+ params = batch_req.get("params", {})+ messages = params.get("messages", [])+ model = params.get("model", "unknown")++ if not messages or not self.config.optimize:+ # No messages or optimization disabled - pass through unchanged+ compressed_requests.append(batch_req)+ continue++ # Apply optimization+ try:+ context_limit = self.anthropic_provider.get_context_limit(model)+ result = self.anthropic_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )++ optimized_messages = result.messages+ for k, v in result.timing.items():+ pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ total_original_tokens += original_tokens+ total_optimized_tokens += optimized_tokens+ tokens_saved = max(0, original_tokens - optimized_tokens)+ total_tokens_saved += tokens_saved++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = params.get("tools")+ if self.config.ccr_inject_tool and tokens_saved > 0:+ injector = CCRToolInjector(+ provider="anthropic",+ inject_tool=True,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, tools, was_injected = injector.process_request(+ optimized_messages, tools+ )+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for batch request '{custom_id}'"+ )++ # Create compressed batch request+ compressed_params = {**params, "messages": optimized_messages}+ if tools is not None:+ compressed_params["tools"] = tools+ compressed_requests.append(+ {+ "custom_id": custom_id,+ "params": compressed_params,+ }+ )++ if tokens_saved > 0:+ logger.debug(+ f"[{request_id}] Batch request '{custom_id}': "+ f"{original_tokens:,} -> {optimized_tokens:,} tokens "+ f"(saved {tokens_saved:,})"+ )++ except Exception as e:+ logger.warning(+ f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}"+ )+ # Pass through unchanged on failure+ compressed_requests.append(batch_req)+ total_optimized_tokens += original_tokens++ # Update body with compressed requests+ body["requests"] = compressed_requests++ optimization_latency = (time.time() - start_time) * 1000++ # Forward request to Anthropic+ url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches"++ try:+ response = await self._retry_request("POST", url, headers, body)++ # Record metrics+ await self.metrics.record_request(+ provider="anthropic",+ model="batch",+ input_tokens=total_optimized_tokens,+ output_tokens=0,+ tokens_saved=total_tokens_saved,+ latency_ms=optimization_latency,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ )++ # Log compression stats+ if total_tokens_saved > 0:+ savings_percent = (+ (total_tokens_saved / total_original_tokens * 100)+ if total_original_tokens > 0+ else 0+ )+ logger.info(+ f"[{request_id}] Batch ({len(compressed_requests)} requests): "+ f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "+ f"(saved {total_tokens_saved:,}, {savings_percent:.1f}%)"+ )++ # Store batch context for CCR result processing+ if response.status_code == 200 and self.config.ccr_inject_tool:+ try:+ response_data = response.json()+ batch_id = response_data.get("id")+ if batch_id:+ await self._store_anthropic_batch_context(+ batch_id,+ requests_list,+ headers.get("x-api-key"),+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Failed to store batch context: {e}")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "type": "error",+ "error": {+ "type": "api_error",+ "message": "An error occurred while processing your batch request. Please try again.",+ },+ },+ )++ async def handle_anthropic_batch_passthrough(+ self,+ request: Request,+ batch_id: str | None = None,+ ) -> Response:+ """Handle Anthropic batch passthrough endpoints.++ Used for:+ - GET /v1/messages/batches - List batches+ - GET /v1/messages/batches/{batch_id} - Get batch+ - GET /v1/messages/batches/{batch_id}/results - Get batch results+ - POST /v1/messages/batches/{batch_id}/cancel - Cancel batch+ """+ from fastapi.responses import Response++ start_time = time.time()+ path = request.url.path+ url = f"{self.ANTHROPIC_API_URL}{path}"++ # Preserve query string parameters (e.g., limit, after_id for list endpoint)+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ body = await request.body()++ response = await self.http_client.request( # type: ignore[union-attr]+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="anthropic",+ model="passthrough:batches",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def _store_anthropic_batch_context(+ self,+ batch_id: str,+ requests_list: list[dict[str, Any]],+ api_key: str | None,+ ) -> None:+ """Store batch context for CCR result processing.++ Args:+ batch_id: The batch ID from the API response.+ requests_list: The original batch requests.+ api_key: The API key for continuation calls.+ """+ from headroom.ccr import BatchContext, BatchRequestContext, get_batch_context_store++ store = get_batch_context_store()+ context = BatchContext(+ batch_id=batch_id,+ provider="anthropic",+ api_key=api_key,+ api_base_url=self.ANTHROPIC_API_URL,+ )++ for batch_req in requests_list:+ custom_id = batch_req.get("custom_id", "")+ params = batch_req.get("params", {})+ context.add_request(+ BatchRequestContext(+ custom_id=custom_id,+ messages=params.get("messages", []),+ tools=params.get("tools"),+ model=params.get("model", ""),+ extras={+ "max_tokens": params.get("max_tokens", 4096),+ "system": params.get("system"),+ },+ )+ )++ await store.store(context)+ logger.debug(f"Stored batch context for {batch_id} with {len(requests_list)} requests")++ async def handle_anthropic_batch_results(+ self,+ request: Request,+ batch_id: str,+ ) -> Response:+ """Handle Anthropic batch results with CCR post-processing.++ This endpoint:+ 1. Fetches raw results from Anthropic+ 2. Detects CCR tool calls in each result+ 3. Executes retrieval and makes continuation calls+ 4. Returns processed results with complete responses+ """+ from fastapi.responses import Response++ from headroom.ccr import BatchResultProcessor, get_batch_context_store++ start_time = time.time()++ # Forward request to get raw results+ url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches/{batch_id}/results"++ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]++ if response.status_code != 200:+ # Error - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Parse results - Anthropic batch results are JSONL format+ raw_content = response.content.decode("utf-8")+ results = []+ for line in raw_content.strip().split("\n"):+ if line.strip():+ try:+ results.append(json.loads(line))+ except json.JSONDecodeError:+ continue++ if not results:+ # No results to process+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Check if we have context and CCR processing is enabled+ store = get_batch_context_store()+ batch_context = await store.get(batch_id)++ if batch_context is None or not self.config.ccr_inject_tool:+ # No context or CCR disabled - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Process results with CCR handler+ processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]+ processed = await processor.process_results(batch_id, results, "anthropic")++ # Convert back to JSONL format+ processed_lines = []+ for p in processed:+ processed_lines.append(json.dumps(p.result))+ if p.was_processed:+ logger.info(+ f"CCR: Processed batch result {p.custom_id} "+ f"({p.continuation_rounds} continuation rounds)"+ )++ processed_content = "\n".join(processed_lines)++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="anthropic",+ model="batch:ccr-processed",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ return Response(+ content=processed_content.encode("utf-8"),+ status_code=200,+ media_type="application/jsonl",+ )diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.pynew file mode 100644index 000000000..27276c351--- /dev/null+++ b/headroom/proxy/handlers/batch.py@@ -0,0 +1,995 @@+"""Batch handler mixin for HeadroomProxy.++Contains all batch API handlers for Google and OpenAI batch operations.+"""++from __future__ import annotations++import json+import logging+import time+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from fastapi import Request+ from fastapi.responses import Response++logger = logging.getLogger("headroom.proxy")+++class BatchHandlerMixin:+ """Mixin providing batch API handler methods for HeadroomProxy."""++ async def handle_google_batch_create(+ self,+ request: Request,+ model: str,+ ) -> Response:+ """Handle Google POST /v1beta/models/{model}:batchGenerateContent endpoint.++ Google batch format:+ {+ "batch": {+ "display_name": "my-batch",+ "input_config": {+ "requests": {+ "requests": [+ {+ "request": {"contents": [{"parts": [{"text": "..."}]}]},+ "metadata": {"key": "request-1"}+ }+ ]+ }+ }+ }+ }++ This method applies compression to each request's contents before forwarding.+ """+ from fastapi.responses import JSONResponse, Response++ from headroom.ccr import CCRToolInjector+ from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "code": 413,+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "status": "INVALID_ARGUMENT",+ }+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "code": 400,+ "message": f"Invalid request body: {e!s}",+ "status": "INVALID_ARGUMENT",+ }+ },+ )++ # Extract batch config+ batch_config = body.get("batch", {})+ input_config = batch_config.get("input_config", {})+ requests_wrapper = input_config.get("requests", {})+ requests_list = requests_wrapper.get("requests", [])++ if not requests_list:+ # No inline requests - might be using file input, pass through+ logger.debug(f"[{request_id}] Google batch: No inline requests, passing through")+ return await self._google_batch_passthrough(request, model, body)++ # Extract headers+ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ # Track compression stats+ total_original_tokens = 0+ total_optimized_tokens = 0+ total_tokens_saved = 0+ compressed_requests = []+ pipeline_timing: dict[str, float] = {}++ # Apply compression to each request in the batch+ for idx, batch_req in enumerate(requests_list):+ req_content = batch_req.get("request", {})+ metadata = batch_req.get("metadata", {})+ contents = req_content.get("contents", [])++ if not contents or not self.config.optimize:+ # No contents or optimization disabled - pass through unchanged+ compressed_requests.append(batch_req)+ continue++ # Convert Google format to messages for compression+ system_instruction = req_content.get("systemInstruction")+ messages, preserved_indices = self._gemini_contents_to_messages(+ contents, system_instruction+ )++ # Store original content entries that have non-text parts before compression+ preserved_contents = {idx: contents[idx] for idx in preserved_indices}++ # Early exit if ALL content has non-text parts (nothing to compress)+ if len(preserved_indices) == len(contents):+ # All content has non-text parts, skip compression+ compressed_requests.append(batch_req)+ continue++ # Apply optimization+ try:+ # Default context limit for most models+ context_limit = 128000++ # Use OpenAI pipeline (similar message format after conversion)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )++ optimized_messages = result.messages+ for k, v in result.timing.items():+ pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ total_original_tokens += original_tokens+ total_optimized_tokens += optimized_tokens+ tokens_saved = max(0, original_tokens - optimized_tokens)+ total_tokens_saved += tokens_saved++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = req_content.get("tools")+ # Extract existing function declarations if present+ existing_funcs = None+ if tools:+ for tool in tools:+ if "functionDeclarations" in tool:+ existing_funcs = tool["functionDeclarations"]+ break++ if self.config.ccr_inject_tool and tokens_saved > 0:+ injector = CCRToolInjector(+ provider="google",+ inject_tool=True,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, injected_funcs, was_injected = injector.process_request(+ optimized_messages, existing_funcs+ )+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for Google batch request {idx}"+ )+ existing_funcs = injected_funcs++ # Convert back to Google contents format+ optimized_contents, optimized_sys_inst = self._messages_to_gemini_contents(+ optimized_messages+ )++ # Restore preserved content entries that had non-text parts+ for orig_idx, original_content in preserved_contents.items():+ if orig_idx < len(optimized_contents):+ optimized_contents[orig_idx] = original_content++ # Create compressed batch request+ compressed_req_content = {**req_content, "contents": optimized_contents}+ if optimized_sys_inst:+ compressed_req_content["systemInstruction"] = optimized_sys_inst+ if existing_funcs is not None:+ compressed_req_content["tools"] = [{"functionDeclarations": existing_funcs}]++ compressed_req = {+ "request": compressed_req_content,+ "metadata": metadata,+ }++ compressed_requests.append(compressed_req)++ if tokens_saved > 0:+ logger.debug(+ f"[{request_id}] Google batch request {idx}: "+ f"{original_tokens:,} -> {optimized_tokens:,} tokens "+ f"(saved {tokens_saved:,})"+ )++ except Exception as e:+ logger.warning(+ f"[{request_id}] Optimization failed for Google batch request {idx}: {e}"+ )+ # Pass through unchanged on failure+ compressed_requests.append(batch_req)+ total_optimized_tokens += original_tokens++ # Update body with compressed requests+ body["batch"]["input_config"]["requests"]["requests"] = compressed_requests++ optimization_latency = (time.time() - start_time) * 1000++ # Forward request to Google+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"++ # Add API key to URL if present in headers+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ url = f"{url}?key={api_key}"++ try:+ response = await self._retry_request("POST", url, headers, body)++ # Record metrics+ await self.metrics.record_request(+ provider="google",+ model=f"batch:{model}",+ input_tokens=total_optimized_tokens,+ output_tokens=0,+ tokens_saved=total_tokens_saved,+ latency_ms=optimization_latency,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ )++ # Log compression stats+ if total_tokens_saved > 0:+ savings_percent = (+ (total_tokens_saved / total_original_tokens * 100)+ if total_original_tokens > 0+ else 0+ )+ logger.info(+ f"[{request_id}] Google batch compression: "+ f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "+ f"({savings_percent:.1f}% saved across {len(requests_list)} requests)"+ )++ # Store batch context for CCR result processing+ if response.status_code == 200 and self.config.ccr_inject_tool:+ try:+ response_data = response.json()+ batch_name = response_data.get("name")+ if batch_name:+ await self._store_google_batch_context(+ batch_name,+ requests_list,+ model,+ api_key,+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Failed to store Google batch context: {e}")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ logger.error(f"[{request_id}] Google batch request failed: {e}")+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "code": 500,+ "message": f"Failed to forward batch request: {e!s}",+ "status": "INTERNAL",+ }+ },+ )++ async def _google_batch_passthrough(+ self,+ request: Request,+ model: str,+ body: dict | None = None,+ ) -> Response:+ """Pass through Google batch request without modification."""+ from fastapi.responses import Response++ start_time = time.time()++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"++ # Add API key to URL if present in headers+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ url = f"{url}?key={api_key}"++ if body is None:+ body_content = await request.body()+ else:+ body_content = json.dumps(body).encode()++ response = await self.http_client.post( # type: ignore[union-attr]+ url,+ headers=headers,+ content=body_content,+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="google",+ model=f"passthrough:batch:{model}",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def handle_google_batch_passthrough(+ self,+ request: Request,+ batch_name: str | None = None,+ ) -> Response:+ """Handle Google batch passthrough endpoints.++ Used for:+ - GET /v1beta/batches/{batch_name} - Get batch status+ - POST /v1beta/batches/{batch_name}:cancel - Cancel batch+ - DELETE /v1beta/batches/{batch_name} - Delete batch+ """+ from fastapi.responses import Response++ start_time = time.time()+ path = request.url.path+ url = f"{self.GEMINI_API_URL}{path}"++ # Preserve query string parameters+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ # Handle API key+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ if "?" in url:+ url = f"{url}&key={api_key}"+ else:+ url = f"{url}?key={api_key}"++ body = await request.body()++ response = await self.http_client.request( # type: ignore[union-attr]+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="google",+ model="passthrough:batches",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def _store_google_batch_context(+ self,+ batch_name: str,+ requests_list: list[dict[str, Any]],+ model: str,+ api_key: str | None,+ ) -> None:+ """Store Google batch context for CCR result processing.++ Args:+ batch_name: The batch name from the API response.+ requests_list: The original batch requests.+ model: The model used for the batch.+ api_key: The API key for continuation calls.+ """+ from headroom.ccr import BatchContext, BatchRequestContext, get_batch_context_store++ store = get_batch_context_store()+ context = BatchContext(+ batch_id=batch_name,+ provider="google",+ api_key=api_key,+ api_base_url=self.GEMINI_API_URL,+ )++ for batch_req in requests_list:+ metadata = batch_req.get("metadata", {})+ custom_id = metadata.get("key", "")+ req_content = batch_req.get("request", {})+ contents = req_content.get("contents", [])+ system_instruction = req_content.get("systemInstruction")++ # Convert contents to messages format for CCR handler+ messages, _ = self._gemini_contents_to_messages(contents, system_instruction)++ # Extract system instruction text if present+ sys_text = None+ if system_instruction:+ parts = system_instruction.get("parts", [])+ if parts and isinstance(parts[0], dict):+ sys_text = parts[0].get("text")++ context.add_request(+ BatchRequestContext(+ custom_id=custom_id,+ messages=messages,+ tools=req_content.get("tools"),+ model=model,+ system_instruction=sys_text,+ )+ )++ await store.store(context)+ logger.debug(+ f"Stored Google batch context for {batch_name} with {len(requests_list)} requests"+ )++ async def handle_google_batch_results(+ self,+ request: Request,+ batch_name: str,+ ) -> Response:+ """Handle Google batch results with CCR post-processing.++ Google batch results endpoint returns the batch operation status.+ When status is SUCCEEDED, results are embedded in the response.+ This handler processes CCR tool calls in those results.+ """+ from fastapi.responses import JSONResponse, Response++ from headroom.ccr import BatchResultProcessor, get_batch_context_store++ start_time = time.time()++ # Forward request to get batch status/results+ url = f"{self.GEMINI_API_URL}/v1beta/{batch_name}"++ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ # Handle API key+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ if "?" in url:+ url = f"{url}&key={api_key}"+ else:+ url = f"{url}?key={api_key}"++ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]++ if response.status_code != 200:+ # Error - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Parse response+ try:+ response_data = response.json()+ except json.JSONDecodeError:+ # Not JSON - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Check if batch has results (state must be SUCCEEDED)+ metadata = response_data.get("metadata", {})+ state = metadata.get("state")++ if state != "SUCCEEDED":+ # Batch not complete - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Extract results from response+ # Google embeds results in the batch response+ results = response_data.get("response", {}).get("responses", [])++ if not results:+ # No results to process+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Check if we have context and CCR processing is enabled+ store = get_batch_context_store()+ batch_context = await store.get(batch_name)++ if batch_context is None or not self.config.ccr_inject_tool:+ # No context or CCR disabled - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Process results with CCR handler+ processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]+ processed = await processor.process_results(batch_name, results, "google")++ # Update response with processed results+ processed_results = [p.result for p in processed]+ response_data["response"]["responses"] = processed_results++ for p in processed:+ if p.was_processed:+ logger.info(+ f"CCR: Processed Google batch result {p.custom_id} "+ f"({p.continuation_rounds} continuation rounds)"+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="google",+ model="batch:ccr-processed",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ return JSONResponse(content=response_data, status_code=200)++ async def handle_batch_create(self, request: Request) -> Response:+ """Handle POST /v1/batches - Create a batch with compression.++ Flow:+ 1. Parse request to get input_file_id+ 2. Download the JSONL file content from OpenAI+ 3. Parse each line and compress the messages+ 4. Create a new compressed JSONL file+ 5. Upload compressed file to OpenAI+ 6. Create batch with the new compressed file_id+ 7. Return batch object with compression stats in metadata+ """+ from fastapi.responses import JSONResponse, Response++ from headroom.proxy.helpers import _read_request_json++ start_time = time.time()+ request_id = await self._next_request_id()++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )++ input_file_id = body.get("input_file_id")+ endpoint = body.get("endpoint")+ completion_window = body.get("completion_window", "24h")+ metadata = body.get("metadata", {})++ if not input_file_id:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": "input_file_id is required",+ "type": "invalid_request_error",+ "code": "missing_parameter",+ }+ },+ )++ if not endpoint:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": "endpoint is required",+ "type": "invalid_request_error",+ "code": "missing_parameter",+ }+ },+ )++ # Only compress chat completions endpoint+ if endpoint != "/v1/chat/completions":+ # Pass through for other endpoints+ return await self._batch_passthrough(request, body)++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ try:+ # Step 1: Download the input file from OpenAI+ logger.info(f"[{request_id}] Batch: Downloading input file {input_file_id}")+ file_content = await self._download_openai_file(input_file_id, headers)++ if file_content is None:+ return JSONResponse(+ status_code=404,+ content={+ "error": {+ "message": f"Failed to download file {input_file_id}",+ "type": "invalid_request_error",+ "code": "file_not_found",+ }+ },+ )++ # Step 2: Parse and compress each line+ logger.info(f"[{request_id}] Batch: Compressing JSONL content")+ compressed_lines, stats = await self._compress_batch_jsonl(file_content, request_id)++ if stats["total_requests"] == 0:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": "No valid requests found in input file",+ "type": "invalid_request_error",+ "code": "empty_file",+ }+ },+ )++ # Step 3: Create compressed JSONL content+ compressed_content = "\n".join(compressed_lines)++ # Step 4: Upload compressed file to OpenAI+ logger.info(f"[{request_id}] Batch: Uploading compressed file")+ new_file_id = await self._upload_openai_file(+ compressed_content, f"compressed_{input_file_id}.jsonl", headers+ )++ if new_file_id is None:+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "message": "Failed to upload compressed file",+ "type": "server_error",+ "code": "upload_failed",+ }+ },+ )++ # Step 5: Create batch with compressed file+ logger.info(f"[{request_id}] Batch: Creating batch with compressed file {new_file_id}")++ # Add compression stats to metadata+ compression_metadata = {+ **metadata,+ "headroom_compressed": "true",+ "headroom_original_file_id": input_file_id,+ "headroom_total_requests": str(stats["total_requests"]),+ "headroom_tokens_saved": str(stats["total_tokens_saved"]),+ "headroom_original_tokens": str(stats["total_original_tokens"]),+ "headroom_compressed_tokens": str(stats["total_compressed_tokens"]),+ "headroom_savings_percent": f"{stats['savings_percent']:.1f}",+ }++ batch_body = {+ "input_file_id": new_file_id,+ "endpoint": endpoint,+ "completion_window": completion_window,+ "metadata": compression_metadata,+ }++ url = f"{self.OPENAI_API_URL}/v1/batches"+ response = await self.http_client.post(url, json=batch_body, headers=headers) # type: ignore[union-attr]++ total_latency = (time.time() - start_time) * 1000++ # Log compression stats+ logger.info(+ f"[{request_id}] Batch created: {stats['total_requests']} requests, "+ f"{stats['total_original_tokens']:,} -> {stats['total_compressed_tokens']:,} tokens "+ f"(saved {stats['total_tokens_saved']:,} tokens, {stats['savings_percent']:.1f}%) "+ f"in {total_latency:.0f}ms"+ )++ # Record metrics+ await self.metrics.record_request(+ provider="openai",+ model="batch",+ input_tokens=stats["total_compressed_tokens"],+ output_tokens=0,+ tokens_saved=stats["total_tokens_saved"],+ latency_ms=total_latency,+ )++ # Return response with compression info in headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ response_headers["x-headroom-tokens-saved"] = str(stats["total_tokens_saved"])+ response_headers["x-headroom-savings-percent"] = f"{stats['savings_percent']:.1f}"++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ logger.error(f"[{request_id}] Batch creation failed: {type(e).__name__}: {e}")+ await self.metrics.record_failed()+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "message": "An error occurred while processing the batch request",+ "type": "server_error",+ "code": "batch_processing_error",+ }+ },+ )++ async def _download_openai_file(self, file_id: str, headers: dict) -> str | None:+ """Download file content from OpenAI."""+ url = f"{self.OPENAI_API_URL}/v1/files/{file_id}/content"+ try:+ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]+ if response.status_code == 200:+ return str(response.text)+ logger.error(f"Failed to download file {file_id}: {response.status_code}")+ return None+ except Exception as e:+ logger.error(f"Error downloading file {file_id}: {e}")+ return None++ async def _upload_openai_file(self, content: str, filename: str, headers: dict) -> str | None:+ """Upload a file to OpenAI for batch processing."""+ url = f"{self.OPENAI_API_URL}/v1/files"++ # Prepare multipart form data+ # We need to use httpx's files parameter for multipart upload+ files = {+ "file": (filename, content.encode("utf-8"), "application/jsonl"),+ }+ data = {+ "purpose": "batch",+ }++ # Remove content-type from headers (httpx will set it for multipart)+ upload_headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}++ try:+ response = await self.http_client.post( # type: ignore[union-attr]+ url, files=files, data=data, headers=upload_headers+ )+ if response.status_code == 200:+ result = response.json()+ file_id: str | None = result.get("id")+ return file_id+ logger.error(f"Failed to upload file: {response.status_code} - {response.text}")+ return None+ except Exception as e:+ logger.error(f"Error uploading file: {e}")+ return None++ async def _compress_batch_jsonl(self, content: str, request_id: str) -> tuple[list[str], dict]:+ """Compress messages in each line of a batch JSONL file.++ Returns:+ Tuple of (compressed_lines, stats_dict)+ """+ from headroom.ccr import CCRToolInjector+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ lines = content.strip().split("\n")+ compressed_lines = []+ total_original_tokens = 0+ total_compressed_tokens = 0+ total_requests = 0+ errors = 0++ tokenizer = get_tokenizer("gpt-4") # Use gpt-4 tokenizer for batch++ for i, line in enumerate(lines):+ if not line.strip():+ continue++ try:+ request_obj = json.loads(line)+ body = request_obj.get("body", {})+ messages = body.get("messages", [])+ model = body.get("model", "gpt-4")++ if not messages:+ # No messages to compress, pass through+ compressed_lines.append(line)+ total_requests += 1+ continue++ # Compress messages using the OpenAI pipeline+ if self.config.optimize:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ compressed_messages = result.messages+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ compressed_tokens = result.tokens_after+ except Exception as e:+ logger.warning(f"[{request_id}] Compression failed for line {i}: {e}")+ compressed_messages = messages+ original_tokens = tokenizer.count_messages(messages)+ compressed_tokens = original_tokens+ else:+ compressed_messages = messages+ original_tokens = tokenizer.count_messages(messages)+ compressed_tokens = original_tokens++ total_original_tokens += original_tokens+ total_compressed_tokens += compressed_tokens+ tokens_saved = original_tokens - compressed_tokens++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = body.get("tools")+ if self.config.ccr_inject_tool and tokens_saved > 0:+ injector = CCRToolInjector(+ provider="openai",+ inject_tool=True,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ compressed_messages, tools, was_injected = injector.process_request(+ compressed_messages, tools+ )+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for batch line {i}"+ )++ # Update body with compressed messages+ body["messages"] = compressed_messages+ if tools is not None:+ body["tools"] = tools+ request_obj["body"] = body++ compressed_lines.append(json.dumps(request_obj))+ total_requests += 1++ except json.JSONDecodeError as e:+ logger.warning(f"[{request_id}] Invalid JSON on line {i}: {e}")+ errors += 1+ # Keep original line on error+ compressed_lines.append(line)+ total_requests += 1++ total_tokens_saved = total_original_tokens - total_compressed_tokens+ savings_percent = (+ (total_tokens_saved / total_original_tokens * 100) if total_original_tokens > 0 else 0+ )++ stats = {+ "total_requests": total_requests,+ "total_original_tokens": total_original_tokens,+ "total_compressed_tokens": total_compressed_tokens,+ "total_tokens_saved": total_tokens_saved,+ "savings_percent": savings_percent,+ "errors": errors,+ }++ return compressed_lines, stats++ async def _batch_passthrough(self, request: Request, body: dict) -> Response:+ """Pass through batch request to OpenAI without compression."""+ from fastapi.responses import Response++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ url = f"{self.OPENAI_API_URL}/v1/batches"+ response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]++ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def handle_batch_list(self, request: Request) -> Response:+ """Handle GET /v1/batches - List batches (passthrough)."""+ return await self.handle_passthrough(request, self.OPENAI_API_URL)++ async def handle_batch_get(self, request: Request, batch_id: str) -> Response:+ """Handle GET /v1/batches/{batch_id} - Get batch (passthrough)."""+ return await self.handle_passthrough(request, self.OPENAI_API_URL)++ async def handle_batch_cancel(self, request: Request, batch_id: str) -> Response:+ """Handle POST /v1/batches/{batch_id}/cancel - Cancel batch (passthrough)."""+ return await self.handle_passthrough(request, self.OPENAI_API_URL)diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pynew file mode 100644index 000000000..3c1f4bb14--- /dev/null+++ b/headroom/proxy/handlers/gemini.py@@ -0,0 +1,649 @@+"""Gemini handler mixin for HeadroomProxy.++Contains all Google Gemini API handlers including format conversion utilities.+"""++from __future__ import annotations++import json+import logging+import time+from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from fastapi import Request+ from fastapi.responses import JSONResponse, Response, StreamingResponse++logger = logging.getLogger("headroom.proxy")+++class GeminiHandlerMixin:+ """Mixin providing Gemini API handler methods for HeadroomProxy."""++ def _has_non_text_parts(self, content: dict) -> bool:+ """Check if a Gemini content entry has non-text parts.++ Non-text parts include:+ - inlineData: Base64-encoded images/media+ - fileData: File references (URI + MIME type)+ - functionCall: Function calls from model+ - functionResponse: Responses to function calls++ Args:+ content: A single Gemini content entry with 'parts' list.++ Returns:+ True if any part contains non-text data.+ """+ parts = content.get("parts", [])+ for part in parts:+ if any(+ key in part+ for key in ("inlineData", "fileData", "functionCall", "functionResponse")+ ):+ return True+ return False++ def _gemini_contents_to_messages(+ self, contents: list[dict], system_instruction: dict | None = None+ ) -> tuple[list[dict], set[int]]:+ """Convert Gemini contents[] format to OpenAI messages[] format for optimization.++ Gemini format:+ contents: [{"role": "user", "parts": [{"text": "..."}]}]+ systemInstruction: {"parts": [{"text": "..."}]}++ OpenAI format:+ messages: [{"role": "user", "content": "..."}]++ Returns:+ Tuple of (messages, preserved_indices) where preserved_indices contains+ the indices of content entries that have non-text parts (images, function+ calls, etc.) and should not be compressed.+ """+ messages = []+ preserved_indices: set[int] = set()++ # Add system instruction as system message+ if system_instruction:+ parts = system_instruction.get("parts", [])+ text_parts = [p.get("text", "") for p in parts if "text" in p]+ if text_parts:+ messages.append({"role": "system", "content": "\n".join(text_parts)})++ # Convert contents to messages+ for idx, content in enumerate(contents):+ # Track content entries with non-text parts+ if self._has_non_text_parts(content):+ preserved_indices.add(idx)++ role = content.get("role", "user")+ # Map Gemini roles to OpenAI roles+ if role == "model":+ role = "assistant"++ parts = content.get("parts", [])+ text_parts = [p.get("text", "") for p in parts if "text" in p]++ if text_parts:+ messages.append({"role": role, "content": "\n".join(text_parts)})++ return messages, preserved_indices++ def _messages_to_gemini_contents(self, messages: list[dict]) -> tuple[list[dict], dict | None]:+ """Convert OpenAI messages[] format back to Gemini contents[] format.++ Returns:+ (contents, system_instruction) tuple+ """+ contents = []+ system_instruction = None++ for msg in messages:+ role = msg.get("role", "user")+ content = msg.get("content", "")++ if role == "system":+ # Extract as systemInstruction+ system_instruction = {"parts": [{"text": content}]}+ else:+ # Map OpenAI roles to Gemini roles+ gemini_role = "model" if role == "assistant" else "user"+ contents.append({"role": gemini_role, "parts": [{"text": content}]})++ return contents, system_instruction++ async def handle_gemini_generate_content(+ self,+ request: Request,+ model: str,+ ) -> Response | StreamingResponse:+ """Handle Gemini native /v1beta/models/{model}:generateContent endpoint.++ Gemini's native API differs from OpenAI:+ - Input: `contents[]` with `parts[]` instead of `messages`+ - System: `systemInstruction` instead of system message+ - Auth: `x-goog-api-key` header instead of `Authorization: Bearer`+ - Output: `candidates[].content.parts[].text`+ """+ from fastapi import HTTPException+ from fastapi.responses import JSONResponse, Response++ from headroom.proxy.helpers import MAX_REQUEST_BODY_SIZE, _read_request_json+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "code": 413,+ }+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "code": 400,+ }+ },+ )++ contents = body.get("contents", [])++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting (use Gemini API key)+ if self.rate_limiter:+ rate_key = headers.get("x-goog-api-key", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Convert Gemini format to messages for optimization+ system_instruction = body.get("systemInstruction")+ messages, preserved_indices = self._gemini_contents_to_messages(+ contents, system_instruction+ )++ # Store original content entries that have non-text parts before compression+ preserved_contents = {idx: contents[idx] for idx in preserved_indices}++ # Early exit if ALL content has non-text parts (nothing to compress)+ if len(preserved_indices) == len(contents):+ # All content has non-text parts, skip compression entirely+ # Just forward the request as-is+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"+ query_params = dict(request.query_params)+ is_streaming = query_params.get("alt") == "sse"+ if "key" in query_params:+ url += f"?key={query_params['key']}"++ if is_streaming:+ stream_url = (+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ )+ if "key" in query_params:+ stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"+ return await self._stream_response(+ stream_url,+ headers,+ body,+ "gemini",+ model,+ request_id,+ 0,+ 0,+ 0,+ [],+ tags,+ 0,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Token counting+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Optimization+ transforms_applied: list[str] = []+ waste_signals_dict: dict[str, int] | None = None+ optimized_messages = messages+ optimized_tokens = original_tokens++ _compression_failed = False+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True+ if self.config.optimize and messages and _license_ok:+ try:+ # Use OpenAI pipeline (similar message format)+ context_limit = self.openai_provider.get_context_limit(model)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ # Use pipeline's token counts for consistency with pipeline logs+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ if result.waste_signals:+ waste_signals_dict = result.waste_signals.to_dict()+ except Exception as e:+ _compression_failed = True+ logger.warning(f"[{request_id}] Gemini optimization failed: {e}")++ tokens_saved = max(0, original_tokens - optimized_tokens)+ optimization_latency = (time.time() - start_time) * 1000++ # Query Echo: disabled — hurts prefix caching in long conversations.++ # Convert back to Gemini format if optimized+ if optimized_messages != messages:+ optimized_contents, optimized_system = self._messages_to_gemini_contents(+ optimized_messages+ )++ # Restore preserved content entries that had non-text parts+ for orig_idx, original_content in preserved_contents.items():+ if orig_idx < len(optimized_contents):+ optimized_contents[orig_idx] = original_content++ body["contents"] = optimized_contents+ if optimized_system:+ body["systemInstruction"] = optimized_system+ elif "systemInstruction" in body:+ del body["systemInstruction"]++ # Build URL - model is extracted from path+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"++ # Check if streaming requested via query param+ query_params = dict(request.query_params)+ is_streaming = query_params.get("alt") == "sse"++ # Preserve API key in query params if present+ if "key" in query_params:+ url += f"?key={query_params['key']}"++ try:+ if is_streaming:+ # For streaming, use streamGenerateContent endpoint+ stream_url = (+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ )+ if "key" in query_params:+ stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"++ return await self._stream_response(+ stream_url,+ headers,+ body,+ "gemini",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ total_input_tokens = optimized_tokens # fallback+ output_tokens = 0+ cache_read_tokens = 0+ try:+ resp_json = response.json()+ usage = resp_json.get("usageMetadata", {})+ total_input_tokens = usage.get("promptTokenCount", optimized_tokens)+ output_tokens = usage.get("candidatesTokenCount", 0)+ # Gemini returns cachedContentTokenCount for context-cached tokens+ # These are charged at 10-25% of the input price depending on model+ cache_read_tokens = usage.get("cachedContentTokenCount", 0)+ except (KeyError, TypeError, AttributeError) as e:+ logger.debug(+ f"[{request_id}] Failed to extract cached tokens from Gemini response: {e}"+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(+ model,+ tokens_saved,+ optimized_tokens,+ cache_read_tokens=cache_read_tokens,+ )++ await self.metrics.record_request(+ provider="gemini",+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ waste_signals=waste_signals_dict,+ cache_read_tokens=cache_read_tokens,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] Gemini {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens)"+ )+ else:+ logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ # Inject Headroom compression metrics (for SaaS metering)+ response_headers["x-headroom-tokens-before"] = str(original_tokens)+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)+ response_headers["x-headroom-model"] = model+ if transforms_applied:+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)+ if cache_read_tokens > 0:+ response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "code": 502,+ }+ },+ )++ async def handle_gemini_stream_generate_content(+ self,+ request: Request,+ model: str,+ ) -> StreamingResponse | JSONResponse:+ """Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""+ from fastapi.responses import JSONResponse++ from headroom.proxy.helpers import _read_request_json+ from headroom.tokenizers import get_tokenizer++ start_time = time.time()+ request_id = await self._next_request_id()++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "code": 400,+ }+ },+ )++ contents = body.get("contents", [])++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Token counting+ tokenizer = get_tokenizer(model)+ original_tokens = 0+ for content in contents:+ parts = content.get("parts", [])+ for part in parts:+ if "text" in part:+ original_tokens += tokenizer.count_text(part["text"])++ optimization_latency = (time.time() - start_time) * 1000++ # Build URL with SSE param+ query_params = dict(request.query_params)+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ if "key" in query_params:+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"++ return await self._stream_response(+ url,+ headers,+ body,+ "gemini",+ model,+ request_id,+ original_tokens,+ original_tokens,+ 0, # tokens_saved+ [], # transforms_applied+ tags,+ optimization_latency,+ )++ async def handle_gemini_count_tokens(+ self,+ request: Request,+ model: str,+ ) -> Response:+ """Handle Gemini /v1beta/models/{model}:countTokens endpoint with compression.++ This endpoint counts tokens AFTER applying compression, so users can see+ how many tokens they'll actually use after optimization.++ The request format is the same as generateContent:+ {"contents": [...], "systemInstruction": {...}}+ """+ from fastapi.responses import JSONResponse, Response++ from headroom.proxy.helpers import _read_request_json+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "code": 400,+ }+ },+ )++ contents = body.get("contents", [])++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ # Convert Gemini format to messages for optimization+ system_instruction = body.get("systemInstruction")+ messages, preserved_indices = self._gemini_contents_to_messages(+ contents, system_instruction+ )++ # Store original content entries that have non-text parts before compression+ preserved_contents = {idx: contents[idx] for idx in preserved_indices}++ # Early exit if ALL content has non-text parts (nothing to compress)+ if len(preserved_indices) == len(contents):+ # All content has non-text parts, skip compression entirely+ # Just forward the countTokens request as-is+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"+ query_params = dict(request.query_params)+ if "key" in query_params:+ url += f"?key={query_params['key']}"++ response = await self._retry_request("POST", url, headers, body)+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Token counting (original)+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Apply compression using the same pipeline as generateContent+ transforms_applied: list[str] = []+ optimized_messages = messages++ if self.config.optimize and messages:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ except Exception as e:+ logger.warning(f"[{request_id}] Gemini countTokens optimization failed: {e}")++ # Convert back to Gemini format for the API call+ if optimized_messages != messages:+ optimized_contents, optimized_system = self._messages_to_gemini_contents(+ optimized_messages+ )++ # Restore preserved content entries that had non-text parts+ for orig_idx, original_content in preserved_contents.items():+ if orig_idx < len(optimized_contents):+ optimized_contents[orig_idx] = original_content++ body["contents"] = optimized_contents+ if optimized_system:+ body["systemInstruction"] = optimized_system+ elif "systemInstruction" in body:+ del body["systemInstruction"]++ # Build URL+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"++ # Preserve API key in query params if present+ query_params = dict(request.query_params)+ if "key" in query_params:+ url += f"?key={query_params['key']}"++ try:+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ # Parse response to get token count+ compressed_tokens = 0+ try:+ resp_json = response.json()+ compressed_tokens = resp_json.get("totalTokens", 0)+ except (json.JSONDecodeError, ValueError) as e:+ logger.debug(f"[{request_id}] Failed to parse Gemini token count response: {e}")++ # Track stats+ tokens_saved = (+ max(0, original_tokens - compressed_tokens) if compressed_tokens > 0 else 0+ )++ await self.metrics.record_request(+ provider="gemini",+ model=model,+ input_tokens=compressed_tokens,+ output_tokens=0,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] Gemini countTokens {model}: {original_tokens:,} → {compressed_tokens:,} "+ f"(saved {tokens_saved:,} tokens, transforms: {transforms_applied})"+ )+ else:+ logger.info(+ f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens"+ )++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "code": 502,+ }+ },+ )diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pynew file mode 100644index 000000000..4bdcc1ec1--- /dev/null+++ b/headroom/proxy/handlers/openai.py@@ -0,0 +1,1320 @@+"""OpenAI handler mixin for HeadroomProxy.++Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.+"""++from __future__ import annotations++import asyncio+import contextlib+import json+import logging+import os+import time+from datetime import datetime+from pathlib import Path+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from fastapi import Request, WebSocket+ from fastapi.responses import JSONResponse, Response, StreamingResponse+++logger = logging.getLogger("headroom.proxy")+++class OpenAIHandlerMixin:+ """Mixin providing OpenAI API handler methods for HeadroomProxy."""++ async def handle_openai_chat(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle OpenAI /v1/chat/completions endpoint."""+ from fastapi import HTTPException+ from fastapi.responses import JSONResponse, Response++ from headroom.ccr import CCRToolInjector+ from headroom.proxy.helpers import (+ COMPRESSION_TIMEOUT_SECONDS,+ MAX_MESSAGE_ARRAY_LENGTH,+ MAX_REQUEST_BODY_SIZE,+ _get_image_compressor,+ _read_request_json,+ )+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "type": "invalid_request_error",+ "code": "request_too_large",+ }+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )+ model = body.get("model", "unknown")+ messages = body.get("messages", [])++ # Validate message array size+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Message array too large ({len(messages)} messages). "+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",+ "type": "invalid_request_error",+ "code": "invalid_request",+ }+ },+ )++ stream = body.get("stream", False)++ # Bypass: skip ALL compression for explicit opt-out+ _bypass = (+ request.headers.get("x-headroom-bypass", "").lower() == "true"+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"+ )+ if _bypass:+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")++ # Image compression (before text optimization)+ if self.config.image_optimize and messages and not _bypass:+ compressor = _get_image_compressor()+ if compressor and compressor.has_images(messages):+ messages = compressor.compress(messages, provider="openai")+ if compressor.last_result:+ logger.info(+ f"Image compression: {compressor.last_result.technique.value} "+ f"({compressor.last_result.savings_percent:.0f}% saved, "+ f"{compressor.last_result.original_tokens} -> "+ f"{compressor.last_result.compressed_tokens} tokens)"+ )++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting+ if self.rate_limiter:+ rate_key = headers.get("authorization", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Check cache+ if self.cache and not stream:+ cached = await self.cache.get(messages, model)+ if cached:+ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0, # Savings already counted when response was cached+ latency_ms=(time.time() - start_time) * 1000,+ cached=True,+ )++ # Remove compression headers from cached response+ response_headers = dict(cached.response_headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(content=cached.response_body, headers=response_headers)++ # Token counting+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Hook: pre_compress+ _hook_biases = None+ if self.config.hooks:+ from headroom.hooks import CompressContext++ _hook_ctx = CompressContext(model=model, provider="openai")+ try:+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)+ _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)+ except Exception as e:+ logger.debug(f"[{request_id}] Hook error: {e}")++ # Optimization+ transforms_applied = []+ pipeline_timing: dict[str, float] = {}+ waste_signals_dict: dict[str, int] | None = None+ optimized_messages = messages+ optimized_tokens = original_tokens++ # Get prefix cache tracker for this session+ openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)+ openai_prefix_tracker = self.session_tracker_store.get_or_create(+ openai_session_id, "openai"+ )+ openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()++ _compression_failed = False+ original_messages = messages # Preserve for 400-retry fallback+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True+ if self.config.optimize and messages and not _bypass and _license_ok:+ try:+ context_limit = self.openai_provider.get_context_limit(model)++ if self.config.mode == "token_headroom":+ comp_cache = self._get_compression_cache(openai_session_id)++ # Zone 1: Swap cached compressed versions+ working_messages = comp_cache.apply_cached(messages)++ # Re-freeze boundary+ openai_frozen_count = comp_cache.compute_frozen_count(messages)++ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=working_messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(working_messages),+ frozen_message_count=openai_frozen_count,+ biases=_hook_biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != working_messages:+ comp_cache.update_from_result(messages, result.messages)++ # Always use pipeline result in token_headroom mode+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)+ # so tokens_saved captures both Zone 1 + Zone 2 savings.+ optimized_tokens = result.tokens_after+ else:+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ frozen_message_count=openai_frozen_count,+ biases=_hook_biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after++ if result.waste_signals:+ waste_signals_dict = result.waste_signals.to_dict()+ except Exception as e:+ logger.warning(f"Optimization failed: {e}")+ # Flag compression failure for observability+ _compression_failed = True++ tokens_saved = max(0, original_tokens - optimized_tokens)+ optimization_latency = (time.time() - start_time) * 1000++ # Hook: post_compress+ if self.config.hooks and tokens_saved > 0:+ from headroom.hooks import CompressEvent++ try:+ self.config.hooks.post_compress(+ CompressEvent(+ tokens_before=original_tokens,+ tokens_after=optimized_tokens,+ tokens_saved=tokens_saved,+ compression_ratio=tokens_saved / original_tokens+ if original_tokens > 0+ else 0,+ transforms_applied=transforms_applied,+ model=model,+ provider="openai",+ )+ )+ except Exception as e:+ logger.debug(f"[{request_id}] post_compress hook error: {e}")++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = body.get("tools")+ _original_tools = tools # Preserve for diagnostic / future retry+ if (+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions+ ) and not _bypass:+ injector = CCRToolInjector(+ provider="openai",+ inject_tool=self.config.ccr_inject_tool,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, tools, was_injected = injector.process_request(+ optimized_messages, tools+ )++ if injector.has_compressed_content:+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"+ )+ else:+ logger.debug(+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"+ )++ # Query Echo: disabled — hurts prefix caching in long conversations.++ body["messages"] = optimized_messages+ if tools is not None:+ body["tools"] = tools++ # Route through LiteLLM/any-llm backend if configured+ if self.anthropic_backend is not None:+ try:+ if stream:+ # Streaming: use stream_openai_message() → SSE events+ return await self._stream_openai_via_backend(+ body,+ headers,+ model,+ request_id,+ start_time,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ pipeline_timing=pipeline_timing,+ )+ else:+ # Non-streaming: use send_openai_message() → JSON+ backend_response = await self.anthropic_backend.send_openai_message(+ body, headers+ )++ if backend_response.error:+ return JSONResponse(+ status_code=backend_response.status_code,+ content=backend_response.body,+ )++ # Track metrics+ total_latency = (time.time() - start_time) * 1000+ usage = backend_response.body.get("usage", {})+ output_tokens = usage.get("completion_tokens", 0)+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)++ await self.metrics.record_request(+ provider=self.anthropic_backend.name,+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cached=False,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"+ )++ return JSONResponse(+ status_code=backend_response.status_code,+ content=backend_response.body,+ )+ except Exception as e:+ logger.error(f"[{request_id}] Backend error: {e}")+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "message": str(e),+ "type": "api_error",+ "code": "backend_error",+ }+ },+ )++ # Direct OpenAI API (no backend configured)+ url = f"{self.OPENAI_API_URL}/v1/chat/completions"++ try:+ if stream:+ # Inject stream_options to get usage stats in streaming response+ # This allows accurate token counting instead of byte-based estimation+ if "stream_options" not in body:+ body["stream_options"] = {"include_usage": True}+ elif isinstance(body.get("stream_options"), dict):+ body["stream_options"]["include_usage"] = True++ return await self._stream_response(+ url,+ headers,+ body,+ "openai",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ pipeline_timing=pipeline_timing,+ prefix_tracker=openai_prefix_tracker,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)++ # Full diagnostic dump on upstream errors (OpenAI handler)+ if response.status_code >= 400:+ try:+ err_body = response.json()+ err_msg = err_body.get("error", {}).get("message", "")+ err_type = err_body.get("error", {}).get("type", "")+ except Exception:+ err_body = {"raw": response.text[:2000]}+ err_msg = str(response.text[:500])+ err_type = "parse_error"++ logger.warning(+ f"[{request_id}] UPSTREAM_ERROR "+ f"status={response.status_code} "+ f"error_type={err_type} "+ f"error_msg={err_msg!r} "+ f"model={model} "+ f"compressed={'yes' if transforms_applied else 'no'} "+ f"transforms={transforms_applied} "+ f"original_tokens={original_tokens} "+ f"optimized_tokens={optimized_tokens} "+ f"message_count={len(body.get('messages', []))} "+ f"stream={stream}"+ )++ try:+ debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"+ debug_dir.mkdir(parents=True, exist_ok=True)+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")+ debug_file = debug_dir / f"{ts}_{request_id}.json"++ safe_headers = {}+ for k, v in headers.items():+ if k.lower() in ("x-api-key", "authorization"):+ safe_headers[k] = v[:12] + "..." if v else ""+ else:+ safe_headers[k] = v++ debug_payload = {+ "request_id": request_id,+ "timestamp": datetime.now().isoformat(),+ "status_code": response.status_code,+ "error_response": err_body,+ "model": model,+ "stream": stream,+ "headers": safe_headers,+ "compression": {+ "was_compressed": bool(transforms_applied),+ "transforms": transforms_applied,+ "original_tokens": original_tokens,+ "optimized_tokens": optimized_tokens,+ "tokens_saved": tokens_saved,+ "compression_failed": _compression_failed,+ },+ "tools_sent": body.get("tools"),+ "tool_count": len(body.get("tools") or []),+ "original_tool_count": len(_original_tools or []),+ "messages_sent": body.get("messages"),+ "message_count": len(body.get("messages", [])),+ "original_messages": (+ original_messages+ if original_messages is not body.get("messages")+ else "__same_as_sent__"+ ),+ "original_message_count": len(original_messages),+ "system_prompt": body.get("system"),+ }++ with open(debug_file, "w") as f:+ json.dump(debug_payload, f, indent=2, default=str)++ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")+ except Exception as dump_err:+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")++ total_latency = (time.time() - start_time) * 1000++ total_input_tokens = optimized_tokens # fallback+ output_tokens = 0+ cache_read_tokens = 0+ try:+ resp_json = response.json()+ usage = resp_json.get("usage", {})+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)+ output_tokens = usage.get("completion_tokens", 0)+ # OpenAI returns cached_tokens in prompt_tokens_details+ # These are charged at 50% of the input price+ prompt_details = usage.get("prompt_tokens_details", {})+ cache_read_tokens = prompt_details.get("cached_tokens", 0)+ except (KeyError, TypeError, AttributeError) as e:+ logger.debug(+ f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"+ )++ # Update prefix cache tracker for next turn+ openai_prefix_tracker.update_from_response(+ cache_read_tokens=cache_read_tokens,+ cache_write_tokens=0, # OpenAI doesn't report write tokens+ messages=optimized_messages,+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(+ model,+ tokens_saved,+ optimized_tokens,+ cache_read_tokens=cache_read_tokens,+ )++ # Cache+ if self.cache and response.status_code == 200:+ await self.cache.set(+ messages, model, response.content, dict(response.headers), tokens_saved+ )++ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ waste_signals=waste_signals_dict,+ cache_read_tokens=cache_read_tokens,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens)"+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ # Inject Headroom compression metrics (for SaaS metering)+ response_headers["x-headroom-tokens-before"] = str(original_tokens)+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)+ response_headers["x-headroom-model"] = model+ if transforms_applied:+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)+ if cache_read_tokens > 0:+ response_headers["x-headroom-cached"] = "true"+ if _compression_failed:+ response_headers["x-headroom-compression-failed"] = "true"++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed()+ # Log full error details internally for debugging+ logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")+ # Return sanitized error message to client (don't expose internal details)+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "type": "server_error",+ "code": "proxy_error",+ }+ },+ )++ async def handle_openai_responses(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle OpenAI /v1/responses endpoint (new Responses API).++ The Responses API differs from /v1/chat/completions:+ - Input: `input` (string or array) instead of `messages`+ - System: `instructions` instead of system message+ - Output: `output[]` array instead of `choices[].message`+ - State: `previous_response_id` for multi-turn+ - Built-in tools: web_search, file_search, code_interpreter+ """+ from fastapi import HTTPException+ from fastapi.responses import JSONResponse, Response++ from headroom.proxy.helpers import (+ COMPRESSION_TIMEOUT_SECONDS,+ MAX_REQUEST_BODY_SIZE,+ _read_request_json,+ )+ from headroom.proxy.responses_converter import (+ messages_to_responses_items,+ responses_items_to_messages,+ )+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "type": "invalid_request_error",+ "code": "request_too_large",+ }+ },+ )++ # Parse request+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )++ model = body.get("model", "unknown")+ stream = body.get("stream", False)++ # Convert Responses API input to messages format for optimization.+ # The Responses API uses a different item model (function_call,+ # function_call_output, reasoning as top-level items) — we convert to+ # Chat Completions messages for the pipeline, then convert back.++ input_data = body.get("input", "")+ instructions = body.get("instructions")+ previous_response_id = body.get("previous_response_id")++ messages: list[dict[str, Any]] = []+ original_items: list[dict[str, Any]] | None = None+ preserved_indices: list[int] = []++ if instructions:+ messages.append({"role": "system", "content": instructions})++ if isinstance(input_data, str):+ messages.append({"role": "user", "content": input_data})+ elif isinstance(input_data, list):+ original_items = input_data+ converted, preserved_indices = responses_items_to_messages(input_data)+ messages.extend(converted)++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting+ if self.rate_limiter:+ rate_key = headers.get("authorization", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Token counting on converted messages+ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ # Optimize: convert items → compress → convert back+ tokens_saved = 0+ transforms_applied: list[str] = []+ optimized_messages = messages+ optimized_tokens = original_tokens++ _bypass = (+ request.headers.get("x-headroom-bypass", "").lower() == "true"+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"+ )+ _should_compress = (+ self.config.optimize+ and original_items is not None+ and not previous_response_id+ and not _bypass+ and len(messages) > 1+ )+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True++ if _should_compress and _license_ok:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )+ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ original_tokens = result.tokens_before+ optimized_tokens = result.tokens_after+ except Exception as e:+ logger.warning(f"[{request_id}] Responses API optimization failed: {e}")++ tokens_saved = max(0, original_tokens - optimized_tokens)+ optimization_latency = (time.time() - start_time) * 1000++ # Convert compressed messages back to Responses API items+ if optimized_messages is not messages and original_items is not None:+ opt_msgs = optimized_messages+ # Strip system message (instructions) — it's separate in Responses API+ if instructions and opt_msgs and opt_msgs[0].get("role") == "system":+ body["instructions"] = opt_msgs[0]["content"]+ opt_msgs = opt_msgs[1:]++ body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)++ # /v1/responses is OpenAI-specific (Codex) — always routes direct.+ # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.+ if self.anthropic_backend is not None:+ logger.debug(+ f"[{request_id}] /v1/responses always routes to OpenAI direct "+ f"(backend '{self.anthropic_backend.name}' not used for Responses API)"+ )++ url = f"{self.OPENAI_API_URL}/v1/responses"++ try:+ if stream:+ # Streaming for Responses API uses semantic events+ return await self._stream_response(+ url,+ headers,+ body,+ "openai",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ total_input_tokens = original_tokens # fallback+ output_tokens = 0+ try:+ resp_json = response.json()+ usage = resp_json.get("usage", {})+ total_input_tokens = usage.get("input_tokens", original_tokens)+ output_tokens = usage.get("output_tokens", 0)+ except (KeyError, TypeError, AttributeError) as e:+ logger.debug(+ f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)++ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=total_input_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ )++ logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "type": "server_error",+ "code": "proxy_error",+ }+ },+ )++ async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:+ """WebSocket proxy for /v1/responses (Codex gpt-5.4+).++ Newer Codex versions use WebSocket instead of HTTP POST for the+ Responses API. This handler:+ 1. Accepts the client WebSocket+ 2. Receives the first message (``response.create`` request)+ 3. Compresses the ``input`` array using the existing pipeline+ 4. Opens an upstream WebSocket to OpenAI+ 5. Sends the compressed request upstream+ 6. Relays all subsequent messages bidirectionally+ """+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS+ from headroom.tokenizers import get_tokenizer+ from headroom.utils import extract_user_query++ try:+ import websockets+ except ImportError:+ await websocket.accept()+ await websocket.close(+ code=1011,+ reason="websockets package not installed. pip install websockets",+ )+ return++ request_id = await self._next_request_id()++ # Forward client headers to upstream, adding required OpenAI-Beta header+ ws_headers = dict(websocket.headers)++ # Extract subprotocol from client — this is an application-level negotiation+ # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).+ # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.+ client_subprotocols: list[str] = []+ raw_protocol = ws_headers.get("sec-websocket-protocol", "")+ if raw_protocol:+ client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]++ # Accept client connection with the requested subprotocol+ if client_subprotocols:+ await websocket.accept(subprotocol=client_subprotocols[0])+ else:+ await websocket.accept()++ # Build upstream WebSocket URL (http→ws, https→wss)+ base = self.OPENAI_API_URL+ ws_base = base.replace("https://", "wss://").replace("http://", "ws://")+ upstream_url = f"{ws_base}/v1/responses"++ # Forward all client headers except hop-by-hop / per-connection headers.+ # These are WebSocket handshake mechanics that the `websockets` library+ # generates fresh for the upstream connection — forwarding them would conflict.+ # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.+ _skip_headers = frozenset(+ {+ "host", # must match upstream, not local proxy+ "connection", # hop-by-hop+ "upgrade", # hop-by-hop+ "sec-websocket-key", # per-connection cryptographic nonce+ "sec-websocket-version", # protocol version (websockets lib sets this)+ "sec-websocket-extensions", # per-connection negotiation+ "sec-websocket-accept", # server-side only+ "sec-websocket-protocol", # handled via subprotocols param below+ "content-length", # hop-by-hop+ "transfer-encoding", # hop-by-hop+ }+ )+ upstream_headers: dict[str, str] = {}+ for k, v in ws_headers.items():+ if k.lower() not in _skip_headers:+ upstream_headers[k] = v++ # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.+ # Safety net for clients that don't forward auth headers via WebSocket upgrade.+ _has_auth = "authorization" in {k.lower() for k in upstream_headers}+ if not _has_auth:+ api_key = os.environ.get("OPENAI_API_KEY")+ if api_key:+ upstream_headers["Authorization"] = f"Bearer {api_key}"+ logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")+ else:+ logger.warning(+ f"[{request_id}] WS: no Authorization header from client and "+ f"OPENAI_API_KEY not set — upstream will likely reject"+ )++ # Ensure the required beta header is present — OpenAI returns 500 without it+ if "openai-beta" not in {k.lower() for k in upstream_headers}:+ upstream_headers["OpenAI-Beta"] = "responses-api=v1"++ logger.debug(+ f"[{request_id}] WS upstream headers: "+ f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "+ f"subprotocols={client_subprotocols}"+ )++ try:+ # Receive the first message from client (the response.create request)+ first_msg_raw = await websocket.receive_text()++ # --- Optional: compress the input in the first message ---+ try:+ body = json.loads(first_msg_raw)+ input_data = body.get("input")+ tokens_saved = 0++ should_compress = (+ self.config.optimize+ and isinstance(input_data, list)+ and len(input_data) > 1+ and not body.get("previous_response_id")+ )+ if should_compress:+ try:+ from headroom.proxy.responses_converter import (+ messages_to_responses_items,+ responses_items_to_messages,+ )++ model = body.get("model", "gpt-4o")+ converted, preserved = responses_items_to_messages(input_data)++ messages: list[dict[str, Any]] = []+ instructions = body.get("instructions")+ if instructions:+ messages.append({"role": "system", "content": instructions})+ messages.extend(converted)++ tokenizer = get_tokenizer(model)+ original_tokens = tokenizer.count_messages(messages)++ context_limit = self.openai_provider.get_context_limit(model)+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(messages),+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )++ if result.messages != messages:+ opt = result.messages+ if instructions and opt and opt[0].get("role") == "system":+ body["instructions"] = opt[0]["content"]+ opt = opt[1:]+ body["input"] = messages_to_responses_items(opt, input_data, preserved)+ tokens_saved = max(0, original_tokens - result.tokens_after)+ first_msg_raw = json.dumps(body)+ logger.info(+ f"[{request_id}] WS /v1/responses compressed: "+ f"saved {tokens_saved} tokens"+ )+ except Exception as e:+ logger.warning(f"[{request_id}] WS compression failed: {e}")++ except json.JSONDecodeError:+ # Not JSON — pass through as-is+ tokens_saved = 0++ # --- Connect to upstream OpenAI WebSocket ---+ logger.debug(f"[{request_id}] WS connecting to {upstream_url}")++ # Use ssl=True to let the websockets library handle SSL natively.+ # Manual ssl.create_default_context() + certifi doesn't load the+ # Windows system cert store, causing HTTP 500 on wss:// connections.+ use_ssl: bool | None = True if upstream_url.startswith("wss://") else None++ async with websockets.connect(+ upstream_url,+ additional_headers=upstream_headers,+ subprotocols=(+ [websockets.Subprotocol(p) for p in client_subprotocols]+ if client_subprotocols and hasattr(websockets, "Subprotocol")+ else client_subprotocols or None+ ),+ ssl=use_ssl,+ ) as upstream:+ # Send (potentially compressed) first message+ await upstream.send(first_msg_raw)++ # Bidirectional relay+ async def _client_to_upstream() -> None:+ try:+ while True:+ msg = await websocket.receive_text()+ await upstream.send(msg)+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ logger.debug(+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"+ )+ with contextlib.suppress(Exception):+ await upstream.close()++ async def _upstream_to_client() -> None:+ try:+ async for msg in upstream:+ if isinstance(msg, str):+ await websocket.send_text(msg)+ elif isinstance(msg, bytes):+ await websocket.send_bytes(msg)+ else:+ await websocket.send_text(str(msg))+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ logger.debug(+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"+ )+ finally:+ with contextlib.suppress(Exception):+ await websocket.close()++ await asyncio.gather(+ _client_to_upstream(),+ _upstream_to_client(),+ return_exceptions=True,+ )++ # Record metrics+ if tokens_saved > 0:+ model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"+ await self.metrics.record_request(+ provider="openai",+ model=model_name,+ input_tokens=0,+ output_tokens=0,+ tokens_saved=tokens_saved,+ latency_ms=0,+ )++ except Exception as e:+ if "WebSocketDisconnect" not in type(e).__name__:+ # Extract response body from websockets InvalidStatus for better debugging+ error_detail = str(e)+ if hasattr(e, "response"):+ try:+ resp = e.response+ body_bytes = getattr(resp, "body", None) or b""+ if body_bytes:+ error_detail += (+ f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"+ )+ except Exception:+ pass+ logger.error(f"[{request_id}] WS proxy error: {error_detail}")+ with contextlib.suppress(Exception):+ await websocket.close(code=1011, reason=str(e)[:120])++ async def handle_compress(self, request: Request) -> JSONResponse:+ """Compress messages without calling an LLM.++ POST /v1/compress+ Body: {"messages": [...], "model": "...", "config": {}}+ Returns compressed messages + metrics.+ """+ from fastapi.responses import JSONResponse++ from headroom.proxy.helpers import _read_request_json++ # Check bypass header+ if request.headers.get("x-headroom-bypass", "").lower() == "true":+ try:+ body = await _read_request_json(request)+ except (json.JSONDecodeError, ValueError) as e:+ return JSONResponse(+ status_code=400,+ content={"error": f"Invalid request body: {e!s}"},+ )+ messages = body.get("messages", [])+ return JSONResponse(+ {+ "messages": messages,+ "tokens_before": 0,+ "tokens_after": 0,+ "tokens_saved": 0,+ "compression_ratio": 1.0,+ "transforms_applied": [],+ "ccr_hashes": [],+ }+ )++ try:+ body = await _read_request_json(request)+ except Exception:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "type": "invalid_request",+ "message": "Invalid JSON in request body.",+ }+ },+ )++ messages = body.get("messages")+ model = body.get("model")++ if messages is None:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "type": "invalid_request",+ "message": "Missing required field: messages",+ }+ },+ )++ if model is None:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "type": "invalid_request",+ "message": "Missing required field: model",+ }+ },+ )++ if not messages:+ return JSONResponse(+ {+ "messages": [],+ "tokens_before": 0,+ "tokens_after": 0,+ "tokens_saved": 0,+ "compression_ratio": 1.0,+ "transforms_applied": [],+ "ccr_hashes": [],+ }+ )++ try:+ # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)+ # Allow optional token_budget to override model's context limit+ # (used by OpenClaw compact() and other callers that need tighter budgets)+ token_budget = body.get("token_budget")+ context_limit = (+ token_budget+ if token_budget and isinstance(token_budget, int)+ else self.openai_provider.get_context_limit(model)+ )+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ )++ return JSONResponse(+ {+ "messages": result.messages,+ "tokens_before": result.tokens_before,+ "tokens_after": result.tokens_after,+ "tokens_saved": result.tokens_before - result.tokens_after,+ "compression_ratio": (+ result.tokens_after / result.tokens_before+ if result.tokens_before > 0+ else 1.0+ ),+ "transforms_applied": result.transforms_applied,+ "transforms_summary": result.transforms_summary,+ "ccr_hashes": result.markers_inserted,+ }+ )+ except Exception as e:+ logger.exception("Compression failed: %s", e)+ return JSONResponse(+ status_code=503,+ content={+ "error": {+ "type": "compression_error",+ "message": str(e),+ }+ },+ )++ async def handle_passthrough(+ self,+ request: Request,+ base_url: str,+ endpoint_name: str | None = None,+ provider: str | None = None,+ ) -> Response:+ """Pass through request unchanged.++ Args:+ request: The incoming request+ base_url: The upstream API base URL+ endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")+ provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")+ """+ from fastapi.responses import Response++ start_time = time.time()+ path = request.url.path+ url = f"{base_url}{path}"++ # Preserve query string parameters+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ body = await request.body()++ response = await self.http_client.request( # type: ignore[union-attr]+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ # Track stats for passthrough requests+ if endpoint_name and provider:+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider=provider,+ model=f"passthrough:{endpoint_name}",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ cached=False,+ )++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def handle_databricks_invocations(+ self,+ request: Request,+ model: str,+ ) -> Response | StreamingResponse:+ """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.++ This enables using the Databricks CLI directly with Headroom:+ databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'++ The request/response format is identical to OpenAI chat completions,+ so we inject the model from the path and delegate to handle_openai_chat.+ """+ from fastapi.responses import JSONResponse++ from headroom.proxy.helpers import _read_request_json++ request_id = await self._next_request_id()++ try:+ body = await _read_request_json(request)+ except Exception as e:+ logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")+ return JSONResponse(+ status_code=400,+ content={+ "error": {"message": f"Invalid JSON: {e}", "type": "invalid_request_error"}+ },+ )++ # Inject model from path into body (Databricks CLI passes model in URL, not body)+ body["model"] = model++ logger.info(f"[{request_id}] Databricks invocation: model={model}")++ # Create a new request with the modified body+ # We reuse the OpenAI chat handler since the format is identical+ from starlette.requests import Request as StarletteRequest++ # Build new scope with the body already parsed+ scope = dict(request.scope)++ # Create a simple receive function that returns our modified body+ body_bytes = json.dumps(body).encode()++ async def receive():+ return {"type": "http.request", "body": body_bytes}++ modified_request = StarletteRequest(scope, receive)++ # Delegate to the OpenAI chat handler (same format)+ return await self.handle_openai_chat(modified_request)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pynew file mode 100644index 000000000..ad9f7ed0a--- /dev/null+++ b/headroom/proxy/handlers/streaming.py@@ -0,0 +1,901 @@+"""Streaming handler mixin for HeadroomProxy.++Contains SSE parsing, streaming response generation, and related utilities.+"""++from __future__ import annotations++import contextlib+import json+import logging+import time+from datetime import datetime+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from fastapi.responses import StreamingResponse+++import httpx++logger = logging.getLogger("headroom.proxy")+++class StreamingMixin:+ """Mixin providing streaming response methods for HeadroomProxy."""++ def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:+ """Parse usage information from SSE chunk.++ For Anthropic: Looks for message_start (input tokens) and message_delta (output tokens)+ For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)+ For Gemini: Looks for usageMetadata in each chunk++ Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens+ Returns None if no usage found in this chunk.+ """+ try:+ text = chunk.decode("utf-8", errors="ignore")+ # SSE format: "data: {...}\n\n" or "event: ...\ndata: {...}\n\n"+ for line in text.split("\n"):+ if not line.startswith("data: "):+ continue+ data_str = line[6:].strip()+ if not data_str or data_str == "[DONE]":+ continue++ try:+ data = json.loads(data_str)+ except json.JSONDecodeError:+ continue++ usage = {}++ if provider == "anthropic":+ # Anthropic sends message_start with input tokens+ # and message_delta with output tokens+ event_type = data.get("type", "")++ if event_type == "message_start":+ msg = data.get("message", {})+ msg_usage = msg.get("usage", {})+ if msg_usage:+ usage["input_tokens"] = msg_usage.get("input_tokens", 0)+ usage["cache_read_input_tokens"] = msg_usage.get(+ "cache_read_input_tokens", 0+ )+ usage["cache_creation_input_tokens"] = msg_usage.get(+ "cache_creation_input_tokens", 0+ )++ elif event_type == "message_delta":+ delta_usage = data.get("usage", {})+ if delta_usage:+ usage["output_tokens"] = delta_usage.get("output_tokens", 0)++ elif provider == "openai":+ # OpenAI sends usage in final chunk (when stream_options.include_usage=true)+ chunk_usage = data.get("usage")+ if chunk_usage:+ usage["input_tokens"] = chunk_usage.get("prompt_tokens", 0)+ usage["output_tokens"] = chunk_usage.get("completion_tokens", 0)+ # OpenAI has cached tokens in prompt_tokens_details+ details = chunk_usage.get("prompt_tokens_details", {})+ usage["cache_read_input_tokens"] = details.get("cached_tokens", 0)++ elif provider == "gemini":+ # Gemini sends usageMetadata in each streaming chunk+ # Format: {"usageMetadata": {"promptTokenCount": N, "candidatesTokenCount": M}}+ usage_meta = data.get("usageMetadata")+ if usage_meta:+ usage["input_tokens"] = usage_meta.get("promptTokenCount", 0)+ usage["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)+ # Gemini also has cachedContentTokenCount for context caching+ usage["cache_read_input_tokens"] = usage_meta.get(+ "cachedContentTokenCount", 0+ )++ if usage:+ return usage++ except (UnicodeDecodeError, KeyError, TypeError) as e:+ # Don't fail streaming on parse errors+ logger.debug(f"SSE usage parsing error for {provider}: {e}")++ return None++ def _parse_sse_usage_from_buffer(+ self, stream_state: dict[str, Any], provider: str+ ) -> dict[str, int] | None:+ """Parse usage from buffered SSE data, handling split chunks.++ Processes complete SSE events (ending with double newline) from the buffer+ and removes them from the buffer. Incomplete events are kept in the buffer+ for the next chunk.+ """+ buffer = stream_state["sse_buffer"]+ usage_found: dict[str, int] = {}++ # Process complete SSE events (separated by double newlines)+ while "\n\n" in buffer:+ event_end = buffer.index("\n\n")+ event_text = buffer[: event_end + 2]+ buffer = buffer[event_end + 2 :]++ # Parse this complete event+ for line in event_text.split("\n"):+ if not line.startswith("data: "):+ continue+ data_str = line[6:].strip()+ if not data_str or data_str == "[DONE]":+ continue++ try:+ data = json.loads(data_str)+ except json.JSONDecodeError:+ continue++ if provider == "anthropic":+ event_type = data.get("type", "")+ if event_type == "message_start":+ msg = data.get("message", {})+ msg_usage = msg.get("usage", {})+ if msg_usage:+ usage_found["input_tokens"] = msg_usage.get("input_tokens", 0)+ usage_found["cache_read_input_tokens"] = msg_usage.get(+ "cache_read_input_tokens", 0+ )+ usage_found["cache_creation_input_tokens"] = msg_usage.get(+ "cache_creation_input_tokens", 0+ )+ # INFO logging for cache token tracking (temporary for debugging)+ logger.info(+ f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "+ f"cache_read={usage_found.get('cache_read_input_tokens')}, "+ f"cache_write={usage_found.get('cache_creation_input_tokens')}"+ )+ elif event_type == "message_delta":+ delta_usage = data.get("usage", {})+ if delta_usage:+ usage_found["output_tokens"] = delta_usage.get("output_tokens", 0)++ elif provider == "openai":+ chunk_usage = data.get("usage")+ if chunk_usage:+ usage_found["input_tokens"] = chunk_usage.get("prompt_tokens", 0)+ usage_found["output_tokens"] = chunk_usage.get("completion_tokens", 0)+ details = chunk_usage.get("prompt_tokens_details", {})+ usage_found["cache_read_input_tokens"] = details.get("cached_tokens", 0)++ elif provider == "gemini":+ usage_meta = data.get("usageMetadata")+ if usage_meta:+ usage_found["input_tokens"] = usage_meta.get("promptTokenCount", 0)+ usage_found["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)+ usage_found["cache_read_input_tokens"] = usage_meta.get(+ "cachedContentTokenCount", 0+ )++ # Update buffer with remaining incomplete data+ stream_state["sse_buffer"] = buffer++ return usage_found if usage_found else None++ def _parse_sse_to_response(self, sse_data: str, provider: str) -> dict[str, Any] | None:+ """Parse SSE data to reconstruct the API response JSON.++ Args:+ sse_data: Raw SSE data string.+ provider: Provider type for parsing.++ Returns:+ Reconstructed response dict or None if parsing fails.+ """+ if provider != "anthropic":+ return None # Only implemented for Anthropic++ response: dict[str, Any] = {"content": [], "usage": {}}+ current_block: dict[str, Any] | None = None++ for line in sse_data.split("\n"):+ if not line.startswith("data: "):+ continue+ data_str = line[6:].strip()+ if not data_str or data_str == "[DONE]":+ continue++ try:+ data = json.loads(data_str)+ except json.JSONDecodeError:+ continue++ event_type = data.get("type", "")++ if event_type == "message_start":+ msg = data.get("message", {})+ response["id"] = msg.get("id")+ response["model"] = msg.get("model")+ response["role"] = msg.get("role", "assistant")+ response["stop_reason"] = msg.get("stop_reason")+ if msg.get("usage"):+ response["usage"].update(msg["usage"])++ elif event_type == "content_block_start":+ block = data.get("content_block", {})+ current_block = {+ "type": block.get("type"),+ "index": data.get("index", len(response["content"])),+ }+ if block.get("type") == "text":+ current_block["text"] = block.get("text", "")+ elif block.get("type") == "tool_use":+ current_block["id"] = block.get("id")+ current_block["name"] = block.get("name")+ current_block["input"] = {}++ elif event_type == "content_block_delta":+ if current_block:+ delta = data.get("delta", {})+ if delta.get("type") == "text_delta":+ current_block["text"] = current_block.get("text", "") + delta.get(+ "text", ""+ )+ elif delta.get("type") == "input_json_delta":+ # Accumulate partial JSON for tool input+ partial = delta.get("partial_json", "")+ current_block["_partial_json"] = (+ current_block.get("_partial_json", "") + partial+ )++ elif event_type == "content_block_stop":+ if current_block:+ # Parse accumulated JSON for tool_use blocks+ if current_block.get("type") == "tool_use" and "_partial_json" in current_block:+ try:+ current_block["input"] = json.loads(current_block["_partial_json"])+ except json.JSONDecodeError:+ current_block["input"] = {}+ del current_block["_partial_json"]+ response["content"].append(current_block)+ current_block = None++ elif event_type == "message_delta":+ delta = data.get("delta", {})+ if delta.get("stop_reason"):+ response["stop_reason"] = delta["stop_reason"]+ if data.get("usage"):+ response["usage"].update(data["usage"])++ return response if response.get("content") else None++ def _response_to_sse(self, response: dict[str, Any], provider: str) -> list[bytes]:+ """Convert a response dict back to SSE format.++ Args:+ response: API response dict.+ provider: Provider type for formatting.++ Returns:+ List of SSE event bytes.+ """+ if provider != "anthropic":+ return []++ events: list[bytes] = []++ # message_start+ msg_start = {+ "type": "message_start",+ "message": {+ "id": response.get("id", "msg_generated"),+ "type": "message",+ "role": response.get("role", "assistant"),+ "model": response.get("model", "unknown"),+ "content": [],+ "stop_reason": None,+ "usage": response.get("usage", {}),+ },+ }+ events.append(f"event: message_start\ndata: {json.dumps(msg_start)}\n\n".encode())++ # Content blocks+ for idx, block in enumerate(response.get("content", [])):+ # content_block_start+ if block.get("type") == "text":+ block_start = {+ "type": "content_block_start",+ "index": idx,+ "content_block": {"type": "text", "text": ""},+ }+ elif block.get("type") == "tool_use":+ block_start = {+ "type": "content_block_start",+ "index": idx,+ "content_block": {+ "type": "tool_use",+ "id": block.get("id", f"toolu_{idx}"),+ "name": block.get("name", ""),+ "input": {},+ },+ }+ else:+ continue++ events.append(+ f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n".encode()+ )++ # content_block_delta(s)+ if block.get("type") == "text" and block.get("text"):+ delta = {+ "type": "content_block_delta",+ "index": idx,+ "delta": {"type": "text_delta", "text": block["text"]},+ }+ events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())+ elif block.get("type") == "tool_use" and block.get("input"):+ delta = {+ "type": "content_block_delta",+ "index": idx,+ "delta": {+ "type": "input_json_delta",+ "partial_json": json.dumps(block["input"]),+ },+ }+ events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())++ # content_block_stop+ block_stop = {"type": "content_block_stop", "index": idx}+ events.append(f"event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n".encode())++ # message_delta+ msg_delta = {+ "type": "message_delta",+ "delta": {"stop_reason": response.get("stop_reason", "end_turn")},+ "usage": {"output_tokens": response.get("usage", {}).get("output_tokens", 0)},+ }+ events.append(f"event: message_delta\ndata: {json.dumps(msg_delta)}\n\n".encode())++ # message_stop+ events.append(b'event: message_stop\ndata: {"type": "message_stop"}\n\n')++ return events++ def _record_ccr_feedback_from_response(+ self, response: dict, provider: str, request_id: str+ ) -> None:+ """Extract headroom_retrieve tool calls from a response and record feedback.++ This closes the TOIN feedback loop for streaming responses where+ the proxy can't intercept and handle retrieval calls inline.+ """+ from headroom.cache.compression_store import get_compression_store++ content = response.get("content", [])+ if not isinstance(content, list):+ return++ store = get_compression_store()++ for block in content:+ if not isinstance(block, dict):+ continue+ if block.get("type") != "tool_use":+ continue+ if block.get("name") != "headroom_retrieve":+ continue++ input_data = block.get("input", {})+ hash_key = input_data.get("hash")+ query = input_data.get("query")++ if not hash_key:+ continue++ logger.info(+ f"[{request_id}] CCR Feedback: Recording retrieval "+ f"hash={hash_key[:8]}... query={query!r}"+ )++ # Call store.retrieve()/search() for the side effect of triggering+ # the feedback chain: _log_retrieval -> process_pending_feedback+ # -> toin.record_retrieval(). We discard the returned content.+ try:+ if query:+ store.search(hash_key, query)+ else:+ store.retrieve(hash_key, query=None)+ except Exception as e:+ logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")++ async def _stream_response(+ self,+ url: str,+ headers: dict,+ body: dict,+ provider: str,+ model: str,+ request_id: str,+ original_tokens: int,+ optimized_tokens: int,+ tokens_saved: int,+ transforms_applied: list[str],+ tags: dict[str, str],+ optimization_latency: float,+ memory_user_id: str | None = None,+ pipeline_timing: dict[str, float] | None = None,+ prefix_tracker: Any | None = None,+ ) -> StreamingResponse:+ """Stream response with metrics tracking and memory tool handling.++ Parses SSE events to extract actual usage information from the API response+ for accurate token counting and cost calculation.++ When memory is enabled (memory_user_id provided), this method:+ 1. Buffers the response to detect memory tool calls+ 2. Executes memory tools if found+ 3. Makes continuation requests until no memory tools remain+ 4. Streams the final response to the client+ """+ from fastapi.responses import StreamingResponse++ from headroom.proxy.cost import _summarize_transforms+ from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE++ start_time = time.time()++ # Mutable state for the generator to update+ stream_state: dict[str, Any] = {+ "input_tokens": None,+ "output_tokens": None,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ "total_bytes": 0,+ "sse_buffer": "", # Buffer for incomplete SSE events+ "ttfb_ms": None, # Time to first byte from upstream+ }++ # Track if we need to handle memory tools+ memory_enabled = (+ memory_user_id is not None+ and self.memory_handler is not None+ and provider == "anthropic"+ )++ # Open connection before generator to capture upstream response headers+ # (needed to forward ratelimit headers to the client via StreamingResponse)+ assert self.http_client is not None, "http_client must be initialized before streaming"+ try:+ _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)+ upstream_response = await self.http_client.send(_upstream_req, stream=True)+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ error_msg = str(e)+ logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")++ async def _error_gen():+ error_event = {+ "type": "error",+ "error": {+ "type": "connection_error",+ "message": f"Failed to connect to upstream API: {error_msg}",+ },+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()++ return StreamingResponse(_error_gen(), media_type="text/event-stream")++ # Forward upstream ratelimit headers to the client+ forwarded_headers = {+ k: v for k, v in upstream_response.headers.items() if "ratelimit" in k.lower()+ }++ async def generate():+ nonlocal body, memory_enabled # May need to modify for continuation requests++ # For memory mode, we buffer the response to check for tool calls+ buffered_chunks: list[bytes] = []+ full_sse_data = ""++ try:+ async with contextlib.aclosing(upstream_response) as response:+ async for chunk in response.aiter_bytes():+ # Record TTFB on first chunk+ if stream_state["ttfb_ms"] is None:+ stream_state["ttfb_ms"] = (time.time() - start_time) * 1000++ stream_state["total_bytes"] += len(chunk)++ # Buffer SSE data to handle chunks split across calls+ chunk_str = chunk.decode("utf-8", errors="ignore")+ stream_state["sse_buffer"] += chunk_str++ # Safety: prevent unbounded buffer growth+ if len(stream_state["sse_buffer"]) > MAX_SSE_BUFFER_SIZE:+ logger.error(+ "SSE buffer exceeded maximum size (%d bytes), "+ "truncating to prevent memory exhaustion",+ MAX_SSE_BUFFER_SIZE,+ )+ stream_state["sse_buffer"] = stream_state["sse_buffer"][+ -MAX_SSE_BUFFER_SIZE // 2 :+ ]++ # Always stream immediately — buffering breaks+ # real-time clients (LangGraph, LangChain, etc.)+ yield chunk++ if memory_enabled:+ # Also buffer for post-stream memory processing+ buffered_chunks.append(chunk)+ full_sse_data += chunk_str+ if len(full_sse_data) > MAX_SSE_BUFFER_SIZE:+ logger.warning(+ "Memory-mode SSE buffer exceeded maximum size, "+ "disabling memory detection for this request"+ )+ memory_enabled = False++ # Parse complete SSE events from buffer+ usage = self._parse_sse_usage_from_buffer(stream_state, provider)+ if usage:+ if "input_tokens" in usage:+ stream_state["input_tokens"] = usage["input_tokens"]+ if "output_tokens" in usage:+ stream_state["output_tokens"] = usage["output_tokens"]+ if "cache_read_input_tokens" in usage:+ stream_state["cache_read_input_tokens"] = usage[+ "cache_read_input_tokens"+ ]+ if "cache_creation_input_tokens" in usage:+ stream_state["cache_creation_input_tokens"] = usage[+ "cache_creation_input_tokens"+ ]++ # Memory tool handling after stream completes+ # Chunks were already yielded in real-time above, so we only+ # do silent background processing here — no yielding.+ if memory_enabled and full_sse_data:+ # Check for Claude Code credential error+ if "only authorized for use with Claude Code" in full_sse_data:+ logger.warning(+ f"[{request_id}] Memory: Claude Code subscription credentials "+ "do not support custom tool injection. Set ANTHROPIC_API_KEY "+ "environment variable or use --no-memory-tools flag."+ )+ return++ # Parse SSE to get response JSON+ parsed_response = self._parse_sse_to_response(full_sse_data, provider)++ if parsed_response and self.memory_handler.has_memory_tool_calls(+ parsed_response, provider+ ):+ logger.info(+ f"[{request_id}] Memory: Detected tool calls in streaming response"+ )++ # Execute memory tool calls silently — response already+ # streamed so we cannot make a continuation request.+ tool_results = await self.memory_handler.handle_memory_tool_calls(+ parsed_response, memory_user_id, provider+ )+ if tool_results:+ logger.info(+ f"[{request_id}] Memory: Tool calls executed silently "+ "(streaming mode — no continuation)"+ )++ # CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.+ # In streaming mode, the client handles actual retrieval, but we+ # still need to record the event so TOIN learns which fields matter.+ if self.config.ccr_inject_tool and full_sse_data:+ ccr_parsed = (+ parsed_response+ if parsed_response+ else self._parse_sse_to_response(full_sse_data, provider)+ )+ if ccr_parsed:+ self._record_ccr_feedback_from_response(ccr_parsed, provider, request_id)++ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ logger.error(f"[{request_id}] Connection error to upstream API: {e}")+ error_event = {+ "type": "error",+ "error": {+ "type": "connection_error",+ "message": f"Failed to connect to upstream API: {e}",+ },+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()+ except httpx.HTTPStatusError as e:+ logger.error(f"[{request_id}] HTTP error from upstream API: {e}")+ # Forward the upstream error response+ yield e.response.content+ except Exception as e:+ logger.error(f"[{request_id}] Unexpected streaming error: {e}")+ error_event = {+ "type": "error",+ "error": {"type": "api_error", "message": str(e)},+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()+ finally:+ # Record metrics after stream completes+ total_latency = (time.time() - start_time) * 1000++ # Use actual output tokens from API if available, otherwise estimate+ output_tokens = stream_state["output_tokens"]+ if output_tokens is None:+ # Fallback: estimate from bytes (but this is inaccurate for SSE)+ # Use a more conservative estimate - SSE overhead is ~10-20x+ output_tokens = stream_state["total_bytes"] // 40+ logger.debug(+ f"[{request_id}] No usage in stream, estimated {output_tokens} output tokens"+ )++ # Use optimized_tokens for dashboard metrics (what we actually sent).+ # API's input_tokens is the non-cached portion only, which is+ # misleading for aggregation (often just 1 with prompt caching).+ cache_read_tokens = stream_state["cache_read_input_tokens"]+ cache_write_tokens = stream_state["cache_creation_input_tokens"]+ uncached_input_tokens = stream_state.get("input_tokens") or 0++ # Structured perf log line for `headroom perf` analysis+ num_msgs = len(body.get("messages", []))+ cache_hit_pct = (+ round(cache_read_tokens / (cache_read_tokens + cache_write_tokens) * 100)+ if (cache_read_tokens + cache_write_tokens) > 0+ else 0+ )+ logger.info(+ f"[{request_id}] PERF "+ f"model={model} msgs={num_msgs} "+ f"tok_before={original_tokens} tok_after={optimized_tokens} "+ f"tok_saved={tokens_saved} "+ f"cache_read={cache_read_tokens} cache_write={cache_write_tokens} "+ f"cache_hit_pct={cache_hit_pct} "+ f"opt_ms={optimization_latency:.0f} "+ f"transforms={_summarize_transforms(transforms_applied)}"+ )++ # Update prefix cache tracker for next turn (streaming path)+ if prefix_tracker is not None:+ prefix_tracker.update_from_response(+ cache_read_tokens=cache_read_tokens,+ cache_write_tokens=cache_write_tokens,+ messages=body.get("messages", []),+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(+ model,+ tokens_saved,+ optimized_tokens,+ cache_read_tokens=cache_read_tokens,+ cache_write_tokens=cache_write_tokens,+ uncached_tokens=uncached_input_tokens,+ )++ await self.metrics.record_request(+ provider=provider,+ model=model,+ input_tokens=optimized_tokens, # What we sent, not API's non-cached count+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ overhead_ms=optimization_latency,+ ttfb_ms=stream_state["ttfb_ms"] or 0,+ pipeline_timing=pipeline_timing,+ cache_read_tokens=cache_read_tokens,+ cache_write_tokens=cache_write_tokens,+ uncached_input_tokens=uncached_input_tokens,+ )++ return StreamingResponse(+ generate(),+ media_type="text/event-stream",+ headers=forwarded_headers,+ )++ async def _stream_response_bedrock(+ self,+ body: dict,+ headers: dict,+ provider: str,+ model: str,+ request_id: str,+ original_tokens: int,+ optimized_tokens: int,+ tokens_saved: int,+ transforms_applied: list[str],+ tags: dict[str, str],+ optimization_latency: float,+ pipeline_timing: dict[str, float] | None = None,+ ) -> StreamingResponse:+ """Stream response from Bedrock backend with metrics tracking.++ Translates Bedrock streaming events to Anthropic SSE format.+ """+ from fastapi.responses import StreamingResponse++ from headroom.proxy.cost import _summarize_transforms+ from headroom.proxy.models import RequestLog++ start_time = time.time()++ # Mutable state for the generator+ stream_state: dict[str, Any] = {+ "input_tokens": 0,+ "output_tokens": 0,+ "ttfb_ms": None,+ }++ async def generate():+ try:+ assert self.anthropic_backend is not None++ async for event in self.anthropic_backend.stream_message(body, headers):+ # Record TTFB on first event+ if stream_state["ttfb_ms"] is None:+ stream_state["ttfb_ms"] = (time.time() - start_time) * 1000++ # Format as SSE+ if event.raw_sse:+ yield event.raw_sse.encode()+ else:+ sse_line = f"event: {event.event_type}\ndata: {json.dumps(event.data)}\n\n"+ yield sse_line.encode()++ # Track usage from message_start event+ if event.event_type == "message_start":+ msg = event.data.get("message", {})+ usage = msg.get("usage", {})+ if "input_tokens" in usage:+ stream_state["input_tokens"] = usage["input_tokens"]++ # Track output tokens from message_delta+ if event.event_type == "message_delta":+ usage = event.data.get("usage", {})+ if "output_tokens" in usage:+ stream_state["output_tokens"] = usage["output_tokens"]++ # Handle errors+ if event.event_type == "error":+ logger.error(f"[{request_id}] Bedrock stream error: {event.data}")++ except Exception as e:+ logger.error(f"[{request_id}] Bedrock streaming error: {e}")+ error_event = {+ "type": "error",+ "error": {"type": "api_error", "message": str(e)},+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()++ finally:+ # Record metrics+ total_latency = (time.time() - start_time) * 1000+ output_tokens = stream_state["output_tokens"]++ _backend_name = (+ self.anthropic_backend.name if self.anthropic_backend else "anthropic"+ )+ await self.metrics.record_request(+ provider=_backend_name,+ model=model,+ input_tokens=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cached=False,+ overhead_ms=optimization_latency,+ ttfb_ms=stream_state["ttfb_ms"] or 0,+ pipeline_timing=pipeline_timing,+ )++ if self.cost_tracker:+ self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)++ # Log request+ if self.logger:+ self.logger.log(+ RequestLog(+ request_id=request_id,+ timestamp=datetime.now().isoformat(),+ provider=_backend_name,+ model=model,+ input_tokens_original=original_tokens,+ input_tokens_optimized=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ savings_percent=(tokens_saved / original_tokens * 100)+ if original_tokens > 0+ else 0,+ optimization_latency_ms=optimization_latency,+ total_latency_ms=total_latency,+ tags=tags,+ cache_hit=False,+ transforms_applied=transforms_applied,+ request_messages=body.get("messages")+ if self.config.log_full_messages+ else None,+ )+ )++ # Structured perf log line for `headroom perf` analysis+ num_msgs = len(body.get("messages", []))+ logger.info(+ f"[{request_id}] PERF "+ f"model={model} msgs={num_msgs} "+ f"tok_before={original_tokens} tok_after={optimized_tokens} "+ f"tok_saved={tokens_saved} "+ f"cache_read=0 cache_write=0 cache_hit_pct=0 "+ f"opt_ms={optimization_latency:.0f} "+ f"transforms={_summarize_transforms(transforms_applied)}"+ )++ return StreamingResponse(+ generate(),+ media_type="text/event-stream",+ )++ async def _stream_openai_via_backend(+ self,+ body: dict,+ headers: dict,+ model: str,+ request_id: str,+ start_time: float,+ original_tokens: int,+ optimized_tokens: int,+ tokens_saved: int,+ transforms_applied: list[str],+ tags: dict[str, str],+ optimization_latency: float,+ pipeline_timing: dict[str, float] | None = None,+ ) -> StreamingResponse:+ """Stream OpenAI chat completion response from backend.++ Routes stream:true requests through the backend's stream_openai_message(),+ yielding SSE events to the client.+ """+ from fastapi.responses import StreamingResponse++ assert self.anthropic_backend is not None++ async def generate():+ try:+ async for sse_chunk in self.anthropic_backend.stream_openai_message(body, headers):+ yield sse_chunk.encode() if isinstance(sse_chunk, str) else sse_chunk+ except Exception as e:+ logger.error(f"[{request_id}] Backend streaming error: {e}")+ error_data = {+ "error": {+ "message": str(e),+ "type": "api_error",+ "code": "backend_error",+ }+ }+ yield f"data: {json.dumps(error_data)}\n\n".encode()+ yield b"data: [DONE]\n\n"+ finally:+ total_latency = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider=self.anthropic_backend.name,+ model=model,+ input_tokens=optimized_tokens,+ output_tokens=0, # Unknown in streaming+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cached=False,+ overhead_ms=optimization_latency,+ pipeline_timing=pipeline_timing,+ )+ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name} [stream]"+ )++ return StreamingResponse(+ generate(),+ media_type="text/event-stream",+ )diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 6a561a67f..83d5a41c8 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -31,7 +31,6 @@ import osimport randomimport sysimport time-from datetime import datetimefrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, Literal@@ -39,7 +38,6 @@ if TYPE_CHECKING:from ..cache.compression_cache import CompressionCachefrom ..memory.tracker import MemoryTracker-import contextlibimport httpx@@ -47,7 +45,7 @@ try:import uvicornfrom fastapi import FastAPI, HTTPException, Request, Response, WebSocketfrom fastapi.middleware.cors import CORSMiddleware- from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse+ from fastapi.responses import HTMLResponse, PlainTextResponseFASTAPI_AVAILABLE = Trueexcept ImportError:@@ -64,15 +62,11 @@ from headroom.cache.compression_store import get_compression_storefrom headroom.ccr import (CCR_TOOL_NAME,# Batch processing- BatchContext,- BatchRequestContext,- BatchResultProcessor,CCRResponseHandler,CCRToolInjector,ContextTracker,ContextTrackerConfig,ResponseHandlerConfig,- get_batch_context_store,parse_tool_call,)from headroom.config import (@@ -118,7 +112,6 @@ from headroom.proxy.request_logger import RequestLogger # noqa: F401from headroom.proxy.semantic_cache import SemanticCache # noqa: F401from headroom.telemetry import get_telemetry_collectorfrom headroom.telemetry.toin import get_toin-from headroom.tokenizers import get_tokenizerfrom headroom.transforms import (CacheAligner,CodeAwareCompressor,@@ -132,7 +125,6 @@ from headroom.transforms import (TransformPipeline,is_tree_sitter_available,)-from headroom.utils import extract_user_querylogging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"@@ -152,12 +144,22 @@ MAX_RATE_LIMITER_BUCKETS = 1000# Compression pipeline timeout in seconds-# =============================================================================-# Caching-# =============================================================================+from headroom.proxy.handlers import ( # noqa: E402+ AnthropicHandlerMixin,+ BatchHandlerMixin,+ GeminiHandlerMixin,+ OpenAIHandlerMixin,+ StreamingMixin,+)-class HeadroomProxy:+class HeadroomProxy(+ StreamingMixin,+ AnthropicHandlerMixin,+ OpenAIHandlerMixin,+ GeminiHandlerMixin,+ BatchHandlerMixin,+):"""Production-ready Headroom optimization proxy."""ANTHROPIC_API_URL = "https://api.anthropic.com"@@ -745,5077 +747,6 @@ class HeadroomProxy:raise last_error # type: ignore[misc]- async def handle_compress(self, request: Request) -> JSONResponse:- """Compress messages without calling an LLM.-- POST /v1/compress- Body: {"messages": [...], "model": "...", "config": {}}- Returns compressed messages + metrics.- """- # Check bypass header- if request.headers.get("x-headroom-bypass", "").lower() == "true":- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={"error": f"Invalid request body: {e!s}"},- )- messages = body.get("messages", [])- return JSONResponse(- {- "messages": messages,- "tokens_before": 0,- "tokens_after": 0,- "tokens_saved": 0,- "compression_ratio": 1.0,- "transforms_applied": [],- "ccr_hashes": [],- }- )-- try:- body = await _read_request_json(request)- except Exception:- return JSONResponse(- status_code=400,- content={- "error": {- "type": "invalid_request",- "message": "Invalid JSON in request body.",- }- },- )-- messages = body.get("messages")- model = body.get("model")-- if messages is None:- return JSONResponse(- status_code=400,- content={- "error": {- "type": "invalid_request",- "message": "Missing required field: messages",- }- },- )-- if model is None:- return JSONResponse(- status_code=400,- content={- "error": {- "type": "invalid_request",- "message": "Missing required field: model",- }- },- )-- if not messages:- return JSONResponse(- {- "messages": [],- "tokens_before": 0,- "tokens_after": 0,- "tokens_saved": 0,- "compression_ratio": 1.0,- "transforms_applied": [],- "ccr_hashes": [],- }- )-- try:- # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)- # Allow optional token_budget to override model's context limit- # (used by OpenClaw compact() and other callers that need tighter budgets)- token_budget = body.get("token_budget")- context_limit = (- token_budget- if token_budget and isinstance(token_budget, int)- else self.openai_provider.get_context_limit(model)- )- result = self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- )-- return JSONResponse(- {- "messages": result.messages,- "tokens_before": result.tokens_before,- "tokens_after": result.tokens_after,- "tokens_saved": result.tokens_before - result.tokens_after,- "compression_ratio": (- result.tokens_after / result.tokens_before- if result.tokens_before > 0- else 1.0- ),- "transforms_applied": result.transforms_applied,- "transforms_summary": result.transforms_summary,- "ccr_hashes": result.markers_inserted,- }- )- except Exception as e:- logger.exception("Compression failed: %s", e)- return JSONResponse(- status_code=503,- content={- "error": {- "type": "compression_error",- "message": str(e),- }- },- )-- async def handle_anthropic_messages(- self,- request: Request,- ) -> Response | StreamingResponse:- """Handle Anthropic /v1/messages endpoint."""- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "type": "error",- "error": {- "type": "request_too_large",- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- },- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "type": "error",- "error": {- "type": "invalid_request_error",- "message": f"Invalid request body: {e!s}",- },- },- )- model = body.get("model", "unknown")- messages = body.get("messages", [])-- # Validate message array size- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:- return JSONResponse(- status_code=400,- content={- "type": "error",- "error": {- "type": "invalid_request_error",- "message": f"Message array too large ({len(messages)} messages). "- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",- },- },- )-- stream = body.get("stream", False)-- # Bypass: skip ALL compression, TOIN learning, and CCR injection- # when the caller explicitly opts out via header.- # Prevents Headroom from corrupting sub-agent API calls- # (e.g., Claude Code sub-agents that inherit ANTHROPIC_BASE_URL).- _bypass = (- request.headers.get("x-headroom-bypass", "").lower() == "true"- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"- )- if _bypass:- logger.info(f"[{request_id}] Bypass: skipping compression (header)")-- # Image compression (before text optimization)- if self.config.image_optimize and messages and not _bypass:- compressor = _get_image_compressor()- if compressor and compressor.has_images(messages):- messages = compressor.compress(messages, provider="anthropic")- if compressor.last_result:- logger.info(- f"Image compression: {compressor.last_result.technique.value} "- f"({compressor.last_result.savings_percent:.0f}% saved, "- f"{compressor.last_result.original_tokens} -> "- f"{compressor.last_result.compressed_tokens} tokens)"- )-- # Extract headers and tags- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- tags = self._extract_tags(headers)-- # Rate limiting- if self.rate_limiter:- api_key = headers.get("x-api-key", "")- client_ip = request.client.host if request.client else "unknown"- rate_key = f"{api_key[:16]}:{client_ip}" if api_key else client_ip- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited()- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",- headers={"Retry-After": str(int(wait_seconds) + 1)},- )-- # Budget check- if self.cost_tracker:- allowed, remaining = self.cost_tracker.check_budget()- if not allowed:- raise HTTPException(- status_code=429,- detail=f"Budget exceeded for {self.config.budget_period} period",- )-- # Memory: Get user ID when memory is enabled (fallback to "default" for simple DevEx)- memory_user_id: str | None = None- if self.memory_handler:- memory_user_id = headers.get("x-headroom-user-id", "default")-- # Check cache (non-streaming only)- cache_hit = False- if self.cache and not stream:- cached = await self.cache.get(messages, model)- if cached:- cache_hit = True- optimization_latency = (time.time() - start_time) * 1000-- await self.metrics.record_request(- provider="anthropic",- model=model,- input_tokens=0,- output_tokens=0,- tokens_saved=0, # Savings already counted when response was cached- latency_ms=optimization_latency,- cached=True,- )-- # Remove compression headers from cached response- response_headers = dict(cached.response_headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=cached.response_body,- headers=response_headers,- media_type="application/json",- )-- # Count original tokens- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Hook: pre_compress — let hooks modify messages before compression- from headroom.transforms.query_echo import extract_user_query-- if self.config.hooks:- from headroom.hooks import CompressContext-- _hook_ctx = CompressContext(- model=model,- user_query=extract_user_query(messages),- provider="anthropic",- )- try:- messages = self.config.hooks.pre_compress(messages, _hook_ctx)- except Exception as e:- logger.debug(f"[{request_id}] pre_compress hook error: {e}")-- # Apply optimization- transforms_applied = []- pipeline_timing: dict[str, float] = {}- waste_signals_dict: dict[str, int] | None = None- optimized_messages = messages- optimized_tokens = original_tokens-- # Get prefix cache tracker for this session- session_id = self.session_tracker_store.compute_session_id(request, model, messages)- prefix_tracker = self.session_tracker_store.get_or_create(session_id, "anthropic")- frozen_message_count = prefix_tracker.get_frozen_message_count()-- _compression_failed = False- original_messages = messages # Preserve for 400-retry fallback- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True- if self.config.optimize and messages and not _bypass and _license_ok:- try:- context_limit = self.anthropic_provider.get_context_limit(model)- biases = (- self.config.hooks.compute_biases(messages, _hook_ctx)- if self.config.hooks- else None- )-- if self.config.mode == "token_headroom":- comp_cache = self._get_compression_cache(session_id)-- # Zone 1: Swap cached compressed versions into working copy- working_messages = comp_cache.apply_cached(messages)-- # Re-freeze boundary: consecutive stable messages from start- frozen_message_count = comp_cache.compute_frozen_count(messages)-- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.anthropic_pipeline.apply(- messages=working_messages,- model=model,- model_limit=context_limit,- context=extract_user_query(working_messages),- frozen_message_count=frozen_message_count,- biases=biases,- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- # Cache newly compressed messages (index-aligned diff)- if result.messages != working_messages:- comp_cache.update_from_result(messages, result.messages)-- # Always use pipeline result — Zone 1 swaps are already applied- optimized_messages = result.messages- transforms_applied = result.transforms_applied- pipeline_timing = result.timing- # Keep original_tokens as the REAL original (pre-Zone-1-swap)- # so tokens_saved captures both Zone 1 + Zone 2 savings.- # original_tokens was set at line ~2183 from uncompressed messages.- optimized_tokens = result.tokens_after- else:- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.anthropic_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- frozen_message_count=frozen_message_count,- biases=biases,- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- pipeline_timing = result.timing- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after-- if result.waste_signals:- waste_signals_dict = result.waste_signals.to_dict()- except Exception as e:- logger.warning(f"Optimization failed: {e}")- # Flag compression failure for observability- _compression_failed = True-- tokens_saved = max(0, original_tokens - optimized_tokens)- optimization_latency = (time.time() - start_time) * 1000-- # Hook: post_compress — let hooks observe compression results- if self.config.hooks and tokens_saved > 0:- from headroom.hooks import CompressEvent-- try:- self.config.hooks.post_compress(- CompressEvent(- tokens_before=original_tokens,- tokens_after=optimized_tokens,- tokens_saved=tokens_saved,- compression_ratio=tokens_saved / original_tokens- if original_tokens > 0- else 0,- transforms_applied=transforms_applied,- model=model,- user_query=_hook_ctx.user_query if self.config.hooks else "",- provider="anthropic",- )- )- except Exception as e:- logger.debug(f"[{request_id}] post_compress hook error: {e}")-- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = body.get("tools")- _original_tools = tools # Preserve for diagnostic / future retry- if (- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions- ) and not _bypass:- # Create fresh injector to avoid state leakage between requests- injector = CCRToolInjector(- provider="anthropic",- inject_tool=self.config.ccr_inject_tool,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- optimized_messages, tools, was_injected = injector.process_request(- optimized_messages, tools- )-- if injector.has_compressed_content:- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"- )- else:- logger.debug(- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"- )-- # Track compression in context tracker for multi-turn awareness- if self.ccr_context_tracker:- self._turn_counter += 1- for hash_key in injector.detected_hashes:- # Get compression metadata from store- store = get_compression_store()- entry = store.get_metadata(hash_key)- if entry:- self.ccr_context_tracker.track_compression(- hash_key=hash_key,- turn_number=self._turn_counter,- tool_name=entry.get("tool_name"),- original_count=entry.get("original_item_count", 0),- compressed_count=entry.get("compressed_item_count", 0),- query_context=entry.get("query_context", ""),- sample_content=entry.get("compressed_content", "")[:500],- )-- # CCR Proactive Expansion: Check if current query needs expanded context- if self.ccr_context_tracker and self.config.ccr_proactive_expansion:- # Extract user query from messages- user_query = ""- for msg in reversed(messages):- if msg.get("role") == "user":- content = msg.get("content", "")- if isinstance(content, str):- user_query = content- elif isinstance(content, list):- for block in content:- if isinstance(block, dict) and block.get("type") == "text":- user_query = block.get("text", "")- break- break-- if user_query:- recommendations = self.ccr_context_tracker.analyze_query(- user_query, self._turn_counter- )- if recommendations:- expansions = self.ccr_context_tracker.execute_expansions(recommendations)- if expansions:- # Add expanded context to the system message or as additional context- expansion_text = self.ccr_context_tracker.format_expansions_for_context(- expansions- )- logger.info(- f"[{request_id}] CCR: Proactively expanded {len(expansions)} context(s) "- f"based on query relevance"- )- # Append to the last user message- if optimized_messages and optimized_messages[-1].get("role") == "user":- last_msg = optimized_messages[-1]- content = last_msg.get("content", "")- if isinstance(content, str):- optimized_messages[-1] = {- **last_msg,- "content": content + "\n\n" + expansion_text,- }-- # Traffic Learner: Extract patterns from inbound tool results- if self.traffic_learner:- try:- # Wire backend on first use (lazy init after memory handler is ready)- if (- self.traffic_learner._backend is None- and self.memory_handler- and self.memory_handler.initialized- and self.memory_handler.backend- ):- self.traffic_learner.set_backend(self.memory_handler.backend)-- # Extract tool results from messages and learn from them- tool_results = self.traffic_learner.extract_tool_results_from_messages(- optimized_messages- )- for tr in tool_results[-5:]: # Only recent results- await self.traffic_learner.on_tool_result(- tool_name=tr["tool_name"],- tool_input=tr["input"],- tool_output=tr["output"],- is_error=tr["is_error"],- )-- # Also extract preference signals from user messages- await self.traffic_learner.on_messages(optimized_messages)- except Exception as e:- logger.debug(f"[{request_id}] Traffic learner: {e}")-- # Memory: Inject context and tools- if self.memory_handler and memory_user_id:- # Search and inject memory context- if self.memory_handler.config.inject_context:- try:- memory_context = await self.memory_handler.search_and_format_context(- memory_user_id, optimized_messages- )- if memory_context:- optimized_messages = self._inject_system_context(- optimized_messages, memory_context, body=body- )- logger.info(- f"[{request_id}] Memory: Injected {len(memory_context)} chars of context"- )- except Exception as e:- logger.warning(f"[{request_id}] Memory: Context injection failed: {e}")-- # Inject memory tools- if self.memory_handler.config.inject_tools:- tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "anthropic")- if mem_tools_injected:- tool_names = [- t.get("name") or t.get("type", "")- for t in tools- if t.get("name", "").startswith("memory")- or t.get("type", "").startswith("memory")- ]- logger.info(f"[{request_id}] Memory: Injected tools: {tool_names}")-- # Add beta headers for native memory tool- beta_headers = self.memory_handler.get_beta_headers()- if beta_headers:- for key, value in beta_headers.items():- # Merge with existing beta header if present- existing = headers.get(key, "")- if existing and value not in existing:- headers[key] = f"{existing},{value}"- else:- headers[key] = value- logger.info(- f"[{request_id}] Memory: Added beta header: {key}={headers[key]}"- )-- # Query Echo: disabled — hurts prefix caching in long conversations.- # The echo changes every turn, invalidating the cached prefix.- # To re-enable, uncomment and set query_echo_enabled on ProxyConfig.-- # Update body- body["messages"] = optimized_messages- if tools is not None:- body["tools"] = tools-- # Forward request - use Bedrock backend if configured, otherwise direct API- if self.anthropic_backend is not None:- # Route through Bedrock backend- try:- if stream:- return await self._stream_response_bedrock(- body,- headers,- "anthropic",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- pipeline_timing=pipeline_timing,- )- else:- backend_response = await self.anthropic_backend.send_message(body, headers)-- if backend_response.error:- return JSONResponse(- status_code=backend_response.status_code,- content=backend_response.body,- )-- # Track metrics- total_latency = (time.time() - start_time) * 1000- usage = backend_response.body.get("usage", {})- output_tokens = usage.get("output_tokens", 0)-- _backend_name = (- self.anthropic_backend.name if self.anthropic_backend else "anthropic"- )- await self.metrics.record_request(- provider=_backend_name,- model=model,- input_tokens=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- cached=False,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)-- # Log request- if self.logger:- self.logger.log(- RequestLog(- request_id=request_id,- timestamp=datetime.now().isoformat(),- provider=_backend_name,- model=model,- input_tokens_original=original_tokens,- input_tokens_optimized=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- savings_percent=(tokens_saved / original_tokens * 100)- if original_tokens > 0- else 0,- optimization_latency_ms=optimization_latency,- total_latency_ms=total_latency,- tags=tags,- cache_hit=False,- transforms_applied=transforms_applied,- request_messages=body.get("messages")- if self.config.log_full_messages- else None,- )- )-- return JSONResponse(- status_code=backend_response.status_code,- content=backend_response.body,- )- except Exception as e:- logger.error(f"[{request_id}] Bedrock backend error: {e}")- return JSONResponse(- status_code=500,- content={- "type": "error",- "error": {"type": "api_error", "message": str(e)},- },- )-- # Direct Anthropic API- url = f"{self.ANTHROPIC_API_URL}/v1/messages"-- try:- if stream:- return await self._stream_response(- url,- headers,- body,- "anthropic",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- memory_user_id=memory_user_id,- pipeline_timing=pipeline_timing,- prefix_tracker=prefix_tracker,- )- else:- response = await self._retry_request("POST", url, headers, body)-- # Full diagnostic dump on upstream errors.- # Writes pre/post compression messages, tools, and error- # to ~/.headroom/logs/debug_400/ for offline analysis.- if response.status_code >= 400:- try:- err_body = response.json()- err_msg = err_body.get("error", {}).get("message", "")- err_type = err_body.get("error", {}).get("type", "")- except Exception:- err_body = {"raw": response.text[:2000]}- err_msg = str(response.text[:500])- err_type = "parse_error"-- logger.warning(- f"[{request_id}] UPSTREAM_ERROR "- f"status={response.status_code} "- f"error_type={err_type} "- f"error_msg={err_msg!r} "- f"model={model} "- f"compressed={'yes' if transforms_applied else 'no'} "- f"transforms={transforms_applied} "- f"original_tokens={original_tokens} "- f"optimized_tokens={optimized_tokens} "- f"message_count={len(body.get('messages', []))} "- f"stream={stream}"- )-- # Dump full request details to debug file- try:- debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"- debug_dir.mkdir(parents=True, exist_ok=True)- ts = datetime.now().strftime("%Y%m%d_%H%M%S")- debug_file = debug_dir / f"{ts}_{request_id}.json"-- # Sanitize headers (redact API keys)- safe_headers = {}- for k, v in headers.items():- if k.lower() in ("x-api-key", "authorization"):- safe_headers[k] = v[:12] + "..." if v else ""- else:- safe_headers[k] = v-- debug_payload = {- "request_id": request_id,- "timestamp": datetime.now().isoformat(),- "status_code": response.status_code,- "error_response": err_body,- "model": model,- "stream": stream,- "headers": safe_headers,- "compression": {- "was_compressed": bool(transforms_applied),- "transforms": transforms_applied,- "original_tokens": original_tokens,- "optimized_tokens": optimized_tokens,- "tokens_saved": tokens_saved,- "compression_failed": _compression_failed,- },- "tools_sent": body.get("tools"),- "tool_count": len(body.get("tools") or []),- "original_tool_count": len(_original_tools or []),- "messages_sent": body.get("messages"),- "message_count": len(body.get("messages", [])),- "original_messages": (- original_messages- if original_messages is not body.get("messages")- else "__same_as_sent__"- ),- "original_message_count": len(original_messages),- "system_prompt": body.get("system"),- }-- with open(debug_file, "w") as f:- json.dump(debug_payload, f, indent=2, default=str)-- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")- except Exception as dump_err:- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")-- # Parse response for CCR handling- resp_json = None- try:- resp_json = response.json()- except (json.JSONDecodeError, ValueError) as e:- logger.debug(- f"[{request_id}] Failed to parse response JSON for CCR handling: {e}"- )-- # CCR Response Handling: Handle headroom_retrieve tool calls automatically- if (- self.ccr_response_handler- and resp_json- and response.status_code == 200- and self.ccr_response_handler.has_ccr_tool_calls(resp_json, "anthropic")- ):- logger.info(f"[{request_id}] CCR: Detected retrieval tool call, handling...")-- # Create API call function for continuation- # Use a fresh client to avoid potential decompression state issues- async def api_call_fn(- msgs: list[dict], tls: list[dict] | None- ) -> dict[str, Any]:- continuation_body = {- **body,- "messages": msgs,- }- if tls is not None:- continuation_body["tools"] = tls-- # Use clean headers for continuation- continuation_headers = {- k: v- for k, v in headers.items()- if k.lower()- not in (- "content-encoding",- "transfer-encoding",- "accept-encoding",- "content-length",- )- }-- # Reuse main client for CCR continuations (connection pooling)- logger.info(f"CCR: Making continuation request with {len(msgs)} messages")- assert self.http_client is not None, "HTTP client not initialized"- try:- cont_response = await self.http_client.post(- url,- json=continuation_body,- headers=continuation_headers,- timeout=httpx.Timeout(120.0), # Override timeout for CCR- )- logger.info(- f"CCR: Got response status={cont_response.status_code}, "- f"content-encoding={cont_response.headers.get('content-encoding')}"- )- result: dict[str, Any] = cont_response.json()- logger.info("CCR: Parsed JSON successfully")- return result- except Exception as e:- resp_headers: str | dict[str, str] = "N/A"- try:- resp_headers = dict(cont_response.headers)- except Exception:- pass- logger.error(- f"CCR: API call failed: {e}, response headers: {resp_headers}"- )- raise-- # Handle CCR tool calls- try:- final_resp_json = await self.ccr_response_handler.handle_response(- resp_json,- optimized_messages,- tools,- api_call_fn,- provider="anthropic",- )- # Update response content with final response- resp_json = final_resp_json- # Remove encoding headers since content is now uncompressed JSON- ccr_response_headers = {- k: v- for k, v in response.headers.items()- if k.lower() not in ("content-encoding", "content-length")- }- try:- ccr_content = json.dumps(final_resp_json).encode()- except (TypeError, ValueError) as json_err:- logger.warning(- f"[{request_id}] CCR: JSON serialization failed: {json_err}"- )- ccr_content = json.dumps(resp_json).encode()- response = httpx.Response(- status_code=200,- content=ccr_content,- headers=ccr_response_headers,- )- logger.info(f"[{request_id}] CCR: Retrieval handled successfully")- except Exception as e:- import traceback-- logger.warning(- f"[{request_id}] CCR: Response handling failed: {e}\n"- f"Traceback: {traceback.format_exc()}"- )- # Continue with original response-- # Memory: Handle memory tool calls in response- if (- self.memory_handler- and memory_user_id- and resp_json- and response.status_code == 200- and self.memory_handler.has_memory_tool_calls(resp_json, "anthropic")- ):- logger.info(f"[{request_id}] Memory: Detected memory tool call, handling...")-- try:- # Execute memory tool calls- tool_results = await self.memory_handler.handle_memory_tool_calls(- resp_json, memory_user_id, "anthropic"- )-- if tool_results:- # Create continuation messages- assistant_msg = {- "role": "assistant",- "content": resp_json.get("content", []),- }- user_msg = {- "role": "user",- "content": tool_results,- }-- continuation_messages = optimized_messages + [assistant_msg, user_msg]-- # Make continuation API call- continuation_body = {**body, "messages": continuation_messages}- if tools:- continuation_body["tools"] = tools-- cont_response = await self._retry_request(- "POST", url, headers, continuation_body- )-- # Update response with continuation- resp_json = cont_response.json()- response = cont_response- logger.info(- f"[{request_id}] Memory: Tool calls handled, continuation complete"- )-- except Exception as e:- logger.warning(f"[{request_id}] Memory: Tool call handling failed: {e}")- # Continue with original response-- total_latency = (time.time() - start_time) * 1000-- # Parse response for output token count and cache metrics- output_tokens = 0- cr_tokens = 0- cw_tokens = 0- uncached_input_tokens = 0- if resp_json:- usage = resp_json.get("usage", {})- output_tokens = usage.get("output_tokens", 0)- cr_tokens = usage.get("cache_read_input_tokens", 0)- cw_tokens = usage.get("cache_creation_input_tokens", 0)- uncached_input_tokens = usage.get("input_tokens", 0)-- # Update prefix cache tracker for next turn- prefix_tracker.update_from_response(- cache_read_tokens=cr_tokens,- cache_write_tokens=cw_tokens,- messages=optimized_messages,- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(- model,- tokens_saved,- optimized_tokens,- cache_read_tokens=cr_tokens,- cache_write_tokens=cw_tokens,- uncached_tokens=uncached_input_tokens,- )-- # Cache response- if self.cache and response.status_code == 200:- await self.cache.set(- messages,- model,- response.content,- dict(response.headers),- tokens_saved=tokens_saved,- )-- # Record metrics — use optimized_tokens (what we sent), not API's- # input_tokens which is just the non-cached portion with prompt caching- await self.metrics.record_request(- provider="anthropic",- model=model,- input_tokens=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- waste_signals=waste_signals_dict,- cache_read_tokens=cr_tokens,- cache_write_tokens=cw_tokens,- uncached_input_tokens=uncached_input_tokens,- )-- # Log request- if self.logger:- self.logger.log(- RequestLog(- request_id=request_id,- timestamp=datetime.now().isoformat(),- provider="anthropic",- model=model,- input_tokens_original=original_tokens,- input_tokens_optimized=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- savings_percent=(tokens_saved / original_tokens * 100)- if original_tokens > 0- else 0,- optimization_latency_ms=optimization_latency,- total_latency_ms=total_latency,- tags=tags,- cache_hit=cache_hit,- transforms_applied=transforms_applied,- waste_signals=waste_signals_dict,- request_messages=messages if self.config.log_full_messages else None,- )- )-- # Structured perf log line for `headroom perf` analysis- num_msgs = len(messages)- resp_usage = resp_json.get("usage", {}) if resp_json else {}- cr = resp_usage.get("cache_read_input_tokens", 0)- cw = resp_usage.get("cache_creation_input_tokens", 0)- chp = round(cr / (cr + cw) * 100) if (cr + cw) > 0 else 0- timing_str = (- " ".join(f"{k}={v:.0f}ms" for k, v in pipeline_timing.items())- if pipeline_timing- else ""- )- logger.info(- f"[{request_id}] PERF "- f"model={model} msgs={num_msgs} "- f"tok_before={original_tokens} tok_after={optimized_tokens} "- f"tok_saved={tokens_saved} "- f"cache_read={cr} cache_write={cw} cache_hit_pct={chp} "- f"opt_ms={optimization_latency:.0f} "- f"transforms={_summarize_transforms(transforms_applied)}"- f"{' timing=' + timing_str if timing_str else ''}"- )-- # Remove compression headers since httpx already decompressed the response- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression-- # Inject Headroom compression metrics (for SaaS metering)- response_headers["x-headroom-tokens-before"] = str(original_tokens)- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)- response_headers["x-headroom-model"] = model- if transforms_applied:- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)- if cache_hit:- response_headers["x-headroom-cached"] = "true"- if _compression_failed:- response_headers["x-headroom-compression-failed"] = "true"-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- except Exception as e:- await self.metrics.record_failed()- # Log full error details internally for debugging- logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")-- # Try fallback if enabled- if self.config.fallback_enabled and self.config.fallback_provider == "openai":- logger.info(f"[{request_id}] Attempting fallback to OpenAI")- # Convert to OpenAI format and retry- # (simplified - would need message format conversion)-- # Return sanitized error message to client (don't expose internal details)- return JSONResponse(- status_code=502,- content={- "type": "error",- "error": {- "type": "api_error",- "message": "An error occurred while processing your request. Please try again.",- },- },- )-- async def handle_anthropic_batch_create(- self,- request: Request,- ) -> Response:- """Handle Anthropic POST /v1/messages/batches endpoint with compression.-- Anthropic batch format:- {- "requests": [- {- "custom_id": "req-1",- "params": {- "model": "claude-sonnet-4-20250514",- "max_tokens": 1024,- "messages": [{"role": "user", "content": "Hello"}]- }- },- ...- ]- }-- This method applies compression to each request's messages before forwarding.- """- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "type": "error",- "error": {- "type": "request_too_large",- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- },- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "type": "error",- "error": {- "type": "invalid_request_error",- "message": f"Invalid request body: {e!s}",- },- },- )-- requests_list = body.get("requests", [])- if not requests_list:- return JSONResponse(- status_code=400,- content={- "type": "error",- "error": {- "type": "invalid_request_error",- "message": "Missing or empty 'requests' field in batch request",- },- },- )-- # Extract headers- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)-- # Track compression stats across all batch requests- total_original_tokens = 0- total_optimized_tokens = 0- total_tokens_saved = 0- compressed_requests = []- pipeline_timing: dict[str, float] = {}-- # Apply compression to each request in the batch- for batch_req in requests_list:- custom_id = batch_req.get("custom_id", "")- params = batch_req.get("params", {})- messages = params.get("messages", [])- model = params.get("model", "unknown")-- if not messages or not self.config.optimize:- # No messages or optimization disabled - pass through unchanged- compressed_requests.append(batch_req)- continue-- # Apply optimization- try:- context_limit = self.anthropic_provider.get_context_limit(model)- result = self.anthropic_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )-- optimized_messages = result.messages- for k, v in result.timing.items():- pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v- # Use pipeline's token counts for consistency with pipeline logs- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after- total_original_tokens += original_tokens- total_optimized_tokens += optimized_tokens- tokens_saved = max(0, original_tokens - optimized_tokens)- total_tokens_saved += tokens_saved-- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = params.get("tools")- if self.config.ccr_inject_tool and tokens_saved > 0:- injector = CCRToolInjector(- provider="anthropic",- inject_tool=True,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- optimized_messages, tools, was_injected = injector.process_request(- optimized_messages, tools- )- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for batch request '{custom_id}'"- )-- # Create compressed batch request- compressed_params = {**params, "messages": optimized_messages}- if tools is not None:- compressed_params["tools"] = tools- compressed_requests.append(- {- "custom_id": custom_id,- "params": compressed_params,- }- )-- if tokens_saved > 0:- logger.debug(- f"[{request_id}] Batch request '{custom_id}': "- f"{original_tokens:,} -> {optimized_tokens:,} tokens "- f"(saved {tokens_saved:,})"- )-- except Exception as e:- logger.warning(- f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}"- )- # Pass through unchanged on failure- compressed_requests.append(batch_req)- total_optimized_tokens += original_tokens-- # Update body with compressed requests- body["requests"] = compressed_requests-- optimization_latency = (time.time() - start_time) * 1000-- # Forward request to Anthropic- url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches"-- try:- response = await self._retry_request("POST", url, headers, body)-- # Record metrics- await self.metrics.record_request(- provider="anthropic",- model="batch",- input_tokens=total_optimized_tokens,- output_tokens=0,- tokens_saved=total_tokens_saved,- latency_ms=optimization_latency,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- )-- # Log compression stats- if total_tokens_saved > 0:- savings_percent = (- (total_tokens_saved / total_original_tokens * 100)- if total_original_tokens > 0- else 0- )- logger.info(- f"[{request_id}] Batch ({len(compressed_requests)} requests): "- f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "- f"(saved {total_tokens_saved:,}, {savings_percent:.1f}%)"- )-- # Store batch context for CCR result processing- if response.status_code == 200 and self.config.ccr_inject_tool:- try:- response_data = response.json()- batch_id = response_data.get("id")- if batch_id:- await self._store_anthropic_batch_context(- batch_id,- requests_list,- headers.get("x-api-key"),- )- except Exception as e:- logger.warning(f"[{request_id}] Failed to store batch context: {e}")-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- except Exception as e:- await self.metrics.record_failed()- logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")- return JSONResponse(- status_code=502,- content={- "type": "error",- "error": {- "type": "api_error",- "message": "An error occurred while processing your batch request. Please try again.",- },- },- )-- async def handle_anthropic_batch_passthrough(- self,- request: Request,- batch_id: str | None = None,- ) -> Response:- """Handle Anthropic batch passthrough endpoints.-- Used for:- - GET /v1/messages/batches - List batches- - GET /v1/messages/batches/{batch_id} - Get batch- - GET /v1/messages/batches/{batch_id}/results - Get batch results- - POST /v1/messages/batches/{batch_id}/cancel - Cancel batch- """- start_time = time.time()- path = request.url.path- url = f"{self.ANTHROPIC_API_URL}{path}"-- # Preserve query string parameters (e.g., limit, after_id for list endpoint)- if request.url.query:- url = f"{url}?{request.url.query}"-- headers = dict(request.headers.items())- headers.pop("host", None)-- body = await request.body()-- response = await self.http_client.request( # type: ignore[union-attr]- method=request.method,- url=url,- headers=headers,- content=body,- )-- # Track metrics- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider="anthropic",- model="passthrough:batches",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- )-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- async def _store_anthropic_batch_context(- self,- batch_id: str,- requests_list: list[dict[str, Any]],- api_key: str | None,- ) -> None:- """Store batch context for CCR result processing.-- Args:- batch_id: The batch ID from the API response.- requests_list: The original batch requests.- api_key: The API key for continuation calls.- """- store = get_batch_context_store()- context = BatchContext(- batch_id=batch_id,- provider="anthropic",- api_key=api_key,- api_base_url=self.ANTHROPIC_API_URL,- )-- for batch_req in requests_list:- custom_id = batch_req.get("custom_id", "")- params = batch_req.get("params", {})- context.add_request(- BatchRequestContext(- custom_id=custom_id,- messages=params.get("messages", []),- tools=params.get("tools"),- model=params.get("model", ""),- extras={- "max_tokens": params.get("max_tokens", 4096),- "system": params.get("system"),- },- )- )-- await store.store(context)- logger.debug(f"Stored batch context for {batch_id} with {len(requests_list)} requests")-- async def handle_anthropic_batch_results(- self,- request: Request,- batch_id: str,- ) -> Response:- """Handle Anthropic batch results with CCR post-processing.-- This endpoint:- 1. Fetches raw results from Anthropic- 2. Detects CCR tool calls in each result- 3. Executes retrieval and makes continuation calls- 4. Returns processed results with complete responses- """- start_time = time.time()-- # Forward request to get raw results- url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches/{batch_id}/results"-- if request.url.query:- url = f"{url}?{request.url.query}"-- headers = dict(request.headers.items())- headers.pop("host", None)-- response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]-- if response.status_code != 200:- # Error - pass through- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Parse results - Anthropic batch results are JSONL format- raw_content = response.content.decode("utf-8")- results = []- for line in raw_content.strip().split("\n"):- if line.strip():- try:- results.append(json.loads(line))- except json.JSONDecodeError:- continue-- if not results:- # No results to process- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Check if we have context and CCR processing is enabled- store = get_batch_context_store()- batch_context = await store.get(batch_id)-- if batch_context is None or not self.config.ccr_inject_tool:- # No context or CCR disabled - pass through- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Process results with CCR handler- processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]- processed = await processor.process_results(batch_id, results, "anthropic")-- # Convert back to JSONL format- processed_lines = []- for p in processed:- processed_lines.append(json.dumps(p.result))- if p.was_processed:- logger.info(- f"CCR: Processed batch result {p.custom_id} "- f"({p.continuation_rounds} continuation rounds)"- )-- processed_content = "\n".join(processed_lines)-- # Track metrics- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider="anthropic",- model="batch:ccr-processed",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- )-- return Response(- content=processed_content.encode("utf-8"),- status_code=200,- media_type="application/jsonl",- )-- # =========================================================================- # Google/Gemini Batch API Handlers- # =========================================================================-- async def handle_google_batch_create(- self,- request: Request,- model: str,- ) -> Response:- """Handle Google POST /v1beta/models/{model}:batchGenerateContent endpoint.-- Google batch format:- {- "batch": {- "display_name": "my-batch",- "input_config": {- "requests": {- "requests": [- {- "request": {"contents": [{"parts": [{"text": "..."}]}]},- "metadata": {"key": "request-1"}- }- ]- }- }- }- }-- This method applies compression to each request's contents before forwarding.- """- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "error": {- "code": 413,- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "status": "INVALID_ARGUMENT",- }- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "code": 400,- "message": f"Invalid request body: {e!s}",- "status": "INVALID_ARGUMENT",- }- },- )-- # Extract batch config- batch_config = body.get("batch", {})- input_config = batch_config.get("input_config", {})- requests_wrapper = input_config.get("requests", {})- requests_list = requests_wrapper.get("requests", [])-- if not requests_list:- # No inline requests - might be using file input, pass through- logger.debug(f"[{request_id}] Google batch: No inline requests, passing through")- return await self._google_batch_passthrough(request, model, body)-- # Extract headers- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)-- # Track compression stats- total_original_tokens = 0- total_optimized_tokens = 0- total_tokens_saved = 0- compressed_requests = []- pipeline_timing: dict[str, float] = {}-- # Apply compression to each request in the batch- for idx, batch_req in enumerate(requests_list):- req_content = batch_req.get("request", {})- metadata = batch_req.get("metadata", {})- contents = req_content.get("contents", [])-- if not contents or not self.config.optimize:- # No contents or optimization disabled - pass through unchanged- compressed_requests.append(batch_req)- continue-- # Convert Google format to messages for compression- system_instruction = req_content.get("systemInstruction")- messages, preserved_indices = self._gemini_contents_to_messages(- contents, system_instruction- )-- # Store original content entries that have non-text parts before compression- preserved_contents = {idx: contents[idx] for idx in preserved_indices}-- # Early exit if ALL content has non-text parts (nothing to compress)- if len(preserved_indices) == len(contents):- # All content has non-text parts, skip compression- compressed_requests.append(batch_req)- continue-- # Apply optimization- try:- # Default context limit for most models- context_limit = 128000-- # Use OpenAI pipeline (similar message format after conversion)- result = self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )-- optimized_messages = result.messages- for k, v in result.timing.items():- pipeline_timing[k] = pipeline_timing.get(k, 0.0) + v- # Use pipeline's token counts for consistency with pipeline logs- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after- total_original_tokens += original_tokens- total_optimized_tokens += optimized_tokens- tokens_saved = max(0, original_tokens - optimized_tokens)- total_tokens_saved += tokens_saved-- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = req_content.get("tools")- # Extract existing function declarations if present- existing_funcs = None- if tools:- for tool in tools:- if "functionDeclarations" in tool:- existing_funcs = tool["functionDeclarations"]- break-- if self.config.ccr_inject_tool and tokens_saved > 0:- injector = CCRToolInjector(- provider="google",- inject_tool=True,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- optimized_messages, injected_funcs, was_injected = injector.process_request(- optimized_messages, existing_funcs- )- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for Google batch request {idx}"- )- existing_funcs = injected_funcs-- # Convert back to Google contents format- optimized_contents, optimized_sys_inst = self._messages_to_gemini_contents(- optimized_messages- )-- # Restore preserved content entries that had non-text parts- for orig_idx, original_content in preserved_contents.items():- if orig_idx < len(optimized_contents):- optimized_contents[orig_idx] = original_content-- # Create compressed batch request- compressed_req_content = {**req_content, "contents": optimized_contents}- if optimized_sys_inst:- compressed_req_content["systemInstruction"] = optimized_sys_inst- if existing_funcs is not None:- compressed_req_content["tools"] = [{"functionDeclarations": existing_funcs}]-- compressed_req = {- "request": compressed_req_content,- "metadata": metadata,- }-- compressed_requests.append(compressed_req)-- if tokens_saved > 0:- logger.debug(- f"[{request_id}] Google batch request {idx}: "- f"{original_tokens:,} -> {optimized_tokens:,} tokens "- f"(saved {tokens_saved:,})"- )-- except Exception as e:- logger.warning(- f"[{request_id}] Optimization failed for Google batch request {idx}: {e}"- )- # Pass through unchanged on failure- compressed_requests.append(batch_req)- total_optimized_tokens += original_tokens-- # Update body with compressed requests- body["batch"]["input_config"]["requests"]["requests"] = compressed_requests-- optimization_latency = (time.time() - start_time) * 1000-- # Forward request to Google- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"-- # Add API key to URL if present in headers- api_key = headers.pop("x-goog-api-key", None)- if api_key:- url = f"{url}?key={api_key}"-- try:- response = await self._retry_request("POST", url, headers, body)-- # Record metrics- await self.metrics.record_request(- provider="google",- model=f"batch:{model}",- input_tokens=total_optimized_tokens,- output_tokens=0,- tokens_saved=total_tokens_saved,- latency_ms=optimization_latency,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- )-- # Log compression stats- if total_tokens_saved > 0:- savings_percent = (- (total_tokens_saved / total_original_tokens * 100)- if total_original_tokens > 0- else 0- )- logger.info(- f"[{request_id}] Google batch compression: "- f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "- f"({savings_percent:.1f}% saved across {len(requests_list)} requests)"- )-- # Store batch context for CCR result processing- if response.status_code == 200 and self.config.ccr_inject_tool:- try:- response_data = response.json()- batch_name = response_data.get("name")- if batch_name:- await self._store_google_batch_context(- batch_name,- requests_list,- model,- api_key,- )- except Exception as e:- logger.warning(f"[{request_id}] Failed to store Google batch context: {e}")-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- except Exception as e:- logger.error(f"[{request_id}] Google batch request failed: {e}")- return JSONResponse(- status_code=500,- content={- "error": {- "code": 500,- "message": f"Failed to forward batch request: {e!s}",- "status": "INTERNAL",- }- },- )-- async def _google_batch_passthrough(- self,- request: Request,- model: str,- body: dict | None = None,- ) -> Response:- """Pass through Google batch request without modification."""- start_time = time.time()-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)-- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"-- # Add API key to URL if present in headers- api_key = headers.pop("x-goog-api-key", None)- if api_key:- url = f"{url}?key={api_key}"-- if body is None:- body_content = await request.body()- else:- body_content = json.dumps(body).encode()-- response = await self.http_client.post( # type: ignore[union-attr]- url,- headers=headers,- content=body_content,- )-- # Track metrics- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider="google",- model=f"passthrough:batch:{model}",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- )-- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- async def handle_google_batch_passthrough(- self,- request: Request,- batch_name: str | None = None,- ) -> Response:- """Handle Google batch passthrough endpoints.-- Used for:- - GET /v1beta/batches/{batch_name} - Get batch status- - POST /v1beta/batches/{batch_name}:cancel - Cancel batch- - DELETE /v1beta/batches/{batch_name} - Delete batch- """- start_time = time.time()- path = request.url.path- url = f"{self.GEMINI_API_URL}{path}"-- # Preserve query string parameters- if request.url.query:- url = f"{url}?{request.url.query}"-- headers = dict(request.headers.items())- headers.pop("host", None)-- # Handle API key- api_key = headers.pop("x-goog-api-key", None)- if api_key:- if "?" in url:- url = f"{url}&key={api_key}"- else:- url = f"{url}?key={api_key}"-- body = await request.body()-- response = await self.http_client.request( # type: ignore[union-attr]- method=request.method,- url=url,- headers=headers,- content=body,- )-- # Track metrics- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider="google",- model="passthrough:batches",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- )-- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- async def _store_google_batch_context(- self,- batch_name: str,- requests_list: list[dict[str, Any]],- model: str,- api_key: str | None,- ) -> None:- """Store Google batch context for CCR result processing.-- Args:- batch_name: The batch name from the API response.- requests_list: The original batch requests.- model: The model used for the batch.- api_key: The API key for continuation calls.- """- store = get_batch_context_store()- context = BatchContext(- batch_id=batch_name,- provider="google",- api_key=api_key,- api_base_url=self.GEMINI_API_URL,- )-- for batch_req in requests_list:- metadata = batch_req.get("metadata", {})- custom_id = metadata.get("key", "")- req_content = batch_req.get("request", {})- contents = req_content.get("contents", [])- system_instruction = req_content.get("systemInstruction")-- # Convert contents to messages format for CCR handler- messages, _ = self._gemini_contents_to_messages(contents, system_instruction)-- # Extract system instruction text if present- sys_text = None- if system_instruction:- parts = system_instruction.get("parts", [])- if parts and isinstance(parts[0], dict):- sys_text = parts[0].get("text")-- context.add_request(- BatchRequestContext(- custom_id=custom_id,- messages=messages,- tools=req_content.get("tools"),- model=model,- system_instruction=sys_text,- )- )-- await store.store(context)- logger.debug(- f"Stored Google batch context for {batch_name} with {len(requests_list)} requests"- )-- async def handle_google_batch_results(- self,- request: Request,- batch_name: str,- ) -> Response:- """Handle Google batch results with CCR post-processing.-- Google batch results endpoint returns the batch operation status.- When status is SUCCEEDED, results are embedded in the response.- This handler processes CCR tool calls in those results.- """- start_time = time.time()-- # Forward request to get batch status/results- url = f"{self.GEMINI_API_URL}/v1beta/{batch_name}"-- if request.url.query:- url = f"{url}?{request.url.query}"-- headers = dict(request.headers.items())- headers.pop("host", None)-- # Handle API key- api_key = headers.pop("x-goog-api-key", None)- if api_key:- if "?" in url:- url = f"{url}&key={api_key}"- else:- url = f"{url}?key={api_key}"-- response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]-- if response.status_code != 200:- # Error - pass through- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Parse response- try:- response_data = response.json()- except json.JSONDecodeError:- # Not JSON - pass through- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Check if batch has results (state must be SUCCEEDED)- metadata = response_data.get("metadata", {})- state = metadata.get("state")-- if state != "SUCCEEDED":- # Batch not complete - pass through- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Extract results from response- # Google embeds results in the batch response- results = response_data.get("response", {}).get("responses", [])-- if not results:- # No results to process- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Check if we have context and CCR processing is enabled- store = get_batch_context_store()- batch_context = await store.get(batch_name)-- if batch_context is None or not self.config.ccr_inject_tool:- # No context or CCR disabled - pass through- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Process results with CCR handler- processor = BatchResultProcessor(self.http_client) # type: ignore[arg-type]- processed = await processor.process_results(batch_name, results, "google")-- # Update response with processed results- processed_results = [p.result for p in processed]- response_data["response"]["responses"] = processed_results-- for p in processed:- if p.was_processed:- logger.info(- f"CCR: Processed Google batch result {p.custom_id} "- f"({p.continuation_rounds} continuation rounds)"- )-- # Track metrics- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider="google",- model="batch:ccr-processed",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- )-- return JSONResponse(content=response_data, status_code=200)-- def _parse_sse_usage(self, chunk: bytes, provider: str) -> dict[str, int] | None:- """Parse usage information from SSE chunk.-- For Anthropic: Looks for message_start (input tokens) and message_delta (output tokens)- For OpenAI: Looks for final chunk with usage object (requires stream_options.include_usage=true)- For Gemini: Looks for usageMetadata in each chunk-- Returns dict with keys: input_tokens, output_tokens, cache_read_input_tokens, cache_creation_input_tokens- Returns None if no usage found in this chunk.- """- try:- text = chunk.decode("utf-8", errors="ignore")- # SSE format: "data: {...}\n\n" or "event: ...\ndata: {...}\n\n"- for line in text.split("\n"):- if not line.startswith("data: "):- continue- data_str = line[6:].strip()- if not data_str or data_str == "[DONE]":- continue-- try:- data = json.loads(data_str)- except json.JSONDecodeError:- continue-- usage = {}-- if provider == "anthropic":- # Anthropic sends message_start with input tokens- # and message_delta with output tokens- event_type = data.get("type", "")-- if event_type == "message_start":- msg = data.get("message", {})- msg_usage = msg.get("usage", {})- if msg_usage:- usage["input_tokens"] = msg_usage.get("input_tokens", 0)- usage["cache_read_input_tokens"] = msg_usage.get(- "cache_read_input_tokens", 0- )- usage["cache_creation_input_tokens"] = msg_usage.get(- "cache_creation_input_tokens", 0- )-- elif event_type == "message_delta":- delta_usage = data.get("usage", {})- if delta_usage:- usage["output_tokens"] = delta_usage.get("output_tokens", 0)-- elif provider == "openai":- # OpenAI sends usage in final chunk (when stream_options.include_usage=true)- chunk_usage = data.get("usage")- if chunk_usage:- usage["input_tokens"] = chunk_usage.get("prompt_tokens", 0)- usage["output_tokens"] = chunk_usage.get("completion_tokens", 0)- # OpenAI has cached tokens in prompt_tokens_details- details = chunk_usage.get("prompt_tokens_details", {})- usage["cache_read_input_tokens"] = details.get("cached_tokens", 0)-- elif provider == "gemini":- # Gemini sends usageMetadata in each streaming chunk- # Format: {"usageMetadata": {"promptTokenCount": N, "candidatesTokenCount": M}}- usage_meta = data.get("usageMetadata")- if usage_meta:- usage["input_tokens"] = usage_meta.get("promptTokenCount", 0)- usage["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)- # Gemini also has cachedContentTokenCount for context caching- usage["cache_read_input_tokens"] = usage_meta.get(- "cachedContentTokenCount", 0- )-- if usage:- return usage-- except (UnicodeDecodeError, KeyError, TypeError) as e:- # Don't fail streaming on parse errors- logger.debug(f"SSE usage parsing error for {provider}: {e}")-- return None-- def _parse_sse_usage_from_buffer(- self, stream_state: dict[str, Any], provider: str- ) -> dict[str, int] | None:- """Parse usage from buffered SSE data, handling split chunks.-- Processes complete SSE events (ending with double newline) from the buffer- and removes them from the buffer. Incomplete events are kept in the buffer- for the next chunk.- """- buffer = stream_state["sse_buffer"]- usage_found: dict[str, int] = {}-- # Process complete SSE events (separated by double newlines)- while "\n\n" in buffer:- event_end = buffer.index("\n\n")- event_text = buffer[: event_end + 2]- buffer = buffer[event_end + 2 :]-- # Parse this complete event- for line in event_text.split("\n"):- if not line.startswith("data: "):- continue- data_str = line[6:].strip()- if not data_str or data_str == "[DONE]":- continue-- try:- data = json.loads(data_str)- except json.JSONDecodeError:- continue-- if provider == "anthropic":- event_type = data.get("type", "")- if event_type == "message_start":- msg = data.get("message", {})- msg_usage = msg.get("usage", {})- if msg_usage:- usage_found["input_tokens"] = msg_usage.get("input_tokens", 0)- usage_found["cache_read_input_tokens"] = msg_usage.get(- "cache_read_input_tokens", 0- )- usage_found["cache_creation_input_tokens"] = msg_usage.get(- "cache_creation_input_tokens", 0- )- # INFO logging for cache token tracking (temporary for debugging)- logger.info(- f"[CACHE] Anthropic usage: input={usage_found.get('input_tokens')}, "- f"cache_read={usage_found.get('cache_read_input_tokens')}, "- f"cache_write={usage_found.get('cache_creation_input_tokens')}"- )- elif event_type == "message_delta":- delta_usage = data.get("usage", {})- if delta_usage:- usage_found["output_tokens"] = delta_usage.get("output_tokens", 0)-- elif provider == "openai":- chunk_usage = data.get("usage")- if chunk_usage:- usage_found["input_tokens"] = chunk_usage.get("prompt_tokens", 0)- usage_found["output_tokens"] = chunk_usage.get("completion_tokens", 0)- details = chunk_usage.get("prompt_tokens_details", {})- usage_found["cache_read_input_tokens"] = details.get("cached_tokens", 0)-- elif provider == "gemini":- usage_meta = data.get("usageMetadata")- if usage_meta:- usage_found["input_tokens"] = usage_meta.get("promptTokenCount", 0)- usage_found["output_tokens"] = usage_meta.get("candidatesTokenCount", 0)- usage_found["cache_read_input_tokens"] = usage_meta.get(- "cachedContentTokenCount", 0- )-- # Update buffer with remaining incomplete data- stream_state["sse_buffer"] = buffer-- return usage_found if usage_found else None-- def _parse_sse_to_response(self, sse_data: str, provider: str) -> dict[str, Any] | None:- """Parse SSE data to reconstruct the API response JSON.-- Args:- sse_data: Raw SSE data string.- provider: Provider type for parsing.-- Returns:- Reconstructed response dict or None if parsing fails.- """- if provider != "anthropic":- return None # Only implemented for Anthropic-- response: dict[str, Any] = {"content": [], "usage": {}}- current_block: dict[str, Any] | None = None-- for line in sse_data.split("\n"):- if not line.startswith("data: "):- continue- data_str = line[6:].strip()- if not data_str or data_str == "[DONE]":- continue-- try:- data = json.loads(data_str)- except json.JSONDecodeError:- continue-- event_type = data.get("type", "")-- if event_type == "message_start":- msg = data.get("message", {})- response["id"] = msg.get("id")- response["model"] = msg.get("model")- response["role"] = msg.get("role", "assistant")- response["stop_reason"] = msg.get("stop_reason")- if msg.get("usage"):- response["usage"].update(msg["usage"])-- elif event_type == "content_block_start":- block = data.get("content_block", {})- current_block = {- "type": block.get("type"),- "index": data.get("index", len(response["content"])),- }- if block.get("type") == "text":- current_block["text"] = block.get("text", "")- elif block.get("type") == "tool_use":- current_block["id"] = block.get("id")- current_block["name"] = block.get("name")- current_block["input"] = {}-- elif event_type == "content_block_delta":- if current_block:- delta = data.get("delta", {})- if delta.get("type") == "text_delta":- current_block["text"] = current_block.get("text", "") + delta.get(- "text", ""- )- elif delta.get("type") == "input_json_delta":- # Accumulate partial JSON for tool input- partial = delta.get("partial_json", "")- current_block["_partial_json"] = (- current_block.get("_partial_json", "") + partial- )-- elif event_type == "content_block_stop":- if current_block:- # Parse accumulated JSON for tool_use blocks- if current_block.get("type") == "tool_use" and "_partial_json" in current_block:- try:- current_block["input"] = json.loads(current_block["_partial_json"])- except json.JSONDecodeError:- current_block["input"] = {}- del current_block["_partial_json"]- response["content"].append(current_block)- current_block = None-- elif event_type == "message_delta":- delta = data.get("delta", {})- if delta.get("stop_reason"):- response["stop_reason"] = delta["stop_reason"]- if data.get("usage"):- response["usage"].update(data["usage"])-- return response if response.get("content") else None-- def _response_to_sse(self, response: dict[str, Any], provider: str) -> list[bytes]:- """Convert a response dict back to SSE format.-- Args:- response: API response dict.- provider: Provider type for formatting.-- Returns:- List of SSE event bytes.- """- if provider != "anthropic":- return []-- events: list[bytes] = []-- # message_start- msg_start = {- "type": "message_start",- "message": {- "id": response.get("id", "msg_generated"),- "type": "message",- "role": response.get("role", "assistant"),- "model": response.get("model", "unknown"),- "content": [],- "stop_reason": None,- "usage": response.get("usage", {}),- },- }- events.append(f"event: message_start\ndata: {json.dumps(msg_start)}\n\n".encode())-- # Content blocks- for idx, block in enumerate(response.get("content", [])):- # content_block_start- if block.get("type") == "text":- block_start = {- "type": "content_block_start",- "index": idx,- "content_block": {"type": "text", "text": ""},- }- elif block.get("type") == "tool_use":- block_start = {- "type": "content_block_start",- "index": idx,- "content_block": {- "type": "tool_use",- "id": block.get("id", f"toolu_{idx}"),- "name": block.get("name", ""),- "input": {},- },- }- else:- continue-- events.append(- f"event: content_block_start\ndata: {json.dumps(block_start)}\n\n".encode()- )-- # content_block_delta(s)- if block.get("type") == "text" and block.get("text"):- delta = {- "type": "content_block_delta",- "index": idx,- "delta": {"type": "text_delta", "text": block["text"]},- }- events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())- elif block.get("type") == "tool_use" and block.get("input"):- delta = {- "type": "content_block_delta",- "index": idx,- "delta": {- "type": "input_json_delta",- "partial_json": json.dumps(block["input"]),- },- }- events.append(f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode())-- # content_block_stop- block_stop = {"type": "content_block_stop", "index": idx}- events.append(f"event: content_block_stop\ndata: {json.dumps(block_stop)}\n\n".encode())-- # message_delta- msg_delta = {- "type": "message_delta",- "delta": {"stop_reason": response.get("stop_reason", "end_turn")},- "usage": {"output_tokens": response.get("usage", {}).get("output_tokens", 0)},- }- events.append(f"event: message_delta\ndata: {json.dumps(msg_delta)}\n\n".encode())-- # message_stop- events.append(b'event: message_stop\ndata: {"type": "message_stop"}\n\n')-- return events-- def _record_ccr_feedback_from_response(- self, response: dict, provider: str, request_id: str- ) -> None:- """Extract headroom_retrieve tool calls from a response and record feedback.-- This closes the TOIN feedback loop for streaming responses where- the proxy can't intercept and handle retrieval calls inline.- """- from headroom.cache.compression_store import get_compression_store-- content = response.get("content", [])- if not isinstance(content, list):- return-- store = get_compression_store()-- for block in content:- if not isinstance(block, dict):- continue- if block.get("type") != "tool_use":- continue- if block.get("name") != "headroom_retrieve":- continue-- input_data = block.get("input", {})- hash_key = input_data.get("hash")- query = input_data.get("query")-- if not hash_key:- continue-- logger.info(- f"[{request_id}] CCR Feedback: Recording retrieval "- f"hash={hash_key[:8]}... query={query!r}"- )-- # Call store.retrieve()/search() for the side effect of triggering- # the feedback chain: _log_retrieval -> process_pending_feedback- # -> toin.record_retrieval(). We discard the returned content.- try:- if query:- store.search(hash_key, query)- else:- store.retrieve(hash_key, query=None)- except Exception as e:- logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")-- async def _stream_response(- self,- url: str,- headers: dict,- body: dict,- provider: str,- model: str,- request_id: str,- original_tokens: int,- optimized_tokens: int,- tokens_saved: int,- transforms_applied: list[str],- tags: dict[str, str],- optimization_latency: float,- memory_user_id: str | None = None,- pipeline_timing: dict[str, float] | None = None,- prefix_tracker: Any | None = None,- ) -> StreamingResponse:- """Stream response with metrics tracking and memory tool handling.-- Parses SSE events to extract actual usage information from the API response- for accurate token counting and cost calculation.-- When memory is enabled (memory_user_id provided), this method:- 1. Buffers the response to detect memory tool calls- 2. Executes memory tools if found- 3. Makes continuation requests until no memory tools remain- 4. Streams the final response to the client- """- start_time = time.time()-- # Mutable state for the generator to update- stream_state: dict[str, Any] = {- "input_tokens": None,- "output_tokens": None,- "cache_read_input_tokens": 0,- "cache_creation_input_tokens": 0,- "total_bytes": 0,- "sse_buffer": "", # Buffer for incomplete SSE events- "ttfb_ms": None, # Time to first byte from upstream- }-- # Track if we need to handle memory tools- memory_enabled = (- memory_user_id is not None- and self.memory_handler is not None- and provider == "anthropic"- )-- # Open connection before generator to capture upstream response headers- # (needed to forward ratelimit headers to the client via StreamingResponse)- assert self.http_client is not None, "http_client must be initialized before streaming"- try:- _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)- upstream_response = await self.http_client.send(_upstream_req, stream=True)- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:- error_msg = str(e)- logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")-- async def _error_gen():- error_event = {- "type": "error",- "error": {- "type": "connection_error",- "message": f"Failed to connect to upstream API: {error_msg}",- },- }- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()-- return StreamingResponse(_error_gen(), media_type="text/event-stream")-- # Forward upstream ratelimit headers to the client- forwarded_headers = {- k: v for k, v in upstream_response.headers.items() if "ratelimit" in k.lower()- }-- async def generate():- nonlocal body, memory_enabled # May need to modify for continuation requests-- # For memory mode, we buffer the response to check for tool calls- buffered_chunks: list[bytes] = []- full_sse_data = ""-- try:- async with contextlib.aclosing(upstream_response) as response:- async for chunk in response.aiter_bytes():- # Record TTFB on first chunk- if stream_state["ttfb_ms"] is None:- stream_state["ttfb_ms"] = (time.time() - start_time) * 1000-- stream_state["total_bytes"] += len(chunk)-- # Buffer SSE data to handle chunks split across calls- chunk_str = chunk.decode("utf-8", errors="ignore")- stream_state["sse_buffer"] += chunk_str-- # Safety: prevent unbounded buffer growth- if len(stream_state["sse_buffer"]) > MAX_SSE_BUFFER_SIZE:- logger.error(- "SSE buffer exceeded maximum size (%d bytes), "- "truncating to prevent memory exhaustion",- MAX_SSE_BUFFER_SIZE,- )- stream_state["sse_buffer"] = stream_state["sse_buffer"][- -MAX_SSE_BUFFER_SIZE // 2 :- ]-- # Always stream immediately — buffering breaks- # real-time clients (LangGraph, LangChain, etc.)- yield chunk-- if memory_enabled:- # Also buffer for post-stream memory processing- buffered_chunks.append(chunk)- full_sse_data += chunk_str- if len(full_sse_data) > MAX_SSE_BUFFER_SIZE:- logger.warning(- "Memory-mode SSE buffer exceeded maximum size, "- "disabling memory detection for this request"- )- memory_enabled = False-- # Parse complete SSE events from buffer- usage = self._parse_sse_usage_from_buffer(stream_state, provider)- if usage:- if "input_tokens" in usage:- stream_state["input_tokens"] = usage["input_tokens"]- if "output_tokens" in usage:- stream_state["output_tokens"] = usage["output_tokens"]- if "cache_read_input_tokens" in usage:- stream_state["cache_read_input_tokens"] = usage[- "cache_read_input_tokens"- ]- if "cache_creation_input_tokens" in usage:- stream_state["cache_creation_input_tokens"] = usage[- "cache_creation_input_tokens"- ]-- # Memory tool handling after stream completes- # Chunks were already yielded in real-time above, so we only- # do silent background processing here — no yielding.- if memory_enabled and full_sse_data:- # Check for Claude Code credential error- if "only authorized for use with Claude Code" in full_sse_data:- logger.warning(- f"[{request_id}] Memory: Claude Code subscription credentials "- "do not support custom tool injection. Set ANTHROPIC_API_KEY "- "environment variable or use --no-memory-tools flag."- )- return-- # Parse SSE to get response JSON- parsed_response = self._parse_sse_to_response(full_sse_data, provider)-- if parsed_response and self.memory_handler.has_memory_tool_calls(- parsed_response, provider- ):- logger.info(- f"[{request_id}] Memory: Detected tool calls in streaming response"- )-- # Execute memory tool calls silently — response already- # streamed so we cannot make a continuation request.- tool_results = await self.memory_handler.handle_memory_tool_calls(- parsed_response, memory_user_id, provider- )- if tool_results:- logger.info(- f"[{request_id}] Memory: Tool calls executed silently "- "(streaming mode — no continuation)"- )-- # CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.- # In streaming mode, the client handles actual retrieval, but we- # still need to record the event so TOIN learns which fields matter.- if self.config.ccr_inject_tool and full_sse_data:- ccr_parsed = (- parsed_response- if parsed_response- else self._parse_sse_to_response(full_sse_data, provider)- )- if ccr_parsed:- self._record_ccr_feedback_from_response(ccr_parsed, provider, request_id)-- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:- logger.error(f"[{request_id}] Connection error to upstream API: {e}")- error_event = {- "type": "error",- "error": {- "type": "connection_error",- "message": f"Failed to connect to upstream API: {e}",- },- }- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()- except httpx.HTTPStatusError as e:- logger.error(f"[{request_id}] HTTP error from upstream API: {e}")- # Forward the upstream error response- yield e.response.content- except Exception as e:- logger.error(f"[{request_id}] Unexpected streaming error: {e}")- error_event = {- "type": "error",- "error": {"type": "api_error", "message": str(e)},- }- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()- finally:- # Record metrics after stream completes- total_latency = (time.time() - start_time) * 1000-- # Use actual output tokens from API if available, otherwise estimate- output_tokens = stream_state["output_tokens"]- if output_tokens is None:- # Fallback: estimate from bytes (but this is inaccurate for SSE)- # Use a more conservative estimate - SSE overhead is ~10-20x- output_tokens = stream_state["total_bytes"] // 40- logger.debug(- f"[{request_id}] No usage in stream, estimated {output_tokens} output tokens"- )-- # Use optimized_tokens for dashboard metrics (what we actually sent).- # API's input_tokens is the non-cached portion only, which is- # misleading for aggregation (often just 1 with prompt caching).- cache_read_tokens = stream_state["cache_read_input_tokens"]- cache_write_tokens = stream_state["cache_creation_input_tokens"]- uncached_input_tokens = stream_state.get("input_tokens") or 0-- # Structured perf log line for `headroom perf` analysis- num_msgs = len(body.get("messages", []))- cache_hit_pct = (- round(cache_read_tokens / (cache_read_tokens + cache_write_tokens) * 100)- if (cache_read_tokens + cache_write_tokens) > 0- else 0- )- logger.info(- f"[{request_id}] PERF "- f"model={model} msgs={num_msgs} "- f"tok_before={original_tokens} tok_after={optimized_tokens} "- f"tok_saved={tokens_saved} "- f"cache_read={cache_read_tokens} cache_write={cache_write_tokens} "- f"cache_hit_pct={cache_hit_pct} "- f"opt_ms={optimization_latency:.0f} "- f"transforms={_summarize_transforms(transforms_applied)}"- )-- # Update prefix cache tracker for next turn (streaming path)- if prefix_tracker is not None:- prefix_tracker.update_from_response(- cache_read_tokens=cache_read_tokens,- cache_write_tokens=cache_write_tokens,- messages=body.get("messages", []),- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(- model,- tokens_saved,- optimized_tokens,- cache_read_tokens=cache_read_tokens,- cache_write_tokens=cache_write_tokens,- uncached_tokens=uncached_input_tokens,- )-- await self.metrics.record_request(- provider=provider,- model=model,- input_tokens=optimized_tokens, # What we sent, not API's non-cached count- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- ttfb_ms=stream_state["ttfb_ms"] or 0,- pipeline_timing=pipeline_timing,- cache_read_tokens=cache_read_tokens,- cache_write_tokens=cache_write_tokens,- uncached_input_tokens=uncached_input_tokens,- )-- return StreamingResponse(- generate(),- media_type="text/event-stream",- headers=forwarded_headers,- )-- async def _stream_response_bedrock(- self,- body: dict,- headers: dict,- provider: str,- model: str,- request_id: str,- original_tokens: int,- optimized_tokens: int,- tokens_saved: int,- transforms_applied: list[str],- tags: dict[str, str],- optimization_latency: float,- pipeline_timing: dict[str, float] | None = None,- ) -> StreamingResponse:- """Stream response from Bedrock backend with metrics tracking.-- Translates Bedrock streaming events to Anthropic SSE format.- """- start_time = time.time()-- # Mutable state for the generator- stream_state: dict[str, Any] = {- "input_tokens": 0,- "output_tokens": 0,- "ttfb_ms": None,- }-- async def generate():- try:- assert self.anthropic_backend is not None-- async for event in self.anthropic_backend.stream_message(body, headers):- # Record TTFB on first event- if stream_state["ttfb_ms"] is None:- stream_state["ttfb_ms"] = (time.time() - start_time) * 1000-- # Format as SSE- if event.raw_sse:- yield event.raw_sse.encode()- else:- sse_line = f"event: {event.event_type}\ndata: {json.dumps(event.data)}\n\n"- yield sse_line.encode()-- # Track usage from message_start event- if event.event_type == "message_start":- msg = event.data.get("message", {})- usage = msg.get("usage", {})- if "input_tokens" in usage:- stream_state["input_tokens"] = usage["input_tokens"]-- # Track output tokens from message_delta- if event.event_type == "message_delta":- usage = event.data.get("usage", {})- if "output_tokens" in usage:- stream_state["output_tokens"] = usage["output_tokens"]-- # Handle errors- if event.event_type == "error":- logger.error(f"[{request_id}] Bedrock stream error: {event.data}")-- except Exception as e:- logger.error(f"[{request_id}] Bedrock streaming error: {e}")- error_event = {- "type": "error",- "error": {"type": "api_error", "message": str(e)},- }- yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()-- finally:- # Record metrics- total_latency = (time.time() - start_time) * 1000- output_tokens = stream_state["output_tokens"]-- _backend_name = (- self.anthropic_backend.name if self.anthropic_backend else "anthropic"- )- await self.metrics.record_request(- provider=_backend_name,- model=model,- input_tokens=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- cached=False,- overhead_ms=optimization_latency,- ttfb_ms=stream_state["ttfb_ms"] or 0,- pipeline_timing=pipeline_timing,- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(model, tokens_saved, optimized_tokens)-- # Log request- if self.logger:- self.logger.log(- RequestLog(- request_id=request_id,- timestamp=datetime.now().isoformat(),- provider=_backend_name,- model=model,- input_tokens_original=original_tokens,- input_tokens_optimized=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- savings_percent=(tokens_saved / original_tokens * 100)- if original_tokens > 0- else 0,- optimization_latency_ms=optimization_latency,- total_latency_ms=total_latency,- tags=tags,- cache_hit=False,- transforms_applied=transforms_applied,- request_messages=body.get("messages")- if self.config.log_full_messages- else None,- )- )-- # Structured perf log line for `headroom perf` analysis- num_msgs = len(body.get("messages", []))- logger.info(- f"[{request_id}] PERF "- f"model={model} msgs={num_msgs} "- f"tok_before={original_tokens} tok_after={optimized_tokens} "- f"tok_saved={tokens_saved} "- f"cache_read=0 cache_write=0 cache_hit_pct=0 "- f"opt_ms={optimization_latency:.0f} "- f"transforms={_summarize_transforms(transforms_applied)}"- )-- return StreamingResponse(- generate(),- media_type="text/event-stream",- )-- async def _stream_openai_via_backend(- self,- body: dict,- headers: dict,- model: str,- request_id: str,- start_time: float,- original_tokens: int,- optimized_tokens: int,- tokens_saved: int,- transforms_applied: list[str],- tags: dict[str, str],- optimization_latency: float,- pipeline_timing: dict[str, float] | None = None,- ) -> StreamingResponse:- """Stream OpenAI chat completion response from backend.-- Routes stream:true requests through the backend's stream_openai_message(),- yielding SSE events to the client.- """- assert self.anthropic_backend is not None-- async def generate():- try:- async for sse_chunk in self.anthropic_backend.stream_openai_message(body, headers):- yield sse_chunk.encode() if isinstance(sse_chunk, str) else sse_chunk- except Exception as e:- logger.error(f"[{request_id}] Backend streaming error: {e}")- error_data = {- "error": {- "message": str(e),- "type": "api_error",- "code": "backend_error",- }- }- yield f"data: {json.dumps(error_data)}\n\n".encode()- yield b"data: [DONE]\n\n"- finally:- total_latency = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider=self.anthropic_backend.name,- model=model,- input_tokens=optimized_tokens,- output_tokens=0, # Unknown in streaming- tokens_saved=tokens_saved,- latency_ms=total_latency,- cached=False,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- )- if tokens_saved > 0:- logger.info(- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name} [stream]"- )-- return StreamingResponse(- generate(),- media_type="text/event-stream",- )-- async def handle_openai_chat(- self,- request: Request,- ) -> Response | StreamingResponse:- """Handle OpenAI /v1/chat/completions endpoint."""- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "error": {- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "type": "invalid_request_error",- "code": "request_too_large",- }- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "type": "invalid_request_error",- "code": "invalid_json",- }- },- )- model = body.get("model", "unknown")- messages = body.get("messages", [])-- # Validate message array size- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Message array too large ({len(messages)} messages). "- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",- "type": "invalid_request_error",- "code": "invalid_request",- }- },- )-- stream = body.get("stream", False)-- # Bypass: skip ALL compression for explicit opt-out- _bypass = (- request.headers.get("x-headroom-bypass", "").lower() == "true"- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"- )- if _bypass:- logger.info(f"[{request_id}] Bypass: skipping compression (header)")-- # Image compression (before text optimization)- if self.config.image_optimize and messages and not _bypass:- compressor = _get_image_compressor()- if compressor and compressor.has_images(messages):- messages = compressor.compress(messages, provider="openai")- if compressor.last_result:- logger.info(- f"Image compression: {compressor.last_result.technique.value} "- f"({compressor.last_result.savings_percent:.0f}% saved, "- f"{compressor.last_result.original_tokens} -> "- f"{compressor.last_result.compressed_tokens} tokens)"- )-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- tags = self._extract_tags(headers)-- # Rate limiting- if self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited()- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",- )-- # Check cache- if self.cache and not stream:- cached = await self.cache.get(messages, model)- if cached:- await self.metrics.record_request(- provider="openai",- model=model,- input_tokens=0,- output_tokens=0,- tokens_saved=0, # Savings already counted when response was cached- latency_ms=(time.time() - start_time) * 1000,- cached=True,- )-- # Remove compression headers from cached response- response_headers = dict(cached.response_headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(content=cached.response_body, headers=response_headers)-- # Token counting- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Hook: pre_compress- _hook_biases = None- if self.config.hooks:- from headroom.hooks import CompressContext-- _hook_ctx = CompressContext(model=model, provider="openai")- try:- messages = self.config.hooks.pre_compress(messages, _hook_ctx)- _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)- except Exception as e:- logger.debug(f"[{request_id}] Hook error: {e}")-- # Optimization- transforms_applied = []- pipeline_timing: dict[str, float] = {}- waste_signals_dict: dict[str, int] | None = None- optimized_messages = messages- optimized_tokens = original_tokens-- # Get prefix cache tracker for this session- openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)- openai_prefix_tracker = self.session_tracker_store.get_or_create(- openai_session_id, "openai"- )- openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()-- _compression_failed = False- original_messages = messages # Preserve for 400-retry fallback- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True- if self.config.optimize and messages and not _bypass and _license_ok:- try:- context_limit = self.openai_provider.get_context_limit(model)-- if self.config.mode == "token_headroom":- comp_cache = self._get_compression_cache(openai_session_id)-- # Zone 1: Swap cached compressed versions- working_messages = comp_cache.apply_cached(messages)-- # Re-freeze boundary- openai_frozen_count = comp_cache.compute_frozen_count(messages)-- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=working_messages,- model=model,- model_limit=context_limit,- context=extract_user_query(working_messages),- frozen_message_count=openai_frozen_count,- biases=_hook_biases,- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != working_messages:- comp_cache.update_from_result(messages, result.messages)-- # Always use pipeline result in token_headroom mode- optimized_messages = result.messages- transforms_applied = result.transforms_applied- pipeline_timing = result.timing- # Keep original_tokens as the REAL original (pre-Zone-1-swap)- # so tokens_saved captures both Zone 1 + Zone 2 savings.- optimized_tokens = result.tokens_after- else:- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- frozen_message_count=openai_frozen_count,- biases=_hook_biases,- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- pipeline_timing = result.timing- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after-- if result.waste_signals:- waste_signals_dict = result.waste_signals.to_dict()- except Exception as e:- logger.warning(f"Optimization failed: {e}")- # Flag compression failure for observability- _compression_failed = True-- tokens_saved = max(0, original_tokens - optimized_tokens)- optimization_latency = (time.time() - start_time) * 1000-- # Hook: post_compress- if self.config.hooks and tokens_saved > 0:- from headroom.hooks import CompressEvent-- try:- self.config.hooks.post_compress(- CompressEvent(- tokens_before=original_tokens,- tokens_after=optimized_tokens,- tokens_saved=tokens_saved,- compression_ratio=tokens_saved / original_tokens- if original_tokens > 0- else 0,- transforms_applied=transforms_applied,- model=model,- provider="openai",- )- )- except Exception as e:- logger.debug(f"[{request_id}] post_compress hook error: {e}")-- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = body.get("tools")- _original_tools = tools # Preserve for diagnostic / future retry- if (- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions- ) and not _bypass:- injector = CCRToolInjector(- provider="openai",- inject_tool=self.config.ccr_inject_tool,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- optimized_messages, tools, was_injected = injector.process_request(- optimized_messages, tools- )-- if injector.has_compressed_content:- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"- )- else:- logger.debug(- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"- )-- # Query Echo: disabled — hurts prefix caching in long conversations.-- body["messages"] = optimized_messages- if tools is not None:- body["tools"] = tools-- # Route through LiteLLM/any-llm backend if configured- if self.anthropic_backend is not None:- try:- if stream:- # Streaming: use stream_openai_message() → SSE events- return await self._stream_openai_via_backend(- body,- headers,- model,- request_id,- start_time,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- pipeline_timing=pipeline_timing,- )- else:- # Non-streaming: use send_openai_message() → JSON- backend_response = await self.anthropic_backend.send_openai_message(- body, headers- )-- if backend_response.error:- return JSONResponse(- status_code=backend_response.status_code,- content=backend_response.body,- )-- # Track metrics- total_latency = (time.time() - start_time) * 1000- usage = backend_response.body.get("usage", {})- output_tokens = usage.get("completion_tokens", 0)- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)-- await self.metrics.record_request(- provider=self.anthropic_backend.name,- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- cached=False,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- )-- if tokens_saved > 0:- logger.info(- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"- )-- return JSONResponse(- status_code=backend_response.status_code,- content=backend_response.body,- )- except Exception as e:- logger.error(f"[{request_id}] Backend error: {e}")- return JSONResponse(- status_code=500,- content={- "error": {- "message": str(e),- "type": "api_error",- "code": "backend_error",- }- },- )-- # Direct OpenAI API (no backend configured)- url = f"{self.OPENAI_API_URL}/v1/chat/completions"-- try:- if stream:- # Inject stream_options to get usage stats in streaming response- # This allows accurate token counting instead of byte-based estimation- if "stream_options" not in body:- body["stream_options"] = {"include_usage": True}- elif isinstance(body.get("stream_options"), dict):- body["stream_options"]["include_usage"] = True-- return await self._stream_response(- url,- headers,- body,- "openai",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- pipeline_timing=pipeline_timing,- prefix_tracker=openai_prefix_tracker,- )- else:- response = await self._retry_request("POST", url, headers, body)-- # Full diagnostic dump on upstream errors (OpenAI handler)- if response.status_code >= 400:- try:- err_body = response.json()- err_msg = err_body.get("error", {}).get("message", "")- err_type = err_body.get("error", {}).get("type", "")- except Exception:- err_body = {"raw": response.text[:2000]}- err_msg = str(response.text[:500])- err_type = "parse_error"-- logger.warning(- f"[{request_id}] UPSTREAM_ERROR "- f"status={response.status_code} "- f"error_type={err_type} "- f"error_msg={err_msg!r} "- f"model={model} "- f"compressed={'yes' if transforms_applied else 'no'} "- f"transforms={transforms_applied} "- f"original_tokens={original_tokens} "- f"optimized_tokens={optimized_tokens} "- f"message_count={len(body.get('messages', []))} "- f"stream={stream}"- )-- try:- debug_dir = Path.home() / ".headroom" / "logs" / "debug_400"- debug_dir.mkdir(parents=True, exist_ok=True)- ts = datetime.now().strftime("%Y%m%d_%H%M%S")- debug_file = debug_dir / f"{ts}_{request_id}.json"-- safe_headers = {}- for k, v in headers.items():- if k.lower() in ("x-api-key", "authorization"):- safe_headers[k] = v[:12] + "..." if v else ""- else:- safe_headers[k] = v-- debug_payload = {- "request_id": request_id,- "timestamp": datetime.now().isoformat(),- "status_code": response.status_code,- "error_response": err_body,- "model": model,- "stream": stream,- "headers": safe_headers,- "compression": {- "was_compressed": bool(transforms_applied),- "transforms": transforms_applied,- "original_tokens": original_tokens,- "optimized_tokens": optimized_tokens,- "tokens_saved": tokens_saved,- "compression_failed": _compression_failed,- },- "tools_sent": body.get("tools"),- "tool_count": len(body.get("tools") or []),- "original_tool_count": len(_original_tools or []),- "messages_sent": body.get("messages"),- "message_count": len(body.get("messages", [])),- "original_messages": (- original_messages- if original_messages is not body.get("messages")- else "__same_as_sent__"- ),- "original_message_count": len(original_messages),- "system_prompt": body.get("system"),- }-- with open(debug_file, "w") as f:- json.dump(debug_payload, f, indent=2, default=str)-- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")- except Exception as dump_err:- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")-- total_latency = (time.time() - start_time) * 1000-- total_input_tokens = optimized_tokens # fallback- output_tokens = 0- cache_read_tokens = 0- try:- resp_json = response.json()- usage = resp_json.get("usage", {})- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)- output_tokens = usage.get("completion_tokens", 0)- # OpenAI returns cached_tokens in prompt_tokens_details- # These are charged at 50% of the input price- prompt_details = usage.get("prompt_tokens_details", {})- cache_read_tokens = prompt_details.get("cached_tokens", 0)- except (KeyError, TypeError, AttributeError) as e:- logger.debug(- f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"- )-- # Update prefix cache tracker for next turn- openai_prefix_tracker.update_from_response(- cache_read_tokens=cache_read_tokens,- cache_write_tokens=0, # OpenAI doesn't report write tokens- messages=optimized_messages,- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(- model,- tokens_saved,- optimized_tokens,- cache_read_tokens=cache_read_tokens,- )-- # Cache- if self.cache and response.status_code == 200:- await self.cache.set(- messages, model, response.content, dict(response.headers), tokens_saved- )-- await self.metrics.record_request(- provider="openai",- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- pipeline_timing=pipeline_timing,- waste_signals=waste_signals_dict,- cache_read_tokens=cache_read_tokens,- )-- if tokens_saved > 0:- logger.info(- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "- f"(saved {tokens_saved:,} tokens)"- )-- # Remove compression headers since httpx already decompressed the response- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression-- # Inject Headroom compression metrics (for SaaS metering)- response_headers["x-headroom-tokens-before"] = str(original_tokens)- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)- response_headers["x-headroom-model"] = model- if transforms_applied:- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)- if cache_read_tokens > 0:- response_headers["x-headroom-cached"] = "true"- if _compression_failed:- response_headers["x-headroom-compression-failed"] = "true"-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )- except Exception as e:- await self.metrics.record_failed()- # Log full error details internally for debugging- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")- # Return sanitized error message to client (don't expose internal details)- return JSONResponse(- status_code=502,- content={- "error": {- "message": "An error occurred while processing your request. Please try again.",- "type": "server_error",- "code": "proxy_error",- }- },- )-- async def handle_passthrough(- self,- request: Request,- base_url: str,- endpoint_name: str | None = None,- provider: str | None = None,- ) -> Response:- """Pass through request unchanged.-- Args:- request: The incoming request- base_url: The upstream API base URL- endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")- provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")- """- start_time = time.time()- path = request.url.path- url = f"{base_url}{path}"-- # Preserve query string parameters- if request.url.query:- url = f"{url}?{request.url.query}"-- headers = dict(request.headers.items())- headers.pop("host", None)-- body = await request.body()-- response = await self.http_client.request( # type: ignore[union-attr]- method=request.method,- url=url,- headers=headers,- content=body,- )-- # Remove compression headers since httpx already decompressed the response- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression-- # Track stats for passthrough requests- if endpoint_name and provider:- latency_ms = (time.time() - start_time) * 1000- await self.metrics.record_request(- provider=provider,- model=f"passthrough:{endpoint_name}",- input_tokens=0,- output_tokens=0,- tokens_saved=0,- latency_ms=latency_ms,- cached=False,- )-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # =========================================================================- # Databricks Native API- # =========================================================================-- async def handle_databricks_invocations(- self,- request: Request,- model: str,- ) -> Response | StreamingResponse:- """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.-- This enables using the Databricks CLI directly with Headroom:- databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'-- The request/response format is identical to OpenAI chat completions,- so we inject the model from the path and delegate to handle_openai_chat.- """- request_id = await self._next_request_id()-- try:- body = await _read_request_json(request)- except Exception as e:- logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")- return JSONResponse(- status_code=400,- content={- "error": {"message": f"Invalid JSON: {e}", "type": "invalid_request_error"}- },- )-- # Inject model from path into body (Databricks CLI passes model in URL, not body)- body["model"] = model-- logger.info(f"[{request_id}] Databricks invocation: model={model}")-- # Create a new request with the modified body- # We reuse the OpenAI chat handler since the format is identical- from starlette.requests import Request as StarletteRequest-- # Build new scope with the body already parsed- scope = dict(request.scope)-- # Create a simple receive function that returns our modified body- body_bytes = json.dumps(body).encode()-- async def receive():- return {"type": "http.request", "body": body_bytes}-- modified_request = StarletteRequest(scope, receive)-- # Delegate to the OpenAI chat handler (same format)- return await self.handle_openai_chat(modified_request)-- # =========================================================================- # OpenAI Batch API with Compression- # =========================================================================-- async def handle_batch_create(self, request: Request) -> Response:- """Handle POST /v1/batches - Create a batch with compression.-- Flow:- 1. Parse request to get input_file_id- 2. Download the JSONL file content from OpenAI- 3. Parse each line and compress the messages- 4. Create a new compressed JSONL file- 5. Upload compressed file to OpenAI- 6. Create batch with the new compressed file_id- 7. Return batch object with compression stats in metadata- """- start_time = time.time()- request_id = await self._next_request_id()-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "type": "invalid_request_error",- "code": "invalid_json",- }- },- )-- input_file_id = body.get("input_file_id")- endpoint = body.get("endpoint")- completion_window = body.get("completion_window", "24h")- metadata = body.get("metadata", {})-- if not input_file_id:- return JSONResponse(- status_code=400,- content={- "error": {- "message": "input_file_id is required",- "type": "invalid_request_error",- "code": "missing_parameter",- }- },- )-- if not endpoint:- return JSONResponse(- status_code=400,- content={- "error": {- "message": "endpoint is required",- "type": "invalid_request_error",- "code": "missing_parameter",- }- },- )-- # Only compress chat completions endpoint- if endpoint != "/v1/chat/completions":- # Pass through for other endpoints- return await self._batch_passthrough(request, body)-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)-- try:- # Step 1: Download the input file from OpenAI- logger.info(f"[{request_id}] Batch: Downloading input file {input_file_id}")- file_content = await self._download_openai_file(input_file_id, headers)-- if file_content is None:- return JSONResponse(- status_code=404,- content={- "error": {- "message": f"Failed to download file {input_file_id}",- "type": "invalid_request_error",- "code": "file_not_found",- }- },- )-- # Step 2: Parse and compress each line- logger.info(f"[{request_id}] Batch: Compressing JSONL content")- compressed_lines, stats = await self._compress_batch_jsonl(file_content, request_id)-- if stats["total_requests"] == 0:- return JSONResponse(- status_code=400,- content={- "error": {- "message": "No valid requests found in input file",- "type": "invalid_request_error",- "code": "empty_file",- }- },- )-- # Step 3: Create compressed JSONL content- compressed_content = "\n".join(compressed_lines)-- # Step 4: Upload compressed file to OpenAI- logger.info(f"[{request_id}] Batch: Uploading compressed file")- new_file_id = await self._upload_openai_file(- compressed_content, f"compressed_{input_file_id}.jsonl", headers- )-- if new_file_id is None:- return JSONResponse(- status_code=500,- content={- "error": {- "message": "Failed to upload compressed file",- "type": "server_error",- "code": "upload_failed",- }- },- )-- # Step 5: Create batch with compressed file- logger.info(f"[{request_id}] Batch: Creating batch with compressed file {new_file_id}")-- # Add compression stats to metadata- compression_metadata = {- **metadata,- "headroom_compressed": "true",- "headroom_original_file_id": input_file_id,- "headroom_total_requests": str(stats["total_requests"]),- "headroom_tokens_saved": str(stats["total_tokens_saved"]),- "headroom_original_tokens": str(stats["total_original_tokens"]),- "headroom_compressed_tokens": str(stats["total_compressed_tokens"]),- "headroom_savings_percent": f"{stats['savings_percent']:.1f}",- }-- batch_body = {- "input_file_id": new_file_id,- "endpoint": endpoint,- "completion_window": completion_window,- "metadata": compression_metadata,- }-- url = f"{self.OPENAI_API_URL}/v1/batches"- response = await self.http_client.post(url, json=batch_body, headers=headers) # type: ignore[union-attr]-- total_latency = (time.time() - start_time) * 1000-- # Log compression stats- logger.info(- f"[{request_id}] Batch created: {stats['total_requests']} requests, "- f"{stats['total_original_tokens']:,} -> {stats['total_compressed_tokens']:,} tokens "- f"(saved {stats['total_tokens_saved']:,} tokens, {stats['savings_percent']:.1f}%) "- f"in {total_latency:.0f}ms"- )-- # Record metrics- await self.metrics.record_request(- provider="openai",- model="batch",- input_tokens=stats["total_compressed_tokens"],- output_tokens=0,- tokens_saved=stats["total_tokens_saved"],- latency_ms=total_latency,- )-- # Return response with compression info in headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- response_headers["x-headroom-tokens-saved"] = str(stats["total_tokens_saved"])- response_headers["x-headroom-savings-percent"] = f"{stats['savings_percent']:.1f}"-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- except Exception as e:- logger.error(f"[{request_id}] Batch creation failed: {type(e).__name__}: {e}")- await self.metrics.record_failed()- return JSONResponse(- status_code=500,- content={- "error": {- "message": "An error occurred while processing the batch request",- "type": "server_error",- "code": "batch_processing_error",- }- },- )-- async def _download_openai_file(self, file_id: str, headers: dict) -> str | None:- """Download file content from OpenAI."""- url = f"{self.OPENAI_API_URL}/v1/files/{file_id}/content"- try:- response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]- if response.status_code == 200:- return str(response.text)- logger.error(f"Failed to download file {file_id}: {response.status_code}")- return None- except Exception as e:- logger.error(f"Error downloading file {file_id}: {e}")- return None-- async def _upload_openai_file(self, content: str, filename: str, headers: dict) -> str | None:- """Upload a file to OpenAI for batch processing."""- url = f"{self.OPENAI_API_URL}/v1/files"-- # Prepare multipart form data- # We need to use httpx's files parameter for multipart upload- files = {- "file": (filename, content.encode("utf-8"), "application/jsonl"),- }- data = {- "purpose": "batch",- }-- # Remove content-type from headers (httpx will set it for multipart)- upload_headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}-- try:- response = await self.http_client.post( # type: ignore[union-attr]- url, files=files, data=data, headers=upload_headers- )- if response.status_code == 200:- result = response.json()- file_id: str | None = result.get("id")- return file_id- logger.error(f"Failed to upload file: {response.status_code} - {response.text}")- return None- except Exception as e:- logger.error(f"Error uploading file: {e}")- return None-- async def _compress_batch_jsonl(self, content: str, request_id: str) -> tuple[list[str], dict]:- """Compress messages in each line of a batch JSONL file.-- Returns:- Tuple of (compressed_lines, stats_dict)- """- lines = content.strip().split("\n")- compressed_lines = []- total_original_tokens = 0- total_compressed_tokens = 0- total_requests = 0- errors = 0-- tokenizer = get_tokenizer("gpt-4") # Use gpt-4 tokenizer for batch-- for i, line in enumerate(lines):- if not line.strip():- continue-- try:- request_obj = json.loads(line)- body = request_obj.get("body", {})- messages = body.get("messages", [])- model = body.get("model", "gpt-4")-- if not messages:- # No messages to compress, pass through- compressed_lines.append(line)- total_requests += 1- continue-- # Compress messages using the OpenAI pipeline- if self.config.optimize:- try:- context_limit = self.openai_provider.get_context_limit(model)- result = self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- compressed_messages = result.messages- # Use pipeline's token counts for consistency with pipeline logs- original_tokens = result.tokens_before- compressed_tokens = result.tokens_after- except Exception as e:- logger.warning(f"[{request_id}] Compression failed for line {i}: {e}")- compressed_messages = messages- original_tokens = tokenizer.count_messages(messages)- compressed_tokens = original_tokens- else:- compressed_messages = messages- original_tokens = tokenizer.count_messages(messages)- compressed_tokens = original_tokens-- total_original_tokens += original_tokens- total_compressed_tokens += compressed_tokens- tokens_saved = original_tokens - compressed_tokens-- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = body.get("tools")- if self.config.ccr_inject_tool and tokens_saved > 0:- injector = CCRToolInjector(- provider="openai",- inject_tool=True,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- compressed_messages, tools, was_injected = injector.process_request(- compressed_messages, tools- )- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for batch line {i}"- )-- # Update body with compressed messages- body["messages"] = compressed_messages- if tools is not None:- body["tools"] = tools- request_obj["body"] = body-- compressed_lines.append(json.dumps(request_obj))- total_requests += 1-- except json.JSONDecodeError as e:- logger.warning(f"[{request_id}] Invalid JSON on line {i}: {e}")- errors += 1- # Keep original line on error- compressed_lines.append(line)- total_requests += 1-- total_tokens_saved = total_original_tokens - total_compressed_tokens- savings_percent = (- (total_tokens_saved / total_original_tokens * 100) if total_original_tokens > 0 else 0- )-- stats = {- "total_requests": total_requests,- "total_original_tokens": total_original_tokens,- "total_compressed_tokens": total_compressed_tokens,- "total_tokens_saved": total_tokens_saved,- "savings_percent": savings_percent,- "errors": errors,- }-- return compressed_lines, stats-- async def _batch_passthrough(self, request: Request, body: dict) -> Response:- """Pass through batch request to OpenAI without compression."""- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)-- url = f"{self.OPENAI_API_URL}/v1/batches"- response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]-- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- async def handle_batch_list(self, request: Request) -> Response:- """Handle GET /v1/batches - List batches (passthrough)."""- return await self.handle_passthrough(request, self.OPENAI_API_URL)-- async def handle_batch_get(self, request: Request, batch_id: str) -> Response:- """Handle GET /v1/batches/{batch_id} - Get batch (passthrough)."""- return await self.handle_passthrough(request, self.OPENAI_API_URL)-- async def handle_batch_cancel(self, request: Request, batch_id: str) -> Response:- """Handle POST /v1/batches/{batch_id}/cancel - Cancel batch (passthrough)."""- return await self.handle_passthrough(request, self.OPENAI_API_URL)-- def _has_non_text_parts(self, content: dict) -> bool:- """Check if a Gemini content entry has non-text parts.-- Non-text parts include:- - inlineData: Base64-encoded images/media- - fileData: File references (URI + MIME type)- - functionCall: Function calls from model- - functionResponse: Responses to function calls-- Args:- content: A single Gemini content entry with 'parts' list.-- Returns:- True if any part contains non-text data.- """- parts = content.get("parts", [])- for part in parts:- if any(- key in part- for key in ("inlineData", "fileData", "functionCall", "functionResponse")- ):- return True- return False-- def _gemini_contents_to_messages(- self, contents: list[dict], system_instruction: dict | None = None- ) -> tuple[list[dict], set[int]]:- """Convert Gemini contents[] format to OpenAI messages[] format for optimization.-- Gemini format:- contents: [{"role": "user", "parts": [{"text": "..."}]}]- systemInstruction: {"parts": [{"text": "..."}]}-- OpenAI format:- messages: [{"role": "user", "content": "..."}]-- Returns:- Tuple of (messages, preserved_indices) where preserved_indices contains- the indices of content entries that have non-text parts (images, function- calls, etc.) and should not be compressed.- """- messages = []- preserved_indices: set[int] = set()-- # Add system instruction as system message- if system_instruction:- parts = system_instruction.get("parts", [])- text_parts = [p.get("text", "") for p in parts if "text" in p]- if text_parts:- messages.append({"role": "system", "content": "\n".join(text_parts)})-- # Convert contents to messages- for idx, content in enumerate(contents):- # Track content entries with non-text parts- if self._has_non_text_parts(content):- preserved_indices.add(idx)-- role = content.get("role", "user")- # Map Gemini roles to OpenAI roles- if role == "model":- role = "assistant"-- parts = content.get("parts", [])- text_parts = [p.get("text", "") for p in parts if "text" in p]-- if text_parts:- messages.append({"role": role, "content": "\n".join(text_parts)})-- return messages, preserved_indices-- def _messages_to_gemini_contents(self, messages: list[dict]) -> tuple[list[dict], dict | None]:- """Convert OpenAI messages[] format back to Gemini contents[] format.-- Returns:- (contents, system_instruction) tuple- """- contents = []- system_instruction = None-- for msg in messages:- role = msg.get("role", "user")- content = msg.get("content", "")-- if role == "system":- # Extract as systemInstruction- system_instruction = {"parts": [{"text": content}]}- else:- # Map OpenAI roles to Gemini roles- gemini_role = "model" if role == "assistant" else "user"- contents.append({"role": gemini_role, "parts": [{"text": content}]})-- return contents, system_instruction-- async def handle_openai_responses(- self,- request: Request,- ) -> Response | StreamingResponse:- """Handle OpenAI /v1/responses endpoint (new Responses API).-- The Responses API differs from /v1/chat/completions:- - Input: `input` (string or array) instead of `messages`- - System: `instructions` instead of system message- - Output: `output[]` array instead of `choices[].message`- - State: `previous_response_id` for multi-turn- - Built-in tools: web_search, file_search, code_interpreter- """- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "error": {- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "type": "invalid_request_error",- "code": "request_too_large",- }- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "type": "invalid_request_error",- "code": "invalid_json",- }- },- )-- model = body.get("model", "unknown")- stream = body.get("stream", False)-- # Convert Responses API input to messages format for optimization.- # The Responses API uses a different item model (function_call,- # function_call_output, reasoning as top-level items) — we convert to- # Chat Completions messages for the pipeline, then convert back.- from headroom.proxy.responses_converter import (- messages_to_responses_items,- responses_items_to_messages,- )-- input_data = body.get("input", "")- instructions = body.get("instructions")- previous_response_id = body.get("previous_response_id")-- messages: list[dict[str, Any]] = []- original_items: list[dict[str, Any]] | None = None- preserved_indices: list[int] = []-- if instructions:- messages.append({"role": "system", "content": instructions})-- if isinstance(input_data, str):- messages.append({"role": "user", "content": input_data})- elif isinstance(input_data, list):- original_items = input_data- converted, preserved_indices = responses_items_to_messages(input_data)- messages.extend(converted)-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- tags = self._extract_tags(headers)-- # Rate limiting- if self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited()- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",- )-- # Token counting on converted messages- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Optimize: convert items → compress → convert back- tokens_saved = 0- transforms_applied: list[str] = []- optimized_messages = messages- optimized_tokens = original_tokens-- _bypass = (- request.headers.get("x-headroom-bypass", "").lower() == "true"- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"- )- _should_compress = (- self.config.optimize- and original_items is not None- and not previous_response_id- and not _bypass- and len(messages) > 1- )- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True-- if _should_compress and _license_ok:- try:- context_limit = self.openai_provider.get_context_limit(model)- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after- except Exception as e:- logger.warning(f"[{request_id}] Responses API optimization failed: {e}")-- tokens_saved = max(0, original_tokens - optimized_tokens)- optimization_latency = (time.time() - start_time) * 1000-- # Convert compressed messages back to Responses API items- if optimized_messages is not messages and original_items is not None:- opt_msgs = optimized_messages- # Strip system message (instructions) — it's separate in Responses API- if instructions and opt_msgs and opt_msgs[0].get("role") == "system":- body["instructions"] = opt_msgs[0]["content"]- opt_msgs = opt_msgs[1:]-- body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)-- # /v1/responses is OpenAI-specific (Codex) — always routes direct.- # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.- if self.anthropic_backend is not None:- logger.debug(- f"[{request_id}] /v1/responses always routes to OpenAI direct "- f"(backend '{self.anthropic_backend.name}' not used for Responses API)"- )-- url = f"{self.OPENAI_API_URL}/v1/responses"-- try:- if stream:- # Streaming for Responses API uses semantic events- return await self._stream_response(- url,- headers,- body,- "openai",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- )- else:- response = await self._retry_request("POST", url, headers, body)- total_latency = (time.time() - start_time) * 1000-- total_input_tokens = original_tokens # fallback- output_tokens = 0- try:- resp_json = response.json()- usage = resp_json.get("usage", {})- total_input_tokens = usage.get("input_tokens", original_tokens)- output_tokens = usage.get("output_tokens", 0)- except (KeyError, TypeError, AttributeError) as e:- logger.debug(- f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)-- await self.metrics.record_request(- provider="openai",- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- )-- logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )- except Exception as e:- await self.metrics.record_failed()- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")- return JSONResponse(- status_code=502,- content={- "error": {- "message": "An error occurred while processing your request. Please try again.",- "type": "server_error",- "code": "proxy_error",- }- },- )-- async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:- """WebSocket proxy for /v1/responses (Codex gpt-5.4+).-- Newer Codex versions use WebSocket instead of HTTP POST for the- Responses API. This handler:- 1. Accepts the client WebSocket- 2. Receives the first message (``response.create`` request)- 3. Compresses the ``input`` array using the existing pipeline- 4. Opens an upstream WebSocket to OpenAI- 5. Sends the compressed request upstream- 6. Relays all subsequent messages bidirectionally- """- try:- import websockets- except ImportError:- await websocket.accept()- await websocket.close(- code=1011,- reason="websockets package not installed. pip install websockets",- )- return-- request_id = await self._next_request_id()-- # Forward client headers to upstream, adding required OpenAI-Beta header- ws_headers = dict(websocket.headers)-- # Extract subprotocol from client — this is an application-level negotiation- # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).- # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.- client_subprotocols: list[str] = []- raw_protocol = ws_headers.get("sec-websocket-protocol", "")- if raw_protocol:- client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]-- # Accept client connection with the requested subprotocol- if client_subprotocols:- await websocket.accept(subprotocol=client_subprotocols[0])- else:- await websocket.accept()-- # Build upstream WebSocket URL (http→ws, https→wss)- base = self.OPENAI_API_URL- ws_base = base.replace("https://", "wss://").replace("http://", "ws://")- upstream_url = f"{ws_base}/v1/responses"-- # Forward all client headers except hop-by-hop / per-connection headers.- # These are WebSocket handshake mechanics that the `websockets` library- # generates fresh for the upstream connection — forwarding them would conflict.- # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.- _skip_headers = frozenset(- {- "host", # must match upstream, not local proxy- "connection", # hop-by-hop- "upgrade", # hop-by-hop- "sec-websocket-key", # per-connection cryptographic nonce- "sec-websocket-version", # protocol version (websockets lib sets this)- "sec-websocket-extensions", # per-connection negotiation- "sec-websocket-accept", # server-side only- "sec-websocket-protocol", # handled via subprotocols param below- "content-length", # hop-by-hop- "transfer-encoding", # hop-by-hop- }- )- upstream_headers: dict[str, str] = {}- for k, v in ws_headers.items():- if k.lower() not in _skip_headers:- upstream_headers[k] = v-- # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.- # Safety net for clients that don't forward auth headers via WebSocket upgrade.- _has_auth = "authorization" in {k.lower() for k in upstream_headers}- if not _has_auth:- api_key = os.environ.get("OPENAI_API_KEY")- if api_key:- upstream_headers["Authorization"] = f"Bearer {api_key}"- logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")- else:- logger.warning(- f"[{request_id}] WS: no Authorization header from client and "- f"OPENAI_API_KEY not set — upstream will likely reject"- )-- # Ensure the required beta header is present — OpenAI returns 500 without it- if "openai-beta" not in {k.lower() for k in upstream_headers}:- upstream_headers["OpenAI-Beta"] = "responses-api=v1"-- logger.debug(- f"[{request_id}] WS upstream headers: "- f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "- f"subprotocols={client_subprotocols}"- )-- try:- # Receive the first message from client (the response.create request)- first_msg_raw = await websocket.receive_text()-- # --- Optional: compress the input in the first message ---- try:- body = json.loads(first_msg_raw)- input_data = body.get("input")- tokens_saved = 0-- should_compress = (- self.config.optimize- and isinstance(input_data, list)- and len(input_data) > 1- and not body.get("previous_response_id")- )- if should_compress:- try:- from headroom.proxy.responses_converter import (- messages_to_responses_items,- responses_items_to_messages,- )-- model = body.get("model", "gpt-4o")- converted, preserved = responses_items_to_messages(input_data)-- messages: list[dict[str, Any]] = []- instructions = body.get("instructions")- if instructions:- messages.append({"role": "system", "content": instructions})- messages.extend(converted)-- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- context_limit = self.openai_provider.get_context_limit(model)- result = await asyncio.wait_for(- asyncio.to_thread(- lambda: self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- ),- timeout=COMPRESSION_TIMEOUT_SECONDS,- )-- if result.messages != messages:- opt = result.messages- if instructions and opt and opt[0].get("role") == "system":- body["instructions"] = opt[0]["content"]- opt = opt[1:]- body["input"] = messages_to_responses_items(opt, input_data, preserved)- tokens_saved = max(0, original_tokens - result.tokens_after)- first_msg_raw = json.dumps(body)- logger.info(- f"[{request_id}] WS /v1/responses compressed: "- f"saved {tokens_saved} tokens"- )- except Exception as e:- logger.warning(f"[{request_id}] WS compression failed: {e}")-- except json.JSONDecodeError:- # Not JSON — pass through as-is- tokens_saved = 0-- # --- Connect to upstream OpenAI WebSocket ---- logger.debug(f"[{request_id}] WS connecting to {upstream_url}")-- # Use ssl=True to let the websockets library handle SSL natively.- # Manual ssl.create_default_context() + certifi doesn't load the- # Windows system cert store, causing HTTP 500 on wss:// connections.- use_ssl: bool | None = True if upstream_url.startswith("wss://") else None-- async with websockets.connect(- upstream_url,- additional_headers=upstream_headers,- subprotocols=(- [websockets.Subprotocol(p) for p in client_subprotocols]- if client_subprotocols and hasattr(websockets, "Subprotocol")- else client_subprotocols or None- ),- ssl=use_ssl,- ) as upstream:- # Send (potentially compressed) first message- await upstream.send(first_msg_raw)-- # Bidirectional relay- async def _client_to_upstream() -> None:- try:- while True:- msg = await websocket.receive_text()- await upstream.send(msg)- except Exception as relay_err:- if "WebSocketDisconnect" not in type(relay_err).__name__:- logger.debug(- f"[{request_id}] WS client→upstream relay ended: {relay_err}"- )- with contextlib.suppress(Exception):- await upstream.close()-- async def _upstream_to_client() -> None:- try:- async for msg in upstream:- if isinstance(msg, str):- await websocket.send_text(msg)- elif isinstance(msg, bytes):- await websocket.send_bytes(msg)- else:- await websocket.send_text(str(msg))- except Exception as relay_err:- if "WebSocketDisconnect" not in type(relay_err).__name__:- logger.debug(- f"[{request_id}] WS upstream→client relay ended: {relay_err}"- )- finally:- with contextlib.suppress(Exception):- await websocket.close()-- await asyncio.gather(- _client_to_upstream(),- _upstream_to_client(),- return_exceptions=True,- )-- # Record metrics- if tokens_saved > 0:- model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"- await self.metrics.record_request(- provider="openai",- model=model_name,- input_tokens=0,- output_tokens=0,- tokens_saved=tokens_saved,- latency_ms=0,- )-- except Exception as e:- if "WebSocketDisconnect" not in type(e).__name__:- # Extract response body from websockets InvalidStatus for better debugging- error_detail = str(e)- if hasattr(e, "response"):- try:- resp = e.response- body_bytes = getattr(resp, "body", None) or b""- if body_bytes:- error_detail += (- f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"- )- except Exception:- pass- logger.error(f"[{request_id}] WS proxy error: {error_detail}")- with contextlib.suppress(Exception):- await websocket.close(code=1011, reason=str(e)[:120])-- async def handle_gemini_generate_content(- self,- request: Request,- model: str,- ) -> Response | StreamingResponse:- """Handle Gemini native /v1beta/models/{model}:generateContent endpoint.-- Gemini's native API differs from OpenAI:- - Input: `contents[]` with `parts[]` instead of `messages`- - System: `systemInstruction` instead of system message- - Auth: `x-goog-api-key` header instead of `Authorization: Bearer`- - Output: `candidates[].content.parts[].text`- """- start_time = time.time()- request_id = await self._next_request_id()-- # Check request body size- content_length = request.headers.get("content-length")- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:- return JSONResponse(- status_code=413,- content={- "error": {- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "code": 413,- }- },- )-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "code": 400,- }- },- )-- contents = body.get("contents", [])-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- tags = self._extract_tags(headers)-- # Rate limiting (use Gemini API key)- if self.rate_limiter:- rate_key = headers.get("x-goog-api-key", "default")[:20]- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited()- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",- )-- # Convert Gemini format to messages for optimization- system_instruction = body.get("systemInstruction")- messages, preserved_indices = self._gemini_contents_to_messages(- contents, system_instruction- )-- # Store original content entries that have non-text parts before compression- preserved_contents = {idx: contents[idx] for idx in preserved_indices}-- # Early exit if ALL content has non-text parts (nothing to compress)- if len(preserved_indices) == len(contents):- # All content has non-text parts, skip compression entirely- # Just forward the request as-is- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"- query_params = dict(request.query_params)- is_streaming = query_params.get("alt") == "sse"- if "key" in query_params:- url += f"?key={query_params['key']}"-- if is_streaming:- stream_url = (- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"- )- if "key" in query_params:- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"- return await self._stream_response(- stream_url,- headers,- body,- "gemini",- model,- request_id,- 0,- 0,- 0,- [],- tags,- 0,- )- else:- response = await self._retry_request("POST", url, headers, body)- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Token counting- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Optimization- transforms_applied: list[str] = []- waste_signals_dict: dict[str, int] | None = None- optimized_messages = messages- optimized_tokens = original_tokens-- _compression_failed = False- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True- if self.config.optimize and messages and _license_ok:- try:- # Use OpenAI pipeline (similar message format)- context_limit = self.openai_provider.get_context_limit(model)- result = self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- # Use pipeline's token counts for consistency with pipeline logs- original_tokens = result.tokens_before- optimized_tokens = result.tokens_after- if result.waste_signals:- waste_signals_dict = result.waste_signals.to_dict()- except Exception as e:- _compression_failed = True- logger.warning(f"[{request_id}] Gemini optimization failed: {e}")-- tokens_saved = max(0, original_tokens - optimized_tokens)- optimization_latency = (time.time() - start_time) * 1000-- # Query Echo: disabled — hurts prefix caching in long conversations.-- # Convert back to Gemini format if optimized- if optimized_messages != messages:- optimized_contents, optimized_system = self._messages_to_gemini_contents(- optimized_messages- )-- # Restore preserved content entries that had non-text parts- for orig_idx, original_content in preserved_contents.items():- if orig_idx < len(optimized_contents):- optimized_contents[orig_idx] = original_content-- body["contents"] = optimized_contents- if optimized_system:- body["systemInstruction"] = optimized_system- elif "systemInstruction" in body:- del body["systemInstruction"]-- # Build URL - model is extracted from path- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"-- # Check if streaming requested via query param- query_params = dict(request.query_params)- is_streaming = query_params.get("alt") == "sse"-- # Preserve API key in query params if present- if "key" in query_params:- url += f"?key={query_params['key']}"-- try:- if is_streaming:- # For streaming, use streamGenerateContent endpoint- stream_url = (- f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"- )- if "key" in query_params:- stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"-- return await self._stream_response(- stream_url,- headers,- body,- "gemini",- model,- request_id,- original_tokens,- optimized_tokens,- tokens_saved,- transforms_applied,- tags,- optimization_latency,- )- else:- response = await self._retry_request("POST", url, headers, body)- total_latency = (time.time() - start_time) * 1000-- total_input_tokens = optimized_tokens # fallback- output_tokens = 0- cache_read_tokens = 0- try:- resp_json = response.json()- usage = resp_json.get("usageMetadata", {})- total_input_tokens = usage.get("promptTokenCount", optimized_tokens)- output_tokens = usage.get("candidatesTokenCount", 0)- # Gemini returns cachedContentTokenCount for context-cached tokens- # These are charged at 10-25% of the input price depending on model- cache_read_tokens = usage.get("cachedContentTokenCount", 0)- except (KeyError, TypeError, AttributeError) as e:- logger.debug(- f"[{request_id}] Failed to extract cached tokens from Gemini response: {e}"- )-- if self.cost_tracker:- self.cost_tracker.record_tokens(- model,- tokens_saved,- optimized_tokens,- cache_read_tokens=cache_read_tokens,- )-- await self.metrics.record_request(- provider="gemini",- model=model,- input_tokens=total_input_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- overhead_ms=optimization_latency,- waste_signals=waste_signals_dict,- cache_read_tokens=cache_read_tokens,- )-- if tokens_saved > 0:- logger.info(- f"[{request_id}] Gemini {model}: {original_tokens:,} → {optimized_tokens:,} "- f"(saved {tokens_saved:,} tokens)"- )- else:- logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- # Inject Headroom compression metrics (for SaaS metering)- response_headers["x-headroom-tokens-before"] = str(original_tokens)- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)- response_headers["x-headroom-model"] = model- if transforms_applied:- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)- if cache_read_tokens > 0:- response_headers["x-headroom-cached"] = "true"- if _compression_failed:- response_headers["x-headroom-compression-failed"] = "true"-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )- except Exception as e:- await self.metrics.record_failed()- logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")- return JSONResponse(- status_code=502,- content={- "error": {- "message": "An error occurred while processing your request. Please try again.",- "code": 502,- }- },- )-- async def handle_gemini_stream_generate_content(- self,- request: Request,- model: str,- ) -> StreamingResponse | JSONResponse:- """Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""- start_time = time.time()- request_id = await self._next_request_id()-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "code": 400,- }- },- )-- contents = body.get("contents", [])-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)- tags = self._extract_tags(headers)-- # Token counting- tokenizer = get_tokenizer(model)- original_tokens = 0- for content in contents:- parts = content.get("parts", [])- for part in parts:- if "text" in part:- original_tokens += tokenizer.count_text(part["text"])-- optimization_latency = (time.time() - start_time) * 1000-- # Build URL with SSE param- query_params = dict(request.query_params)- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"- if "key" in query_params:- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"-- return await self._stream_response(- url,- headers,- body,- "gemini",- model,- request_id,- original_tokens,- original_tokens,- 0, # tokens_saved- [], # transforms_applied- tags,- optimization_latency,- )-- async def handle_gemini_count_tokens(- self,- request: Request,- model: str,- ) -> Response:- """Handle Gemini /v1beta/models/{model}:countTokens endpoint with compression.-- This endpoint counts tokens AFTER applying compression, so users can see- how many tokens they'll actually use after optimization.-- The request format is the same as generateContent:- {"contents": [...], "systemInstruction": {...}}- """- start_time = time.time()- request_id = await self._next_request_id()-- # Parse request- try:- body = await _read_request_json(request)- except (json.JSONDecodeError, ValueError) as e:- return JSONResponse(- status_code=400,- content={- "error": {- "message": f"Invalid request body: {e!s}",- "code": 400,- }- },- )-- contents = body.get("contents", [])-- headers = dict(request.headers.items())- headers.pop("host", None)- headers.pop("content-length", None)-- # Convert Gemini format to messages for optimization- system_instruction = body.get("systemInstruction")- messages, preserved_indices = self._gemini_contents_to_messages(- contents, system_instruction- )-- # Store original content entries that have non-text parts before compression- preserved_contents = {idx: contents[idx] for idx in preserved_indices}-- # Early exit if ALL content has non-text parts (nothing to compress)- if len(preserved_indices) == len(contents):- # All content has non-text parts, skip compression entirely- # Just forward the countTokens request as-is- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"- query_params = dict(request.query_params)- if "key" in query_params:- url += f"?key={query_params['key']}"-- response = await self._retry_request("POST", url, headers, body)- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )-- # Token counting (original)- tokenizer = get_tokenizer(model)- original_tokens = tokenizer.count_messages(messages)-- # Apply compression using the same pipeline as generateContent- transforms_applied: list[str] = []- optimized_messages = messages-- if self.config.optimize and messages:- try:- context_limit = self.openai_provider.get_context_limit(model)- result = self.openai_pipeline.apply(- messages=messages,- model=model,- model_limit=context_limit,- context=extract_user_query(messages),- )- if result.messages != messages:- optimized_messages = result.messages- transforms_applied = result.transforms_applied- except Exception as e:- logger.warning(f"[{request_id}] Gemini countTokens optimization failed: {e}")-- # Convert back to Gemini format for the API call- if optimized_messages != messages:- optimized_contents, optimized_system = self._messages_to_gemini_contents(- optimized_messages- )-- # Restore preserved content entries that had non-text parts- for orig_idx, original_content in preserved_contents.items():- if orig_idx < len(optimized_contents):- optimized_contents[orig_idx] = original_content-- body["contents"] = optimized_contents- if optimized_system:- body["systemInstruction"] = optimized_system- elif "systemInstruction" in body:- del body["systemInstruction"]-- # Build URL- url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"-- # Preserve API key in query params if present- query_params = dict(request.query_params)- if "key" in query_params:- url += f"?key={query_params['key']}"-- try:- response = await self._retry_request("POST", url, headers, body)- total_latency = (time.time() - start_time) * 1000-- # Parse response to get token count- compressed_tokens = 0- try:- resp_json = response.json()- compressed_tokens = resp_json.get("totalTokens", 0)- except (json.JSONDecodeError, ValueError) as e:- logger.debug(f"[{request_id}] Failed to parse Gemini token count response: {e}")-- # Track stats- tokens_saved = (- max(0, original_tokens - compressed_tokens) if compressed_tokens > 0 else 0- )-- await self.metrics.record_request(- provider="gemini",- model=model,- input_tokens=compressed_tokens,- output_tokens=0,- tokens_saved=tokens_saved,- latency_ms=total_latency,- )-- if tokens_saved > 0:- logger.info(- f"[{request_id}] Gemini countTokens {model}: {original_tokens:,} → {compressed_tokens:,} "- f"(saved {tokens_saved:,} tokens, transforms: {transforms_applied})"- )- else:- logger.info(- f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens"- )-- # Remove compression headers- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)-- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )- except Exception as e:- await self.metrics.record_failed()- logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")- return JSONResponse(- status_code=502,- content={- "error": {- "message": "An error occurred while processing your request. Please try again.",- "code": 502,- }- },- )---# =============================================================================-# FastAPI App-# =============================================================================-async def _log_toin_stats_periodically(interval_seconds: int = 300) -> None:"""Background task that logs TOIN stats periodically.diff --git a/pyproject.toml b/pyproject.tomlindex 3f2e27e86..328e42780 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -272,6 +272,12 @@ module = [disallow_untyped_defs = falsewarn_return_any = false+# Handler mixins use self.* from HeadroomProxy via duck typing — mypy can't resolve these+[[tool.mypy.overrides]]+module = ["headroom.proxy.handlers.*"]+disallow_untyped_defs = false+ignore_errors = true+# Ignore third-party stubs with syntax errors[[tool.mypy.overrides]]module = ["mlx.*"]