Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add multi-provider batch API support with CCR post-processing
chopratejas95fd6d868822
diff --git a/headroom/ccr/__init__.py b/headroom/ccr/__init__.pyindex fe612e1b0..84aee7f3a 100644--- a/headroom/ccr/__init__.py+++ b/headroom/ccr/__init__.py@@ -3,18 +3,37 @@This module provides tool injection and retrieval handling for the CCR architecture.When tool outputs are compressed, the LLM can retrieve more data if needed.-Three key components:+Four key components:1. Tool Injection: Proxy injects headroom_retrieve tool into requests2. Response Handler: Intercepts responses, handles CCR tool calls automatically3. Context Tracker: Tracks compressed content across turns, enables proactive expansion+4. Batch Processing: Handles CCR tool calls in batch API results (async processing)Two distribution channels for the retrieval tool:1. Tool Injection: Proxy injects tool into request when compression occurs2. MCP Server: Standalone server exposes tool via MCP protocolWhen MCP is configured, tool injection is skipped to avoid duplicates.++Batch API Support:+- On batch submit: Store request context (messages, tools) in BatchContextStore+- On batch results: Detect CCR tool calls, execute retrieval, make continuation calls+- Works with all providers: Anthropic, OpenAI, Google"""+from .batch_processor import (+ BatchResultProcessor,+ BatchResultProcessorConfig,+ ProcessedBatchResult,+ process_batch_results,+)+from .batch_store import (+ BatchContext,+ BatchContextStore,+ BatchRequestContext,+ get_batch_context_store,+ reset_batch_context_store,+)from .context_tracker import (CompressedContext,ContextTracker,@@ -70,6 +89,16 @@ __all__ = ["ExpansionRecommendation","get_context_tracker","reset_context_tracker",+ # Batch processing+ "BatchContext",+ "BatchContextStore",+ "BatchRequestContext",+ "BatchResultProcessor",+ "BatchResultProcessorConfig",+ "ProcessedBatchResult",+ "get_batch_context_store",+ "process_batch_results",+ "reset_batch_context_store",# MCP server"CCRMCPServer","create_ccr_mcp_server",diff --git a/headroom/ccr/batch_processor.py b/headroom/ccr/batch_processor.pynew file mode 100644index 000000000..584354805--- /dev/null+++ b/headroom/ccr/batch_processor.py@@ -0,0 +1,534 @@+"""Batch result post-processor for CCR tool call handling.++When batch results are retrieved, this processor:+1. Detects CCR tool calls in each result+2. Executes the retrieval locally (from compression store)+3. Makes continuation API calls to get final responses+4. Returns the processed results with complete answers++This module works with all three providers:+- Anthropic: Batch Message API+- OpenAI: Batch API+- Google/Gemini: Batch API++Each provider has different result formats, but the logic is the same:+1. Parse result to detect CCR tool calls+2. Execute retrieval+3. Make continuation call with tool result+4. Replace partial result with complete result+"""++from __future__ import annotations++import logging+from dataclasses import dataclass+from typing import Any, Protocol++import httpx++from .batch_store import BatchContext, BatchRequestContext, get_batch_context_store+from .response_handler import CCRResponseHandler, ResponseHandlerConfig+from .tool_injection import CCR_TOOL_NAME++logger = logging.getLogger(__name__)+++class APIClient(Protocol):+ """Protocol for making API calls."""++ async def post(+ self,+ url: str,+ headers: dict[str, str],+ json: dict[str, Any],+ ) -> httpx.Response:+ """Make a POST request."""+ ...+++@dataclass+class BatchResultProcessorConfig:+ """Configuration for batch result processing."""++ # Whether to process CCR tool calls automatically+ enabled: bool = True++ # Timeout for continuation API calls (seconds)+ continuation_timeout: int = 120++ # Maximum continuation rounds per result+ max_continuation_rounds: int = 3+++@dataclass+class ProcessedBatchResult:+ """A processed batch result."""++ custom_id: str+ result: dict[str, Any]+ was_processed: bool = False # True if CCR tool calls were handled+ continuation_rounds: int = 0+ error: str | None = None+++class BatchResultProcessor:+ """Processes batch results to handle CCR tool calls.++ When a batch result contains a CCR tool call (headroom_retrieve),+ this processor:+ 1. Looks up the original request context+ 2. Executes the retrieval from the compression store+ 3. Makes a continuation API call with the tool result+ 4. Returns the final (complete) response++ Usage:+ processor = BatchResultProcessor(http_client)++ # Process results as they come in+ processed = await processor.process_results(+ batch_id="batch_123",+ results=raw_results,+ provider="anthropic"+ )++ # Results now have complete responses (CCR handled)+ """++ def __init__(+ self,+ http_client: httpx.AsyncClient,+ config: BatchResultProcessorConfig | None = None,+ ) -> None:+ self.http_client = http_client+ self.config = config or BatchResultProcessorConfig()+ self.ccr_handler = CCRResponseHandler(+ ResponseHandlerConfig(+ enabled=True,+ max_retrieval_rounds=self.config.max_continuation_rounds,+ )+ )++ # API base URLs+ self.api_urls = {+ "anthropic": "https://api.anthropic.com",+ "openai": "https://api.openai.com",+ "google": "https://generativelanguage.googleapis.com",+ }++ async def process_results(+ self,+ batch_id: str,+ results: list[dict[str, Any]],+ provider: str,+ ) -> list[ProcessedBatchResult]:+ """Process batch results, handling CCR tool calls.++ Args:+ batch_id: The batch ID (to look up context).+ results: Raw batch results from the provider.+ provider: The provider type.++ Returns:+ List of processed results (with CCR handled).+ """+ if not self.config.enabled:+ return [+ ProcessedBatchResult(+ custom_id=self._get_custom_id(r, provider),+ result=r,+ )+ for r in results+ ]++ # Get batch context+ store = get_batch_context_store()+ batch_context = await store.get(batch_id)++ if batch_context is None:+ logger.warning(+ f"Batch context not found for {batch_id}, returning results without CCR processing"+ )+ return [+ ProcessedBatchResult(+ custom_id=self._get_custom_id(r, provider),+ result=r,+ )+ for r in results+ ]++ # Process each result+ processed = []+ for result in results:+ custom_id = self._get_custom_id(result, provider)+ request_context = batch_context.get_request(custom_id)++ if request_context is None:+ logger.warning(f"Request context not found for {custom_id} in batch {batch_id}")+ processed.append(ProcessedBatchResult(custom_id=custom_id, result=result))+ continue++ # Check if result contains CCR tool calls+ response = self._extract_response(result, provider)++ if response and self.ccr_handler.has_ccr_tool_calls(response, provider):+ # Process the CCR tool calls+ try:+ final_result = await self._process_single_result(+ result,+ response,+ request_context,+ batch_context,+ provider,+ )+ processed.append(final_result)+ except Exception as e:+ logger.error(f"Failed to process CCR for {custom_id}: {e}")+ processed.append(+ ProcessedBatchResult(+ custom_id=custom_id,+ result=result,+ error=str(e),+ )+ )+ else:+ # No CCR tool calls, pass through+ processed.append(ProcessedBatchResult(custom_id=custom_id, result=result))++ return processed++ def _get_custom_id(self, result: dict[str, Any], provider: str) -> str:+ """Extract the custom ID from a result."""+ if provider == "anthropic":+ return result.get("custom_id", "")+ elif provider == "openai":+ return result.get("custom_id", "")+ elif provider == "google":+ # Google uses metadata.key+ return result.get("metadata", {}).get("key", "")+ return result.get("custom_id", result.get("id", ""))++ def _extract_response(+ self,+ result: dict[str, Any],+ provider: str,+ ) -> dict[str, Any] | None:+ """Extract the actual response from a batch result."""+ if provider == "anthropic":+ # Anthropic: result.result.message+ return result.get("result", {}).get("message")+ elif provider == "openai":+ # OpenAI: response.body (the full chat completion)+ return result.get("response", {}).get("body")+ elif provider == "google":+ # Google: response (the generateContent response)+ return result.get("response")+ return result.get("response")++ async def _process_single_result(+ self,+ original_result: dict[str, Any],+ response: dict[str, Any],+ request_context: BatchRequestContext,+ batch_context: BatchContext,+ provider: str,+ ) -> ProcessedBatchResult:+ """Process a single result with CCR tool calls.++ Args:+ original_result: The original batch result.+ response: The extracted response (with CCR tool calls).+ request_context: The original request context.+ batch_context: The batch context.+ provider: The provider type.++ Returns:+ Processed result with complete response.+ """+ custom_id = request_context.custom_id++ # Create API call function for continuations+ async def api_call_fn(+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ ) -> dict[str, Any]:+ return await self._make_continuation_call(+ messages,+ tools,+ request_context,+ batch_context,+ provider,+ )++ # Use CCR handler to process the response+ final_response = await self.ccr_handler.handle_response(+ response,+ request_context.messages,+ request_context.tools,+ api_call_fn,+ provider,+ )++ # Update the result with the final response+ updated_result = self._update_result(+ original_result,+ final_response,+ provider,+ )++ return ProcessedBatchResult(+ custom_id=custom_id,+ result=updated_result,+ was_processed=True,+ continuation_rounds=self.ccr_handler._retrieval_count,+ )++ async def _make_continuation_call(+ self,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ request_context: BatchRequestContext,+ batch_context: BatchContext,+ provider: str,+ ) -> dict[str, Any]:+ """Make a continuation API call.++ Args:+ messages: The messages including tool results.+ tools: The tools list.+ request_context: The request context.+ batch_context: The batch context.+ provider: The provider type.++ Returns:+ The API response.+ """+ if provider == "anthropic":+ return await self._anthropic_continuation(+ messages, tools, request_context, batch_context+ )+ elif provider == "openai":+ return await self._openai_continuation(messages, tools, request_context, batch_context)+ elif provider == "google":+ return await self._google_continuation(messages, tools, request_context, batch_context)+ else:+ raise ValueError(f"Unknown provider: {provider}")++ async def _anthropic_continuation(+ self,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ request_context: BatchRequestContext,+ batch_context: BatchContext,+ ) -> dict[str, Any]:+ """Make Anthropic continuation call."""+ url = f"{self.api_urls['anthropic']}/v1/messages"++ headers = {+ "Content-Type": "application/json",+ "anthropic-version": "2023-06-01",+ }+ if batch_context.api_key:+ headers["x-api-key"] = batch_context.api_key++ body = {+ "model": request_context.model,+ "messages": messages,+ "max_tokens": request_context.extras.get("max_tokens", 4096),+ }+ if tools:+ body["tools"] = tools++ response = await self.http_client.post(+ url,+ headers=headers,+ json=body,+ timeout=self.config.continuation_timeout,+ )+ response.raise_for_status()+ return response.json()++ async def _openai_continuation(+ self,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ request_context: BatchRequestContext,+ batch_context: BatchContext,+ ) -> dict[str, Any]:+ """Make OpenAI continuation call."""+ url = f"{self.api_urls['openai']}/v1/chat/completions"++ headers = {+ "Content-Type": "application/json",+ }+ if batch_context.api_key:+ headers["Authorization"] = f"Bearer {batch_context.api_key}"++ body = {+ "model": request_context.model,+ "messages": messages,+ }+ if tools:+ body["tools"] = tools++ response = await self.http_client.post(+ url,+ headers=headers,+ json=body,+ timeout=self.config.continuation_timeout,+ )+ response.raise_for_status()+ return response.json()++ async def _google_continuation(+ self,+ messages: list[dict[str, Any]],+ tools: list[dict[str, Any]] | None,+ request_context: BatchRequestContext,+ batch_context: BatchContext,+ ) -> dict[str, Any]:+ """Make Google/Gemini continuation call.++ Note: Google format uses 'contents' not 'messages',+ and 'parts' format for messages.+ """+ model = request_context.model+ url = f"{self.api_urls['google']}/v1beta/models/{model}:generateContent"++ if batch_context.api_key:+ url = f"{url}?key={batch_context.api_key}"++ headers = {"Content-Type": "application/json"}++ # Convert messages to Google format (contents with parts)+ contents = self._messages_to_google_contents(messages)++ body: dict[str, Any] = {"contents": contents}++ # Add system instruction if present+ if request_context.system_instruction:+ body["systemInstruction"] = {"parts": [{"text": request_context.system_instruction}]}++ # Add tools+ if tools:+ body["tools"] = [{"functionDeclarations": tools}]++ response = await self.http_client.post(+ url,+ headers=headers,+ json=body,+ timeout=self.config.continuation_timeout,+ )+ response.raise_for_status()+ return response.json()++ def _messages_to_google_contents(+ self,+ messages: list[dict[str, Any]],+ ) -> list[dict[str, Any]]:+ """Convert standard messages to Google contents format."""+ contents = []++ for msg in messages:+ role = msg.get("role", "")+ content = msg.get("content")++ # Handle Google format messages (already have parts)+ if "parts" in msg:+ google_role = "model" if role in ("assistant", "model") else "user"+ contents.append({"role": google_role, "parts": msg["parts"]})+ continue++ # Map roles+ if role == "system":+ # Skip system messages (handled separately)+ continue+ elif role == "assistant":+ google_role = "model"+ else:+ google_role = "user"++ # Convert content to parts+ if isinstance(content, str):+ contents.append({"role": google_role, "parts": [{"text": content}]})+ elif isinstance(content, list):+ # Handle structured content (tool results, etc.)+ parts = []+ for block in content:+ if isinstance(block, dict):+ if block.get("type") == "text":+ parts.append({"text": block.get("text", "")})+ elif block.get("type") == "tool_result":+ parts.append(+ {+ "functionResponse": {+ "name": block.get("tool_use_id", CCR_TOOL_NAME),+ "response": {"content": block.get("content", "")},+ }+ }+ )+ elif block.get("type") == "tool_use":+ parts.append(+ {+ "functionCall": {+ "name": block.get("name", ""),+ "args": block.get("input", {}),+ }+ }+ )+ if parts:+ contents.append({"role": google_role, "parts": parts})++ return contents++ def _update_result(+ self,+ original_result: dict[str, Any],+ final_response: dict[str, Any],+ provider: str,+ ) -> dict[str, Any]:+ """Update a batch result with the final processed response."""+ result = dict(original_result)++ if provider == "anthropic":+ # Update result.result.message+ if "result" not in result:+ result["result"] = {}+ result["result"]["message"] = final_response+ # Update type if it was tool_use+ result["result"]["type"] = "succeeded"++ elif provider == "openai":+ # Update response.body+ if "response" not in result:+ result["response"] = {}+ result["response"]["body"] = final_response++ elif provider == "google":+ # Update response directly+ result["response"] = final_response++ return result+++# Convenience function+async def process_batch_results(+ batch_id: str,+ results: list[dict[str, Any]],+ provider: str,+ http_client: httpx.AsyncClient,+) -> list[ProcessedBatchResult]:+ """Process batch results with CCR handling.++ This is a convenience function for one-off processing.++ Args:+ batch_id: The batch ID.+ results: Raw batch results.+ provider: The provider type.+ http_client: HTTP client for API calls.++ Returns:+ Processed results.+ """+ processor = BatchResultProcessor(http_client)+ return await processor.process_results(batch_id, results, provider)diff --git a/headroom/ccr/batch_store.py b/headroom/ccr/batch_store.pynew file mode 100644index 000000000..f2ff3c73e--- /dev/null+++ b/headroom/ccr/batch_store.py@@ -0,0 +1,253 @@+"""Batch context storage for CCR post-processing.++When batches are submitted, we store the request context (messages, tools, model)+so that when results are retrieved, we can handle CCR tool calls and make+continuation API calls.++This module provides:+1. BatchContext: Data class for stored batch context+2. BatchContextStore: TTL-based cache for batch contexts+"""++from __future__ import annotations++import asyncio+import logging+import time+from dataclasses import dataclass, field+from typing import Any++logger = logging.getLogger(__name__)++# Default TTL for batch contexts (24 hours - batches can take a while)+DEFAULT_BATCH_CONTEXT_TTL = 86400++# Maximum contexts to store (prevent memory issues)+MAX_BATCH_CONTEXTS = 10000+++@dataclass+class BatchRequestContext:+ """Context for a single request within a batch."""++ custom_id: (+ str # The request ID within the batch (custom_id for Anthropic, metadata.key for Google)+ )+ messages: list[dict[str, Any]]+ tools: list[dict[str, Any]] | None = None+ model: str = ""+ system_instruction: str | None = None # For Google format++ # Provider-specific extras+ extras: dict[str, Any] = field(default_factory=dict)+++@dataclass+class BatchContext:+ """Context for an entire batch submission.++ Stores all request contexts so we can handle CCR tool calls+ when results are retrieved.+ """++ batch_id: str+ provider: str # "anthropic", "openai", "google"+ created_at: float = field(default_factory=time.time)+ expires_at: float = 0++ # Map of custom_id -> BatchRequestContext+ requests: dict[str, BatchRequestContext] = field(default_factory=dict)++ # API configuration for continuation calls+ api_key: str | None = None+ api_base_url: str | None = None++ def __post_init__(self) -> None:+ if self.expires_at == 0:+ self.expires_at = self.created_at + DEFAULT_BATCH_CONTEXT_TTL++ @property+ def is_expired(self) -> bool:+ """Check if this context has expired."""+ return time.time() > self.expires_at++ def add_request(self, request: BatchRequestContext) -> None:+ """Add a request context to this batch."""+ self.requests[request.custom_id] = request++ def get_request(self, custom_id: str) -> BatchRequestContext | None:+ """Get a request context by custom_id."""+ return self.requests.get(custom_id)+++class BatchContextStore:+ """Thread-safe store for batch contexts.++ Stores batch submission contexts with TTL so that when results+ are retrieved, we can handle CCR tool calls.++ Features:+ - TTL-based expiration+ - Automatic cleanup of expired entries+ - Thread-safe operations+ - Memory limits++ Usage:+ store = BatchContextStore()++ # On batch submit+ context = BatchContext(batch_id="batch_123", provider="anthropic")+ for req in batch_requests:+ context.add_request(BatchRequestContext(+ custom_id=req["custom_id"],+ messages=req["params"]["messages"],+ tools=req["params"].get("tools"),+ model=req["params"]["model"],+ ))+ store.store(context)++ # On batch results retrieval+ context = store.get("batch_123")+ if context:+ for result in results:+ req_ctx = context.get_request(result["custom_id"])+ # ... handle CCR tool calls using req_ctx+ """++ def __init__(+ self,+ ttl: int = DEFAULT_BATCH_CONTEXT_TTL,+ max_contexts: int = MAX_BATCH_CONTEXTS,+ ) -> None:+ self._contexts: dict[str, BatchContext] = {}+ self._ttl = ttl+ self._max_contexts = max_contexts+ self._lock = asyncio.Lock()+ self._cleanup_task: asyncio.Task | None = None++ async def store(self, context: BatchContext) -> None:+ """Store a batch context.++ Args:+ context: The batch context to store.+ """+ async with self._lock:+ # Enforce memory limit+ if len(self._contexts) >= self._max_contexts:+ # Remove oldest entries+ await self._cleanup_oldest()++ # Set expiration+ context.expires_at = time.time() + self._ttl+ self._contexts[context.batch_id] = context++ logger.debug(+ f"Stored batch context {context.batch_id} with "+ f"{len(context.requests)} requests (provider={context.provider})"+ )++ async def get(self, batch_id: str) -> BatchContext | None:+ """Get a batch context by ID.++ Args:+ batch_id: The batch ID to look up.++ Returns:+ The batch context, or None if not found or expired.+ """+ async with self._lock:+ context = self._contexts.get(batch_id)++ if context is None:+ return None++ if context.is_expired:+ del self._contexts[batch_id]+ logger.debug(f"Batch context {batch_id} expired and removed")+ return None++ return context++ async def remove(self, batch_id: str) -> bool:+ """Remove a batch context.++ Args:+ batch_id: The batch ID to remove.++ Returns:+ True if removed, False if not found.+ """+ async with self._lock:+ if batch_id in self._contexts:+ del self._contexts[batch_id]+ return True+ return False++ async def cleanup_expired(self) -> int:+ """Remove all expired entries.++ Returns:+ Number of entries removed.+ """+ async with self._lock:+ now = time.time()+ expired = [batch_id for batch_id, ctx in self._contexts.items() if ctx.expires_at < now]++ for batch_id in expired:+ del self._contexts[batch_id]++ if expired:+ logger.debug(f"Cleaned up {len(expired)} expired batch contexts")++ return len(expired)++ async def _cleanup_oldest(self) -> None:+ """Remove oldest entries to make room for new ones."""+ # Sort by creation time, remove oldest 10%+ if not self._contexts:+ return++ sorted_entries = sorted(+ self._contexts.items(),+ key=lambda x: x[1].created_at,+ )++ to_remove = max(1, len(sorted_entries) // 10)+ for batch_id, _ in sorted_entries[:to_remove]:+ del self._contexts[batch_id]++ logger.debug(f"Cleaned up {to_remove} oldest batch contexts")++ def stats(self) -> dict[str, Any]:+ """Get store statistics."""+ return {+ "total_contexts": len(self._contexts),+ "max_contexts": self._max_contexts,+ "ttl_seconds": self._ttl,+ "providers": self._count_by_provider(),+ }++ def _count_by_provider(self) -> dict[str, int]:+ """Count contexts by provider."""+ counts: dict[str, int] = {}+ for ctx in self._contexts.values():+ counts[ctx.provider] = counts.get(ctx.provider, 0) + 1+ return counts+++# Global store instance+_batch_context_store: BatchContextStore | None = None+++def get_batch_context_store() -> BatchContextStore:+ """Get the global batch context store instance."""+ global _batch_context_store+ if _batch_context_store is None:+ _batch_context_store = BatchContextStore()+ return _batch_context_store+++def reset_batch_context_store() -> None:+ """Reset the global batch context store (for testing)."""+ global _batch_context_store+ _batch_context_store = Nonediff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.pyindex 9589e706d..6bfd8fe4a 100644--- a/headroom/ccr/response_handler.py+++ b/headroom/ccr/response_handler.py@@ -113,7 +113,9 @@ class CCRResponseHandler:"""tool_calls = self._extract_tool_calls(response, provider)return any(- tc.get("name") == CCR_TOOL_NAME or tc.get("function", {}).get("name") == CCR_TOOL_NAME+ tc.get("name") == CCR_TOOL_NAME+ or tc.get("function", {}).get("name") == CCR_TOOL_NAME+ or tc.get("functionCall", {}).get("name") == CCR_TOOL_NAME # Google formatfor tc in tool_calls)@@ -136,6 +138,15 @@ class CCRResponseHandler:tool_calls = message.get("tool_calls", [])return list(tool_calls) if tool_calls else []+ elif provider == "google":+ # Google/Gemini format: candidates[0].content.parts contains functionCall objects+ # Each part with a functionCall has: {"functionCall": {"name": "...", "args": {...}}}+ candidates = response.get("candidates", [])+ if not candidates:+ return []+ parts = candidates[0].get("content", {}).get("parts", [])+ return [part for part in parts if "functionCall" in part]+return []def _parse_ccr_tool_calls(@@ -157,8 +168,14 @@ class CCRResponseHandler:hash_key, query = parse_tool_call(tc, provider)if hash_key is not None:- # This is a CCR tool call- tool_call_id = tc.get("id", "")+ # This is a CCR tool call - extract tool_call_id based on provider+ if provider == "google":+ # Google uses function name as identifier for matching responses+ # The functionResponse.name must match the functionCall.name+ tool_call_id = tc.get("functionCall", {}).get("name", CCR_TOOL_NAME)+ else:+ # Anthropic and OpenAI use explicit IDs+ tool_call_id = tc.get("id", "")ccr_calls.append(CCRToolCall(tool_call_id=tool_call_id,@@ -295,6 +312,29 @@ class CCRResponseHandler:]}+ elif provider == "google":+ # Google/Gemini: user message with functionResponse parts+ # Format: {"role": "user", "parts": [{"functionResponse": {"name": "...", "response": {...}}}]}+ parts = []+ for result in results:+ # Parse the content JSON to include as response object+ try:+ response_data = json.loads(result.content)+ except json.JSONDecodeError:+ response_data = {"content": result.content}+ parts.append(+ {+ "functionResponse": {+ "name": result.tool_call_id, # tool_call_id contains the function name for Google+ "response": response_data,+ }+ }+ )+ return {+ "role": "user",+ "parts": parts,+ }+else:# Generic formatreturn {@@ -330,6 +370,17 @@ class CCRResponseHandler:"content": message.get("content"),"tool_calls": message.get("tool_calls"),}+ elif provider == "google":+ # Google/Gemini format: role is "model", content is in candidates[0].content.parts+ candidates = response.get("candidates", [])+ if candidates:+ parts = candidates[0].get("content", {}).get("parts", [])+ else:+ parts = []+ return {+ "role": "model",+ "parts": parts,+ }else:return {"role": "assistant",diff --git a/headroom/ccr/tool_injection.py b/headroom/ccr/tool_injection.pyindex cbf21ab1b..5a873e6bb 100644--- a/headroom/ccr/tool_injection.py+++ b/headroom/ccr/tool_injection.py@@ -194,10 +194,23 @@ class CCRToolInjector:# Detected compression markers_detected_hashes: list[str] = field(default_factory=list)- _marker_pattern: re.Pattern = field(- default_factory=lambda: re.compile(- r"\[(\d+) items compressed to (\d+)\. Retrieve more: hash=([a-f0-9]+)\]"- )+ # Multiple marker patterns to match different compressors:+ # - SmartCrusher: [100 items compressed to 10. Retrieve more: hash=abc123]+ # - LLMLingua: [1000 items compressed to 300. Retrieve more: hash=abc123]+ # - TextCompressor: [100 lines compressed to 10. Retrieve more: hash=abc123]+ # - LogCompressor: [200 lines compressed to 20. Retrieve more: hash=abc123]+ # - SearchCompressor: [50 matches compressed to 5. Retrieve more: hash=abc123]+ # - Generic: any [... compressed ... hash=xxx] pattern+ _marker_patterns: list[re.Pattern] = field(+ default_factory=lambda: [+ # Standard format: [N <type> compressed to M. Retrieve more: hash=xxx]+ # Matches items, lines, matches, or any other type+ re.compile(r"\[(\d+) \w+ compressed to (\d+)\. Retrieve more: hash=([a-f0-9]+)\]"),+ # Legacy format without "to M" or "Retrieve more:" (old TextCompressor)+ re.compile(r"\[(\d+) \w+ compressed\. hash=([a-f0-9]+)\]"),+ # Generic fallback: any compression marker with hash (8+ chars)+ re.compile(r"\[.*?compressed.*?hash=([a-f0-9]{8,})\]", re.IGNORECASE),+ ])def __post_init__(self) -> None:@@ -249,14 +262,39 @@ class CCRToolInjector:if isinstance(item, dict) and item.get("type") == "text":self._scan_text(item.get("text", ""))+ # Handle Google/Gemini format with parts+ parts = message.get("parts", [])+ if isinstance(parts, list):+ for part in parts:+ if isinstance(part, dict):+ # Text parts+ if "text" in part:+ self._scan_text(part.get("text", ""))+ # Function response parts (tool results)+ elif "functionResponse" in part:+ response = part.get("functionResponse", {}).get("response", {})+ if isinstance(response, str):+ self._scan_text(response)+ elif isinstance(response, dict):+ # Scan string values in response+ for value in response.values():+ if isinstance(value, str):+ self._scan_text(value)+return self._detected_hashesdef _scan_text(self, text: str) -> None:- """Scan text for compression markers."""- matches = self._marker_pattern.findall(text)- for _original, _compressed, hash_key in matches:- if hash_key not in self._detected_hashes:- self._detected_hashes.append(hash_key)+ """Scan text for compression markers from any compressor."""+ for pattern in self._marker_patterns:+ matches = pattern.findall(text)+ for match in matches:+ # Extract hash_key from match (last group is always the hash)+ if isinstance(match, tuple):+ hash_key = match[-1] # Last capture group is the hash+ else:+ hash_key = match # Single capture group (generic pattern)+ if hash_key and hash_key not in self._detected_hashes:+ self._detected_hashes.append(hash_key)def inject_tool_definition(self,@@ -389,7 +427,7 @@ def parse_tool_call(Returns:Tuple of (hash, query) or (None, None) if not a CCR tool call."""- # Get tool name+ # Get tool name and input data based on provider formatif provider == "anthropic":name = tool_call.get("name")input_data = tool_call.get("input", {})@@ -402,7 +440,13 @@ def parse_tool_call(input_data = json.loads(args_str)except json.JSONDecodeError:input_data = {}+ elif provider == "google":+ # Google/Gemini format: {"functionCall": {"name": "...", "args": {...}}}+ function_call = tool_call.get("functionCall", {})+ name = function_call.get("name")+ input_data = function_call.get("args", {})else:+ # Generic fallbackname = tool_call.get("name")input_data = tool_call.get("input", tool_call.get("args", {}))diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex b02c79290..4e689a320 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -57,11 +57,16 @@ from headroom.cache.compression_feedback import get_compression_feedbackfrom headroom.cache.compression_store import get_compression_storefrom headroom.ccr import (CCR_TOOL_NAME,+ # Batch processing+ BatchContext,+ BatchRequestContext,+ BatchResultProcessor,CCRResponseHandler,CCRToolInjector,ContextTracker,ContextTrackerConfig,ResponseHandlerConfig,+ get_batch_context_store,parse_tool_call,)from headroom.config import CacheAlignerConfig, CCRConfig, RollingWindowConfig, SmartCrusherConfig@@ -171,6 +176,7 @@ class ProxyConfig:host: str = "127.0.0.1"port: int = 8787openai_api_url: str | None = None # Custom OpenAI API URL override+ gemini_api_url: str | None = None # Custom Gemini API URL override# Optimizationoptimize: bool = True@@ -747,6 +753,7 @@ class HeadroomProxy:ANTHROPIC_API_URL = "https://api.anthropic.com"OPENAI_API_URL = "https://api.openai.com"+ GEMINI_API_URL = "https://generativelanguage.googleapis.com"def __init__(self, config: ProxyConfig):self.config = config@@ -755,6 +762,10 @@ class HeadroomProxy:if config.openai_api_url:HeadroomProxy.OPENAI_API_URL = config.openai_api_url+ # Override GEMINI_API_URL with config if set+ if config.gemini_api_url:+ HeadroomProxy.GEMINI_API_URL = config.gemini_api_url+# Initialize providersself.anthropic_provider = AnthropicProvider()self.openai_provider = OpenAIProvider()@@ -1579,85 +1590,465 @@ class HeadroomProxy:},)- async def _stream_response(+ async def handle_anthropic_batch_create(self,- url: str,- headers: dict,- body: dict,- provider: str,- model: str,- request_id: str,- original_tokens: int,- optimized_tokens: int,- tokens_saved: int,- transforms_applied: list[str],- tags: dict[str, str],- optimization_latency: float,- ) -> StreamingResponse:- """Stream response with metrics tracking.+ request: Request,+ ) -> Response:+ """Handle Anthropic POST /v1/messages/batches endpoint with compression.- Calculates output size incrementally to avoid accumulating all chunks in memory.+ Anthropic batch format:+ {+ "requests": [+ {+ "custom_id": "req-1",+ "params": {+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 1024,+ "messages": [{"role": "user", "content": "Hello"}]+ }+ },+ ...+ ]+ }++ This method applies compression to each request's messages before forwarding."""start_time = time.time()+ request_id = await self._next_request_id()- async def generate():- # Track total bytes incrementally instead of accumulating chunks- total_bytes = 0- try:- async with self.http_client.stream(- "POST", url, json=body, headers=headers- ) as response:- async for chunk in response.aiter_bytes():- total_bytes += len(chunk)- yield chunk- finally:- # Record metrics after stream completes- total_latency = (time.time() - start_time) * 1000+ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "type": "error",+ "error": {+ "type": "request_too_large",+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ },+ },+ )- # Estimate output tokens from total bytes (rough estimate: ~4 bytes per token)- output_tokens = total_bytes // 4+ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": f"Invalid JSON in request body: {e!s}",+ },+ },+ )- # Calculate cost- cost_usd = None- savings_usd = None- if self.cost_tracker:- cost_usd = self.cost_tracker.estimate_cost(- model, optimized_tokens, output_tokens+ requests_list = body.get("requests", [])+ if not requests_list:+ return JSONResponse(+ status_code=400,+ content={+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": "Missing or empty 'requests' field in batch request",+ },+ },+ )++ # Extract headers+ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ # Track compression stats across all batch requests+ total_original_tokens = 0+ total_optimized_tokens = 0+ total_tokens_saved = 0+ compressed_requests = []++ # Apply compression to each request in the batch+ for batch_req in requests_list:+ custom_id = batch_req.get("custom_id", "")+ params = batch_req.get("params", {})+ messages = params.get("messages", [])+ model = params.get("model", "unknown")++ if not messages or not self.config.optimize:+ # No messages or optimization disabled - pass through unchanged+ compressed_requests.append(batch_req)+ continue++ # Count original tokens+ tokenizer = get_tokenizer(model)+ original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ total_original_tokens += original_tokens++ # Apply optimization+ try:+ context_limit = self.anthropic_provider.get_context_limit(model)+ result = self.anthropic_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ )++ optimized_messages = result.messages+ optimized_tokens = sum(+ tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages+ )+ total_optimized_tokens += optimized_tokens+ tokens_saved = original_tokens - optimized_tokens+ total_tokens_saved += tokens_saved++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = params.get("tools")+ if self.config.ccr_inject_tool and tokens_saved > 0:+ injector = CCRToolInjector(+ provider="anthropic",+ inject_tool=True,+ inject_system_instructions=self.config.ccr_inject_system_instructions,)- original_cost = self.cost_tracker.estimate_cost(- model, original_tokens, output_tokens+ optimized_messages, tools, was_injected = injector.process_request(+ optimized_messages, tools)- if cost_usd and original_cost:- savings_usd = original_cost - cost_usd- self.cost_tracker.record_cost(cost_usd)- self.cost_tracker.record_savings(savings_usd)+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for batch request '{custom_id}'"+ )- await self.metrics.record_request(- provider=provider,- model=model,- input_tokens=optimized_tokens,- output_tokens=output_tokens,- tokens_saved=tokens_saved,- latency_ms=total_latency,- cost_usd=cost_usd or 0,- savings_usd=savings_usd or 0,+ # Create compressed batch request+ compressed_params = {**params, "messages": optimized_messages}+ if tools is not None:+ compressed_params["tools"] = tools+ compressed_requests.append(+ {+ "custom_id": custom_id,+ "params": compressed_params,+ })if tokens_saved > 0:- logger.info(- f"[{request_id}] {model}: saved {tokens_saved:,} tokens (streaming)"+ logger.debug(+ f"[{request_id}] Batch request '{custom_id}': "+ f"{original_tokens:,} -> {optimized_tokens:,} tokens "+ f"(saved {tokens_saved:,})")- return StreamingResponse(- generate(),- media_type="text/event-stream",+ except Exception as e:+ logger.warning(+ f"[{request_id}] Optimization failed for batch request '{custom_id}': {e}"+ )+ # Pass through unchanged on failure+ compressed_requests.append(batch_req)+ total_optimized_tokens += original_tokens++ # Update body with compressed requests+ body["requests"] = compressed_requests++ optimization_latency = (time.time() - start_time) * 1000++ # Forward request to Anthropic+ url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches"++ try:+ response = await self._retry_request("POST", url, headers, body)++ # Record metrics+ await self.metrics.record_request(+ provider="anthropic",+ model="batch",+ input_tokens=total_optimized_tokens,+ output_tokens=0,+ tokens_saved=total_tokens_saved,+ latency_ms=optimization_latency,+ )++ # Log compression stats+ if total_tokens_saved > 0:+ savings_percent = (+ (total_tokens_saved / total_original_tokens * 100)+ if total_original_tokens > 0+ else 0+ )+ logger.info(+ f"[{request_id}] Batch ({len(compressed_requests)} requests): "+ f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "+ f"(saved {total_tokens_saved:,}, {savings_percent:.1f}%)"+ )++ # Store batch context for CCR result processing+ if response.status_code == 200 and self.config.ccr_inject_tool:+ try:+ response_data = response.json()+ batch_id = response_data.get("id")+ if batch_id:+ await self._store_anthropic_batch_context(+ batch_id,+ requests_list,+ headers.get("x-api-key"),+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Failed to store batch context: {e}")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "type": "error",+ "error": {+ "type": "api_error",+ "message": "An error occurred while processing your batch request. Please try again.",+ },+ },+ )++ async def handle_anthropic_batch_passthrough(+ self,+ request: Request,+ batch_id: str | None = None,+ ) -> Response:+ """Handle Anthropic batch passthrough endpoints.++ Used for:+ - GET /v1/messages/batches - List batches+ - GET /v1/messages/batches/{batch_id} - Get batch+ - GET /v1/messages/batches/{batch_id}/results - Get batch results+ - POST /v1/messages/batches/{batch_id}/cancel - Cancel batch+ """+ start_time = time.time()+ path = request.url.path+ url = f"{self.ANTHROPIC_API_URL}{path}"++ # Preserve query string parameters (e.g., limit, after_id for list endpoint)+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ body = await request.body()++ response = await self.http_client.request(+ method=request.method,+ url=url,+ headers=headers,+ content=body,)- async def handle_openai_chat(+ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="anthropic",+ model="passthrough:batches",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def _store_anthropic_batch_context(+ self,+ batch_id: str,+ requests_list: list[dict[str, Any]],+ api_key: str | None,+ ) -> None:+ """Store batch context for CCR result processing.++ Args:+ batch_id: The batch ID from the API response.+ requests_list: The original batch requests.+ api_key: The API key for continuation calls.+ """+ store = get_batch_context_store()+ context = BatchContext(+ batch_id=batch_id,+ provider="anthropic",+ api_key=api_key,+ api_base_url=self.ANTHROPIC_API_URL,+ )++ for batch_req in requests_list:+ custom_id = batch_req.get("custom_id", "")+ params = batch_req.get("params", {})+ context.add_request(+ BatchRequestContext(+ custom_id=custom_id,+ messages=params.get("messages", []),+ tools=params.get("tools"),+ model=params.get("model", ""),+ extras={+ "max_tokens": params.get("max_tokens", 4096),+ "system": params.get("system"),+ },+ )+ )++ await store.store(context)+ logger.debug(f"Stored batch context for {batch_id} with {len(requests_list)} requests")++ async def handle_anthropic_batch_results(self,request: Request,- ) -> Response | StreamingResponse:- """Handle OpenAI /v1/chat/completions endpoint."""+ batch_id: str,+ ) -> Response:+ """Handle Anthropic batch results with CCR post-processing.++ This endpoint:+ 1. Fetches raw results from Anthropic+ 2. Detects CCR tool calls in each result+ 3. Executes retrieval and makes continuation calls+ 4. Returns processed results with complete responses+ """+ start_time = time.time()++ # Forward request to get raw results+ url = f"{self.ANTHROPIC_API_URL}/v1/messages/batches/{batch_id}/results"++ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ response = await self.http_client.get(url, headers=headers)++ if response.status_code != 200:+ # Error - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Parse results - Anthropic batch results are JSONL format+ raw_content = response.content.decode("utf-8")+ results = []+ for line in raw_content.strip().split("\n"):+ if line.strip():+ try:+ results.append(json.loads(line))+ except json.JSONDecodeError:+ continue++ if not results:+ # No results to process+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Check if we have context and CCR processing is enabled+ store = get_batch_context_store()+ batch_context = await store.get(batch_id)++ if batch_context is None or not self.config.ccr_inject_tool:+ # No context or CCR disabled - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Process results with CCR handler+ processor = BatchResultProcessor(self.http_client)+ processed = await processor.process_results(batch_id, results, "anthropic")++ # Convert back to JSONL format+ processed_lines = []+ for p in processed:+ processed_lines.append(json.dumps(p.result))+ if p.was_processed:+ logger.info(+ f"CCR: Processed batch result {p.custom_id} "+ f"({p.continuation_rounds} continuation rounds)"+ )++ processed_content = "\n".join(processed_lines)++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="anthropic",+ model="batch:ccr-processed",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ return Response(+ content=processed_content.encode("utf-8"),+ status_code=200,+ media_type="application/jsonl",+ )++ # =========================================================================+ # Google/Gemini Batch API Handlers+ # =========================================================================++ async def handle_google_batch_create(+ self,+ request: Request,+ model: str,+ ) -> Response:+ """Handle Google POST /v1beta/models/{model}:batchGenerateContent endpoint.++ Google batch format:+ {+ "batch": {+ "display_name": "my-batch",+ "input_config": {+ "requests": {+ "requests": [+ {+ "request": {"contents": [{"parts": [{"text": "..."}]}]},+ "metadata": {"key": "request-1"}+ }+ ]+ }+ }+ }+ }++ This method applies compression to each request's contents before forwarding.+ """start_time = time.time()request_id = await self._next_request_id()@@ -1668,9 +2059,9 @@ class HeadroomProxy:status_code=413,content={"error": {+ "code": 413,"message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",- "type": "invalid_request_error",- "code": "request_too_large",+ "status": "INVALID_ARGUMENT",}},)@@ -1683,30 +2074,638 @@ class HeadroomProxy:status_code=400,content={"error": {+ "code": 400,"message": f"Invalid JSON in request body: {e!s}",- "type": "invalid_request_error",- "code": "invalid_json",+ "status": "INVALID_ARGUMENT",}},)- model = body.get("model", "unknown")- messages = body.get("messages", [])- stream = body.get("stream", False)+ # Extract batch config+ batch_config = body.get("batch", {})+ input_config = batch_config.get("input_config", {})+ requests_wrapper = input_config.get("requests", {})+ requests_list = requests_wrapper.get("requests", [])++ if not requests_list:+ # No inline requests - might be using file input, pass through+ logger.debug(f"[{request_id}] Google batch: No inline requests, passing through")+ return await self._google_batch_passthrough(request, model, body)++ # Extract headersheaders = dict(request.headers.items())headers.pop("host", None)headers.pop("content-length", None)- tags = self._extract_tags(headers)- # Rate limiting- if self.rate_limiter:- rate_key = headers.get("authorization", "default")[:20]- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)- if not allowed:- await self.metrics.record_rate_limited()- raise HTTPException(- status_code=429,- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ # Track compression stats+ total_original_tokens = 0+ total_optimized_tokens = 0+ total_tokens_saved = 0+ compressed_requests = []++ # Apply compression to each request in the batch+ for idx, batch_req in enumerate(requests_list):+ req_content = batch_req.get("request", {})+ metadata = batch_req.get("metadata", {})+ contents = req_content.get("contents", [])++ if not contents or not self.config.optimize:+ # No contents or optimization disabled - pass through unchanged+ compressed_requests.append(batch_req)+ continue++ # Convert Google format to messages for compression+ system_instruction = req_content.get("systemInstruction")+ messages = self._gemini_contents_to_messages(contents, system_instruction)++ # Count original tokens+ tokenizer = get_tokenizer(model)+ original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)+ total_original_tokens += original_tokens++ # Apply optimization+ try:+ # Default context limit for most models+ context_limit = 128000++ # Use OpenAI pipeline (similar message format after conversion)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ )++ optimized_messages = result.messages+ optimized_tokens = sum(+ tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages+ )+ total_optimized_tokens += optimized_tokens+ tokens_saved = original_tokens - optimized_tokens+ total_tokens_saved += tokens_saved++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = req_content.get("tools")+ # Extract existing function declarations if present+ existing_funcs = None+ if tools:+ for tool in tools:+ if "functionDeclarations" in tool:+ existing_funcs = tool["functionDeclarations"]+ break++ if self.config.ccr_inject_tool and tokens_saved > 0:+ injector = CCRToolInjector(+ provider="google",+ inject_tool=True,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, injected_funcs, was_injected = injector.process_request(+ optimized_messages, existing_funcs+ )+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for Google batch request {idx}"+ )+ existing_funcs = injected_funcs++ # Convert back to Google contents format+ optimized_contents, optimized_sys_inst = self._messages_to_gemini_contents(+ optimized_messages+ )++ # Create compressed batch request+ compressed_req_content = {**req_content, "contents": optimized_contents}+ if optimized_sys_inst:+ compressed_req_content["systemInstruction"] = optimized_sys_inst+ if existing_funcs is not None:+ compressed_req_content["tools"] = [{"functionDeclarations": existing_funcs}]++ compressed_req = {+ "request": compressed_req_content,+ "metadata": metadata,+ }++ compressed_requests.append(compressed_req)++ if tokens_saved > 0:+ logger.debug(+ f"[{request_id}] Google batch request {idx}: "+ f"{original_tokens:,} -> {optimized_tokens:,} tokens "+ f"(saved {tokens_saved:,})"+ )++ except Exception as e:+ logger.warning(+ f"[{request_id}] Optimization failed for Google batch request {idx}: {e}"+ )+ # Pass through unchanged on failure+ compressed_requests.append(batch_req)+ total_optimized_tokens += original_tokens++ # Update body with compressed requests+ body["batch"]["input_config"]["requests"]["requests"] = compressed_requests++ optimization_latency = (time.time() - start_time) * 1000++ # Forward request to Google+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"++ # Add API key to URL if present in headers+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ url = f"{url}?key={api_key}"++ try:+ response = await self._retry_request("POST", url, headers, body)++ # Record metrics+ await self.metrics.record_request(+ provider="google",+ model=f"batch:{model}",+ input_tokens=total_optimized_tokens,+ output_tokens=0,+ tokens_saved=total_tokens_saved,+ latency_ms=optimization_latency,+ )++ # Log compression stats+ if total_tokens_saved > 0:+ savings_percent = (+ (total_tokens_saved / total_original_tokens * 100)+ if total_original_tokens > 0+ else 0+ )+ logger.info(+ f"[{request_id}] Google batch compression: "+ f"{total_original_tokens:,} -> {total_optimized_tokens:,} tokens "+ f"({savings_percent:.1f}% saved across {len(requests_list)} requests)"+ )++ # Store batch context for CCR result processing+ if response.status_code == 200 and self.config.ccr_inject_tool:+ try:+ response_data = response.json()+ batch_name = response_data.get("name")+ if batch_name:+ await self._store_google_batch_context(+ batch_name,+ requests_list,+ model,+ api_key,+ )+ except Exception as e:+ logger.warning(f"[{request_id}] Failed to store Google batch context: {e}")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ logger.error(f"[{request_id}] Google batch request failed: {e}")+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "code": 500,+ "message": f"Failed to forward batch request: {e!s}",+ "status": "INTERNAL",+ }+ },+ )++ async def _google_batch_passthrough(+ self,+ request: Request,+ model: str,+ body: dict | None = None,+ ) -> Response:+ """Pass through Google batch request without modification."""+ start_time = time.time()++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:batchGenerateContent"++ # Add API key to URL if present in headers+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ url = f"{url}?key={api_key}"++ if body is None:+ body_content = await request.body()+ else:+ body_content = json.dumps(body).encode()++ response = await self.http_client.post(+ url,+ headers=headers,+ content=body_content,+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="google",+ model=f"passthrough:batch:{model}",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def handle_google_batch_passthrough(+ self,+ request: Request,+ batch_name: str | None = None,+ ) -> Response:+ """Handle Google batch passthrough endpoints.++ Used for:+ - GET /v1beta/batches/{batch_name} - Get batch status+ - POST /v1beta/batches/{batch_name}:cancel - Cancel batch+ - DELETE /v1beta/batches/{batch_name} - Delete batch+ """+ start_time = time.time()+ path = request.url.path+ url = f"{self.GEMINI_API_URL}{path}"++ # Preserve query string parameters+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ # Handle API key+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ if "?" in url:+ url = f"{url}&key={api_key}"+ else:+ url = f"{url}?key={api_key}"++ body = await request.body()++ response = await self.http_client.request(+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="google",+ model="passthrough:batches",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def _store_google_batch_context(+ self,+ batch_name: str,+ requests_list: list[dict[str, Any]],+ model: str,+ api_key: str | None,+ ) -> None:+ """Store Google batch context for CCR result processing.++ Args:+ batch_name: The batch name from the API response.+ requests_list: The original batch requests.+ model: The model used for the batch.+ api_key: The API key for continuation calls.+ """+ store = get_batch_context_store()+ context = BatchContext(+ batch_id=batch_name,+ provider="google",+ api_key=api_key,+ api_base_url=self.GEMINI_API_URL,+ )++ for batch_req in requests_list:+ metadata = batch_req.get("metadata", {})+ custom_id = metadata.get("key", "")+ req_content = batch_req.get("request", {})+ contents = req_content.get("contents", [])+ system_instruction = req_content.get("systemInstruction")++ # Convert contents to messages format for CCR handler+ messages = self._gemini_contents_to_messages(contents, system_instruction)++ # Extract system instruction text if present+ sys_text = None+ if system_instruction:+ parts = system_instruction.get("parts", [])+ if parts and isinstance(parts[0], dict):+ sys_text = parts[0].get("text")++ context.add_request(+ BatchRequestContext(+ custom_id=custom_id,+ messages=messages,+ tools=req_content.get("tools"),+ model=model,+ system_instruction=sys_text,+ )+ )++ await store.store(context)+ logger.debug(+ f"Stored Google batch context for {batch_name} with {len(requests_list)} requests"+ )++ async def handle_google_batch_results(+ self,+ request: Request,+ batch_name: str,+ ) -> Response:+ """Handle Google batch results with CCR post-processing.++ Google batch results endpoint returns the batch operation status.+ When status is SUCCEEDED, results are embedded in the response.+ This handler processes CCR tool calls in those results.+ """+ start_time = time.time()++ # Forward request to get batch status/results+ url = f"{self.GEMINI_API_URL}/v1beta/{batch_name}"++ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ # Handle API key+ api_key = headers.pop("x-goog-api-key", None)+ if api_key:+ if "?" in url:+ url = f"{url}&key={api_key}"+ else:+ url = f"{url}?key={api_key}"++ response = await self.http_client.get(url, headers=headers)++ if response.status_code != 200:+ # Error - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Parse response+ try:+ response_data = response.json()+ except json.JSONDecodeError:+ # Not JSON - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Check if batch has results (state must be SUCCEEDED)+ metadata = response_data.get("metadata", {})+ state = metadata.get("state")++ if state != "SUCCEEDED":+ # Batch not complete - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Extract results from response+ # Google embeds results in the batch response+ results = response_data.get("response", {}).get("responses", [])++ if not results:+ # No results to process+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Check if we have context and CCR processing is enabled+ store = get_batch_context_store()+ batch_context = await store.get(batch_name)++ if batch_context is None or not self.config.ccr_inject_tool:+ # No context or CCR disabled - pass through+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # Process results with CCR handler+ processor = BatchResultProcessor(self.http_client)+ processed = await processor.process_results(batch_name, results, "google")++ # Update response with processed results+ processed_results = [p.result for p in processed]+ response_data["response"]["responses"] = processed_results++ for p in processed:+ if p.was_processed:+ logger.info(+ f"CCR: Processed Google batch result {p.custom_id} "+ f"({p.continuation_rounds} continuation rounds)"+ )++ # Track metrics+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider="google",+ model="batch:ccr-processed",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ )++ return JSONResponse(content=response_data, status_code=200)++ async def _stream_response(+ self,+ url: str,+ headers: dict,+ body: dict,+ provider: str,+ model: str,+ request_id: str,+ original_tokens: int,+ optimized_tokens: int,+ tokens_saved: int,+ transforms_applied: list[str],+ tags: dict[str, str],+ optimization_latency: float,+ ) -> StreamingResponse:+ """Stream response with metrics tracking.++ Calculates output size incrementally to avoid accumulating all chunks in memory.+ """+ start_time = time.time()++ async def generate():+ # Track total bytes incrementally instead of accumulating chunks+ total_bytes = 0+ try:+ async with self.http_client.stream(+ "POST", url, json=body, headers=headers+ ) as response:+ async for chunk in response.aiter_bytes():+ total_bytes += len(chunk)+ yield chunk+ finally:+ # Record metrics after stream completes+ total_latency = (time.time() - start_time) * 1000++ # Estimate output tokens from total bytes (rough estimate: ~4 bytes per token)+ output_tokens = total_bytes // 4++ # Calculate cost+ cost_usd = None+ savings_usd = None+ if self.cost_tracker:+ cost_usd = self.cost_tracker.estimate_cost(+ model, optimized_tokens, output_tokens+ )+ original_cost = self.cost_tracker.estimate_cost(+ model, original_tokens, output_tokens+ )+ if cost_usd and original_cost:+ savings_usd = original_cost - cost_usd+ self.cost_tracker.record_cost(cost_usd)+ self.cost_tracker.record_savings(savings_usd)++ await self.metrics.record_request(+ provider=provider,+ model=model,+ input_tokens=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cost_usd=cost_usd or 0,+ savings_usd=savings_usd or 0,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: saved {tokens_saved:,} tokens (streaming)"+ )++ return StreamingResponse(+ generate(),+ media_type="text/event-stream",+ )++ async def handle_openai_chat(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle OpenAI /v1/chat/completions endpoint."""+ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "type": "invalid_request_error",+ "code": "request_too_large",+ }+ },+ )++ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON in request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )+ model = body.get("model", "unknown")+ messages = body.get("messages", [])+ stream = body.get("stream", False)++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting+ if self.rate_limiter:+ rate_key = headers.get("authorization", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",)# Check cache@@ -1716,31 +2715,911 @@ class HeadroomProxy:await self.metrics.record_request(provider="openai",model=model,- input_tokens=0,- output_tokens=0,- tokens_saved=cached.tokens_saved_per_hit,- latency_ms=(time.time() - start_time) * 1000,- cached=True,+ input_tokens=0,+ output_tokens=0,+ tokens_saved=cached.tokens_saved_per_hit,+ latency_ms=(time.time() - start_time) * 1000,+ cached=True,+ )++ # Remove compression headers from cached response+ response_headers = dict(cached.response_headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(content=cached.response_body, headers=response_headers)++ # Token counting+ tokenizer = get_tokenizer(model)+ original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)++ # Optimization+ transforms_applied = []+ optimized_messages = messages+ optimized_tokens = original_tokens++ if self.config.optimize and messages:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ )+ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ optimized_tokens = sum(+ tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages+ )+ except Exception as e:+ logger.warning(f"Optimization failed: {e}")++ tokens_saved = original_tokens - optimized_tokens+ optimization_latency = (time.time() - start_time) * 1000++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = body.get("tools")+ if self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions:+ injector = CCRToolInjector(+ provider="openai",+ inject_tool=self.config.ccr_inject_tool,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ optimized_messages, tools, was_injected = injector.process_request(+ optimized_messages, tools+ )++ if injector.has_compressed_content:+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"+ )+ else:+ logger.debug(+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"+ )++ body["messages"] = optimized_messages+ if tools is not None:+ body["tools"] = tools+ url = f"{self.OPENAI_API_URL}/v1/chat/completions"++ try:+ if stream:+ return await self._stream_response(+ url,+ headers,+ body,+ "openai",+ model,+ request_id,+ original_tokens,+ optimized_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ output_tokens = 0+ try:+ resp_json = response.json()+ usage = resp_json.get("usage", {})+ output_tokens = usage.get("completion_tokens", 0)+ except Exception:+ pass++ # Cost tracking+ cost_usd = savings_usd = None+ if self.cost_tracker:+ cost_usd = self.cost_tracker.estimate_cost(+ model, optimized_tokens, output_tokens+ )+ original_cost = self.cost_tracker.estimate_cost(+ model, original_tokens, output_tokens+ )+ if cost_usd and original_cost:+ savings_usd = original_cost - cost_usd+ self.cost_tracker.record_cost(cost_usd)+ self.cost_tracker.record_savings(savings_usd)++ # Cache+ if self.cache and response.status_code == 200:+ await self.cache.set(+ messages, model, response.content, dict(response.headers), tokens_saved+ )++ # Metrics+ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=optimized_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cost_usd=cost_usd or 0,+ savings_usd=savings_usd or 0,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"(saved {tokens_saved:,} tokens)"+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed()+ # Log full error details internally for debugging+ logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")+ # Return sanitized error message to client (don't expose internal details)+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "type": "server_error",+ "code": "proxy_error",+ }+ },+ )++ async def handle_passthrough(+ self,+ request: Request,+ base_url: str,+ endpoint_name: str | None = None,+ provider: str | None = None,+ ) -> Response:+ """Pass through request unchanged.++ Args:+ request: The incoming request+ base_url: The upstream API base URL+ endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")+ provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")+ """+ start_time = time.time()+ path = request.url.path+ url = f"{base_url}{path}"++ # Preserve query string parameters+ if request.url.query:+ url = f"{url}?{request.url.query}"++ headers = dict(request.headers.items())+ headers.pop("host", None)++ body = await request.body()++ response = await self.http_client.request( # type: ignore[union-attr]+ method=request.method,+ url=url,+ headers=headers,+ content=body,+ )++ # Remove compression headers since httpx already decompressed the response+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None) # Length changed after decompression++ # Track stats for passthrough requests+ if endpoint_name and provider:+ latency_ms = (time.time() - start_time) * 1000+ await self.metrics.record_request(+ provider=provider,+ model=f"passthrough:{endpoint_name}",+ input_tokens=0,+ output_tokens=0,+ tokens_saved=0,+ latency_ms=latency_ms,+ cached=False,+ cost_usd=0,+ savings_usd=0,+ )++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ # =========================================================================+ # OpenAI Batch API with Compression+ # =========================================================================++ async def handle_batch_create(self, request: Request) -> Response:+ """Handle POST /v1/batches - Create a batch with compression.++ Flow:+ 1. Parse request to get input_file_id+ 2. Download the JSONL file content from OpenAI+ 3. Parse each line and compress the messages+ 4. Create a new compressed JSONL file+ 5. Upload compressed file to OpenAI+ 6. Create batch with the new compressed file_id+ 7. Return batch object with compression stats in metadata+ """+ start_time = time.time()+ request_id = await self._next_request_id()++ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON in request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )++ input_file_id = body.get("input_file_id")+ endpoint = body.get("endpoint")+ completion_window = body.get("completion_window", "24h")+ metadata = body.get("metadata", {})++ if not input_file_id:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": "input_file_id is required",+ "type": "invalid_request_error",+ "code": "missing_parameter",+ }+ },+ )++ if not endpoint:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": "endpoint is required",+ "type": "invalid_request_error",+ "code": "missing_parameter",+ }+ },+ )++ # Only compress chat completions endpoint+ if endpoint != "/v1/chat/completions":+ # Pass through for other endpoints+ return await self._batch_passthrough(request, body)++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ try:+ # Step 1: Download the input file from OpenAI+ logger.info(f"[{request_id}] Batch: Downloading input file {input_file_id}")+ file_content = await self._download_openai_file(input_file_id, headers)++ if file_content is None:+ return JSONResponse(+ status_code=404,+ content={+ "error": {+ "message": f"Failed to download file {input_file_id}",+ "type": "invalid_request_error",+ "code": "file_not_found",+ }+ },+ )++ # Step 2: Parse and compress each line+ logger.info(f"[{request_id}] Batch: Compressing JSONL content")+ compressed_lines, stats = await self._compress_batch_jsonl(file_content, request_id)++ if stats["total_requests"] == 0:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": "No valid requests found in input file",+ "type": "invalid_request_error",+ "code": "empty_file",+ }+ },+ )++ # Step 3: Create compressed JSONL content+ compressed_content = "\n".join(compressed_lines)++ # Step 4: Upload compressed file to OpenAI+ logger.info(f"[{request_id}] Batch: Uploading compressed file")+ new_file_id = await self._upload_openai_file(+ compressed_content, f"compressed_{input_file_id}.jsonl", headers+ )++ if new_file_id is None:+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "message": "Failed to upload compressed file",+ "type": "server_error",+ "code": "upload_failed",+ }+ },+ )++ # Step 5: Create batch with compressed file+ logger.info(f"[{request_id}] Batch: Creating batch with compressed file {new_file_id}")++ # Add compression stats to metadata+ compression_metadata = {+ **metadata,+ "headroom_compressed": "true",+ "headroom_original_file_id": input_file_id,+ "headroom_total_requests": str(stats["total_requests"]),+ "headroom_tokens_saved": str(stats["total_tokens_saved"]),+ "headroom_original_tokens": str(stats["total_original_tokens"]),+ "headroom_compressed_tokens": str(stats["total_compressed_tokens"]),+ "headroom_savings_percent": f"{stats['savings_percent']:.1f}",+ }++ batch_body = {+ "input_file_id": new_file_id,+ "endpoint": endpoint,+ "completion_window": completion_window,+ "metadata": compression_metadata,+ }++ url = f"{self.OPENAI_API_URL}/v1/batches"+ response = await self.http_client.post(url, json=batch_body, headers=headers) # type: ignore[union-attr]++ total_latency = (time.time() - start_time) * 1000++ # Log compression stats+ logger.info(+ f"[{request_id}] Batch created: {stats['total_requests']} requests, "+ f"{stats['total_original_tokens']:,} -> {stats['total_compressed_tokens']:,} tokens "+ f"(saved {stats['total_tokens_saved']:,} tokens, {stats['savings_percent']:.1f}%) "+ f"in {total_latency:.0f}ms"+ )++ # Record metrics+ await self.metrics.record_request(+ provider="openai",+ model="batch",+ input_tokens=stats["total_compressed_tokens"],+ output_tokens=0,+ tokens_saved=stats["total_tokens_saved"],+ latency_ms=total_latency,+ )++ # Return response with compression info in headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)+ response_headers["x-headroom-tokens-saved"] = str(stats["total_tokens_saved"])+ response_headers["x-headroom-savings-percent"] = f"{stats['savings_percent']:.1f}"++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ except Exception as e:+ logger.error(f"[{request_id}] Batch creation failed: {type(e).__name__}: {e}")+ await self.metrics.record_failed()+ return JSONResponse(+ status_code=500,+ content={+ "error": {+ "message": "An error occurred while processing the batch request",+ "type": "server_error",+ "code": "batch_processing_error",+ }+ },+ )++ async def _download_openai_file(self, file_id: str, headers: dict) -> str | None:+ """Download file content from OpenAI."""+ url = f"{self.OPENAI_API_URL}/v1/files/{file_id}/content"+ try:+ response = await self.http_client.get(url, headers=headers) # type: ignore[union-attr]+ if response.status_code == 200:+ return response.text+ logger.error(f"Failed to download file {file_id}: {response.status_code}")+ return None+ except Exception as e:+ logger.error(f"Error downloading file {file_id}: {e}")+ return None++ async def _upload_openai_file(self, content: str, filename: str, headers: dict) -> str | None:+ """Upload a file to OpenAI for batch processing."""+ url = f"{self.OPENAI_API_URL}/v1/files"++ # Prepare multipart form data+ # We need to use httpx's files parameter for multipart upload+ files = {+ "file": (filename, content.encode("utf-8"), "application/jsonl"),+ }+ data = {+ "purpose": "batch",+ }++ # Remove content-type from headers (httpx will set it for multipart)+ upload_headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}++ try:+ response = await self.http_client.post( # type: ignore[union-attr]+ url, files=files, data=data, headers=upload_headers+ )+ if response.status_code == 200:+ result = response.json()+ return result.get("id")+ logger.error(f"Failed to upload file: {response.status_code} - {response.text}")+ return None+ except Exception as e:+ logger.error(f"Error uploading file: {e}")+ return None++ async def _compress_batch_jsonl(self, content: str, request_id: str) -> tuple[list[str], dict]:+ """Compress messages in each line of a batch JSONL file.++ Returns:+ Tuple of (compressed_lines, stats_dict)+ """+ lines = content.strip().split("\n")+ compressed_lines = []+ total_original_tokens = 0+ total_compressed_tokens = 0+ total_requests = 0+ errors = 0++ tokenizer = get_tokenizer("gpt-4") # Use gpt-4 tokenizer for batch++ for i, line in enumerate(lines):+ if not line.strip():+ continue++ try:+ request_obj = json.loads(line)+ body = request_obj.get("body", {})+ messages = body.get("messages", [])+ model = body.get("model", "gpt-4")++ if not messages:+ # No messages to compress, pass through+ compressed_lines.append(line)+ total_requests += 1+ continue++ # Count original tokens+ original_tokens = sum(+ tokenizer.count_text(str(m.get("content", ""))) for m in messages+ )+ total_original_tokens += original_tokens++ # Compress messages using the OpenAI pipeline+ if self.config.optimize:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ )+ compressed_messages = result.messages+ except Exception as e:+ logger.warning(f"[{request_id}] Compression failed for line {i}: {e}")+ compressed_messages = messages+ else:+ compressed_messages = messages++ # Count compressed tokens+ compressed_tokens = sum(+ tokenizer.count_text(str(m.get("content", ""))) for m in compressed_messages+ )+ total_compressed_tokens += compressed_tokens+ tokens_saved = original_tokens - compressed_tokens++ # CCR Tool Injection: Inject retrieval tool if compression occurred+ tools = body.get("tools")+ if self.config.ccr_inject_tool and tokens_saved > 0:+ injector = CCRToolInjector(+ provider="openai",+ inject_tool=True,+ inject_system_instructions=self.config.ccr_inject_system_instructions,+ )+ compressed_messages, tools, was_injected = injector.process_request(+ compressed_messages, tools+ )+ if was_injected:+ logger.debug(+ f"[{request_id}] CCR: Injected retrieval tool for batch line {i}"+ )++ # Update body with compressed messages+ body["messages"] = compressed_messages+ if tools is not None:+ body["tools"] = tools+ request_obj["body"] = body++ compressed_lines.append(json.dumps(request_obj))+ total_requests += 1++ except json.JSONDecodeError as e:+ logger.warning(f"[{request_id}] Invalid JSON on line {i}: {e}")+ errors += 1+ # Keep original line on error+ compressed_lines.append(line)+ total_requests += 1++ total_tokens_saved = total_original_tokens - total_compressed_tokens+ savings_percent = (+ (total_tokens_saved / total_original_tokens * 100) if total_original_tokens > 0 else 0+ )++ stats = {+ "total_requests": total_requests,+ "total_original_tokens": total_original_tokens,+ "total_compressed_tokens": total_compressed_tokens,+ "total_tokens_saved": total_tokens_saved,+ "savings_percent": savings_percent,+ "errors": errors,+ }++ return compressed_lines, stats++ async def _batch_passthrough(self, request: Request, body: dict) -> Response:+ """Pass through batch request to OpenAI without compression."""+ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ url = f"{self.OPENAI_API_URL}/v1/batches"+ response = await self.http_client.post(url, json=body, headers=headers) # type: ignore[union-attr]++ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )++ async def handle_batch_list(self, request: Request) -> Response:+ """Handle GET /v1/batches - List batches (passthrough)."""+ return await self.handle_passthrough(request, self.OPENAI_API_URL)++ async def handle_batch_get(self, request: Request, batch_id: str) -> Response:+ """Handle GET /v1/batches/{batch_id} - Get batch (passthrough)."""+ return await self.handle_passthrough(request, self.OPENAI_API_URL)++ async def handle_batch_cancel(self, request: Request, batch_id: str) -> Response:+ """Handle POST /v1/batches/{batch_id}/cancel - Cancel batch (passthrough)."""+ return await self.handle_passthrough(request, self.OPENAI_API_URL)++ def _gemini_contents_to_messages(+ self, contents: list[dict], system_instruction: dict | None = None+ ) -> list[dict]:+ """Convert Gemini contents[] format to OpenAI messages[] format for optimization.++ Gemini format:+ contents: [{"role": "user", "parts": [{"text": "..."}]}]+ systemInstruction: {"parts": [{"text": "..."}]}++ OpenAI format:+ messages: [{"role": "user", "content": "..."}]+ """+ messages = []++ # Add system instruction as system message+ if system_instruction:+ parts = system_instruction.get("parts", [])+ text_parts = [p.get("text", "") for p in parts if "text" in p]+ if text_parts:+ messages.append({"role": "system", "content": "\n".join(text_parts)})++ # Convert contents to messages+ for content in contents:+ role = content.get("role", "user")+ # Map Gemini roles to OpenAI roles+ if role == "model":+ role = "assistant"++ parts = content.get("parts", [])+ text_parts = [p.get("text", "") for p in parts if "text" in p]++ if text_parts:+ messages.append({"role": role, "content": "\n".join(text_parts)})++ return messages++ def _messages_to_gemini_contents(self, messages: list[dict]) -> tuple[list[dict], dict | None]:+ """Convert OpenAI messages[] format back to Gemini contents[] format.++ Returns:+ (contents, system_instruction) tuple+ """+ contents = []+ system_instruction = None++ for msg in messages:+ role = msg.get("role", "user")+ content = msg.get("content", "")++ if role == "system":+ # Extract as systemInstruction+ system_instruction = {"parts": [{"text": content}]}+ else:+ # Map OpenAI roles to Gemini roles+ gemini_role = "model" if role == "assistant" else "user"+ contents.append({"role": gemini_role, "parts": [{"text": content}]})++ return contents, system_instruction++ async def handle_openai_responses(+ self,+ request: Request,+ ) -> Response | StreamingResponse:+ """Handle OpenAI /v1/responses endpoint (new Responses API).++ The Responses API differs from /v1/chat/completions:+ - Input: `input` (string or array) instead of `messages`+ - System: `instructions` instead of system message+ - Output: `output[]` array instead of `choices[].message`+ - State: `previous_response_id` for multi-turn+ - Built-in tools: web_search, file_search, code_interpreter+ """+ start_time = time.time()+ request_id = await self._next_request_id()++ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "type": "invalid_request_error",+ "code": "request_too_large",+ }+ },+ )++ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON in request body: {e!s}",+ "type": "invalid_request_error",+ "code": "invalid_json",+ }+ },+ )++ model = body.get("model", "unknown")+ stream = body.get("stream", False)++ # Convert Responses API input to messages format for optimization+ # The Responses API accepts either a string or array of messages+ input_data = body.get("input", "")+ instructions = body.get("instructions")++ messages = []+ if instructions:+ messages.append({"role": "system", "content": instructions})++ if isinstance(input_data, str):+ messages.append({"role": "user", "content": input_data})+ elif isinstance(input_data, list):+ # Input is already an array of message objects+ messages.extend(input_data)++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting+ if self.rate_limiter:+ rate_key = headers.get("authorization", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Token counting on converted messages+ tokenizer = get_tokenizer(model)+ original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)++ # Note: We pass through to OpenAI without optimization for now+ # The Responses API has different semantics that may not work well with compression+ tokens_saved = 0+ transforms_applied: list[str] = []+ optimization_latency = (time.time() - start_time) * 1000++ url = f"{self.OPENAI_API_URL}/v1/responses"++ try:+ if stream:+ # Streaming for Responses API uses semantic events+ return await self._stream_response(+ url,+ headers,+ body,+ "openai",+ model,+ request_id,+ original_tokens,+ original_tokens,+ tokens_saved,+ transforms_applied,+ tags,+ optimization_latency,+ )+ else:+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ output_tokens = 0+ try:+ resp_json = response.json()+ usage = resp_json.get("usage", {})+ output_tokens = usage.get("output_tokens", 0)+ except Exception:+ pass++ # Cost tracking+ cost_usd = savings_usd = None+ if self.cost_tracker:+ cost_usd = self.cost_tracker.estimate_cost(+ model, original_tokens, output_tokens+ )+ if cost_usd:+ self.cost_tracker.record_cost(cost_usd)++ # Metrics+ await self.metrics.record_request(+ provider="openai",+ model=model,+ input_tokens=original_tokens,+ output_tokens=output_tokens,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cost_usd=cost_usd or 0,+ savings_usd=savings_usd or 0,+ )++ logger.info(f"[{request_id}] /v1/responses {model}: {original_tokens:,} tokens")++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,)+ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "type": "server_error",+ "code": "proxy_error",+ }+ },+ )++ async def handle_gemini_generate_content(+ self,+ request: Request,+ model: str,+ ) -> Response | StreamingResponse:+ """Handle Gemini native /v1beta/models/{model}:generateContent endpoint.- # Remove compression headers from cached response- response_headers = dict(cached.response_headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None)+ Gemini's native API differs from OpenAI:+ - Input: `contents[]` with `parts[]` instead of `messages`+ - System: `systemInstruction` instead of system message+ - Auth: `x-goog-api-key` header instead of `Authorization: Bearer`+ - Output: `candidates[].content.parts[].text`+ """+ start_time = time.time()+ request_id = await self._next_request_id()- return Response(content=cached.response_body, headers=response_headers)+ # Check request body size+ content_length = request.headers.get("content-length")+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:+ return JSONResponse(+ status_code=413,+ content={+ "error": {+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",+ "code": 413,+ }+ },+ )++ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON in request body: {e!s}",+ "code": 400,+ }+ },+ )++ contents = body.get("contents", [])++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)++ # Rate limiting (use Gemini API key)+ if self.rate_limiter:+ rate_key = headers.get("x-goog-api-key", "default")[:20]+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)+ if not allowed:+ await self.metrics.record_rate_limited()+ raise HTTPException(+ status_code=429,+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",+ )++ # Convert Gemini format to messages for optimization+ system_instruction = body.get("systemInstruction")+ messages = self._gemini_contents_to_messages(contents, system_instruction)# Token countingtokenizer = get_tokenizer(model)original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)# Optimization- transforms_applied = []+ transforms_applied: list[str] = []optimized_messages = messagesoptimized_tokens = original_tokensif self.config.optimize and messages:try:+ # Use OpenAI pipeline (similar message format)context_limit = self.openai_provider.get_context_limit(model)result = self.openai_pipeline.apply(messages=messages,@@ -1754,45 +3633,47 @@ class HeadroomProxy:tokenizer.count_text(str(m.get("content", ""))) for m in optimized_messages)except Exception as e:- logger.warning(f"Optimization failed: {e}")+ logger.warning(f"[{request_id}] Gemini optimization failed: {e}")tokens_saved = original_tokens - optimized_tokensoptimization_latency = (time.time() - start_time) * 1000- # CCR Tool Injection: Inject retrieval tool if compression occurred- tools = body.get("tools")- if self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions:- injector = CCRToolInjector(- provider="openai",- inject_tool=self.config.ccr_inject_tool,- inject_system_instructions=self.config.ccr_inject_system_instructions,- )- optimized_messages, tools, was_injected = injector.process_request(- optimized_messages, tools+ # Convert back to Gemini format if optimized+ if optimized_messages != messages:+ optimized_contents, optimized_system = self._messages_to_gemini_contents(+ optimized_messages)+ body["contents"] = optimized_contents+ if optimized_system:+ body["systemInstruction"] = optimized_system+ elif "systemInstruction" in body:+ del body["systemInstruction"]- if injector.has_compressed_content:- if was_injected:- logger.debug(- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"- )- else:- logger.debug(- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"- )+ # Build URL - model is extracted from path+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:generateContent"- body["messages"] = optimized_messages- if tools is not None:- body["tools"] = tools- url = f"{self.OPENAI_API_URL}/v1/chat/completions"+ # Check if streaming requested via query param+ query_params = dict(request.query_params)+ is_streaming = query_params.get("alt") == "sse"++ # Preserve API key in query params if present+ if "key" in query_params:+ url += f"?key={query_params['key']}"try:- if stream:+ if is_streaming:+ # For streaming, use streamGenerateContent endpoint+ stream_url = (+ f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ )+ if "key" in query_params:+ stream_url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"+return await self._stream_response(- url,+ stream_url,headers,body,- "openai",+ "gemini",model,request_id,original_tokens,@@ -1809,8 +3690,8 @@ class HeadroomProxy:output_tokens = 0try:resp_json = response.json()- usage = resp_json.get("usage", {})- output_tokens = usage.get("completion_tokens", 0)+ usage = resp_json.get("usageMetadata", {})+ output_tokens = usage.get("candidatesTokenCount", 0)except Exception:pass@@ -1828,15 +3709,9 @@ class HeadroomProxy:self.cost_tracker.record_cost(cost_usd)self.cost_tracker.record_savings(savings_usd)- # Cache- if self.cache and response.status_code == 200:- await self.cache.set(- messages, model, response.content, dict(response.headers), tokens_saved- )-# Metricsawait self.metrics.record_request(- provider="openai",+ provider="gemini",model=model,input_tokens=optimized_tokens,output_tokens=output_tokens,@@ -1848,14 +3723,16 @@ class HeadroomProxy:if tokens_saved > 0:logger.info(- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "+ f"[{request_id}] Gemini {model}: {original_tokens:,} → {optimized_tokens:,} "f"(saved {tokens_saved:,} tokens)")+ else:+ logger.info(f"[{request_id}] Gemini {model}: {original_tokens:,} tokens")- # Remove compression headers since httpx already decompressed the response+ # Remove compression headersresponse_headers = dict(response.headers)response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression+ response_headers.pop("content-length", None)return Response(content=response.content,@@ -1864,47 +3741,218 @@ class HeadroomProxy:)except Exception as e:await self.metrics.record_failed()- # Log full error details internally for debugging- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")- # Return sanitized error message to client (don't expose internal details)+ logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")return JSONResponse(status_code=502,content={"error": {"message": "An error occurred while processing your request. Please try again.",- "type": "server_error",- "code": "proxy_error",+ "code": 502,}},)- async def handle_passthrough(self, request: Request, base_url: str) -> Response:- """Pass through request unchanged."""- path = request.url.path- url = f"{base_url}{path}"+ async def handle_gemini_stream_generate_content(+ self,+ request: Request,+ model: str,+ ) -> StreamingResponse:+ """Handle Gemini streaming endpoint /v1beta/models/{model}:streamGenerateContent."""+ start_time = time.time()+ request_id = await self._next_request_id()++ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON in request body: {e!s}",+ "code": 400,+ }+ },+ )++ contents = body.get("contents", [])headers = dict(request.headers.items())headers.pop("host", None)+ headers.pop("content-length", None)+ tags = self._extract_tags(headers)- body = await request.body()+ # Token counting+ tokenizer = get_tokenizer(model)+ original_tokens = 0+ for content in contents:+ parts = content.get("parts", [])+ for part in parts:+ if "text" in part:+ original_tokens += tokenizer.count_text(part["text"])- response = await self.http_client.request( # type: ignore[union-attr]- method=request.method,- url=url,- headers=headers,- content=body,+ optimization_latency = (time.time() - start_time) * 1000++ # Build URL with SSE param+ query_params = dict(request.query_params)+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?alt=sse"+ if "key" in query_params:+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:streamGenerateContent?key={query_params['key']}&alt=sse"++ return await self._stream_response(+ url,+ headers,+ body,+ "gemini",+ model,+ request_id,+ original_tokens,+ original_tokens,+ 0, # tokens_saved+ [], # transforms_applied+ tags,+ optimization_latency,)- # Remove compression headers since httpx already decompressed the response- response_headers = dict(response.headers)- response_headers.pop("content-encoding", None)- response_headers.pop("content-length", None) # Length changed after decompression+ async def handle_gemini_count_tokens(+ self,+ request: Request,+ model: str,+ ) -> Response:+ """Handle Gemini /v1beta/models/{model}:countTokens endpoint with compression.- return Response(- content=response.content,- status_code=response.status_code,- headers=response_headers,- )+ This endpoint counts tokens AFTER applying compression, so users can see+ how many tokens they'll actually use after optimization.++ The request format is the same as generateContent:+ {"contents": [...], "systemInstruction": {...}}+ """+ start_time = time.time()+ request_id = await self._next_request_id()++ # Parse request+ try:+ body = await request.json()+ except json.JSONDecodeError as e:+ return JSONResponse(+ status_code=400,+ content={+ "error": {+ "message": f"Invalid JSON in request body: {e!s}",+ "code": 400,+ }+ },+ )++ contents = body.get("contents", [])++ headers = dict(request.headers.items())+ headers.pop("host", None)+ headers.pop("content-length", None)++ # Convert Gemini format to messages for optimization+ system_instruction = body.get("systemInstruction")+ messages = self._gemini_contents_to_messages(contents, system_instruction)++ # Token counting (original)+ tokenizer = get_tokenizer(model)+ original_tokens = sum(tokenizer.count_text(str(m.get("content", ""))) for m in messages)++ # Apply compression using the same pipeline as generateContent+ transforms_applied: list[str] = []+ optimized_messages = messages++ if self.config.optimize and messages:+ try:+ context_limit = self.openai_provider.get_context_limit(model)+ result = self.openai_pipeline.apply(+ messages=messages,+ model=model,+ model_limit=context_limit,+ )+ if result.messages != messages:+ optimized_messages = result.messages+ transforms_applied = result.transforms_applied+ except Exception as e:+ logger.warning(f"[{request_id}] Gemini countTokens optimization failed: {e}")++ # Convert back to Gemini format for the API call+ if optimized_messages != messages:+ optimized_contents, optimized_system = self._messages_to_gemini_contents(+ optimized_messages+ )+ body["contents"] = optimized_contents+ if optimized_system:+ body["systemInstruction"] = optimized_system+ elif "systemInstruction" in body:+ del body["systemInstruction"]++ # Build URL+ url = f"{self.GEMINI_API_URL}/v1beta/models/{model}:countTokens"++ # Preserve API key in query params if present+ query_params = dict(request.query_params)+ if "key" in query_params:+ url += f"?key={query_params['key']}"++ try:+ response = await self._retry_request("POST", url, headers, body)+ total_latency = (time.time() - start_time) * 1000++ # Parse response to get token count+ compressed_tokens = 0+ try:+ resp_json = response.json()+ compressed_tokens = resp_json.get("totalTokens", 0)+ except Exception:+ pass++ # Track stats+ tokens_saved = original_tokens - compressed_tokens if compressed_tokens > 0 else 0++ await self.metrics.record_request(+ provider="gemini",+ model=model,+ input_tokens=compressed_tokens,+ output_tokens=0,+ tokens_saved=tokens_saved,+ latency_ms=total_latency,+ cost_usd=0,+ savings_usd=0,+ )++ if tokens_saved > 0:+ logger.info(+ f"[{request_id}] Gemini countTokens {model}: {original_tokens:,} → {compressed_tokens:,} "+ f"(saved {tokens_saved:,} tokens, transforms: {transforms_applied})"+ )+ else:+ logger.info(+ f"[{request_id}] Gemini countTokens {model}: {compressed_tokens:,} tokens"+ )++ # Remove compression headers+ response_headers = dict(response.headers)+ response_headers.pop("content-encoding", None)+ response_headers.pop("content-length", None)++ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=response_headers,+ )+ except Exception as e:+ await self.metrics.record_failed()+ logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "message": "An error occurred while processing your request. Please try again.",+ "code": 502,+ }+ },+ )# =============================================================================@@ -2562,13 +4610,231 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:@app.post("/v1/messages/count_tokens")async def anthropic_count_tokens(request: Request):- return await proxy.handle_passthrough(request, proxy.ANTHROPIC_API_URL)+ return await proxy.handle_passthrough(+ request, proxy.ANTHROPIC_API_URL, "count_tokens", "anthropic"+ )++ # Anthropic Message Batches API endpoints+ @app.post("/v1/messages/batches")+ async def anthropic_batch_create(request: Request):+ """Create a message batch with compression applied to all requests."""+ return await proxy.handle_anthropic_batch_create(request)++ @app.get("/v1/messages/batches")+ async def anthropic_batch_list(request: Request):+ """List message batches (passthrough)."""+ return await proxy.handle_anthropic_batch_passthrough(request)++ @app.get("/v1/messages/batches/{batch_id}")+ async def anthropic_batch_get(request: Request, batch_id: str):+ """Get a specific message batch (passthrough)."""+ return await proxy.handle_anthropic_batch_passthrough(request, batch_id)++ @app.get("/v1/messages/batches/{batch_id}/results")+ async def anthropic_batch_results(request: Request, batch_id: str):+ """Get results for a message batch with CCR post-processing."""+ return await proxy.handle_anthropic_batch_results(request, batch_id)++ @app.post("/v1/messages/batches/{batch_id}/cancel")+ async def anthropic_batch_cancel(request: Request, batch_id: str):+ """Cancel a message batch (passthrough)."""+ return await proxy.handle_anthropic_batch_passthrough(request, batch_id)# OpenAI endpoints@app.post("/v1/chat/completions")async def openai_chat(request: Request):return await proxy.handle_openai_chat(request)+ @app.post("/v1/responses")+ async def openai_responses(request: Request):+ """OpenAI Responses API (new API introduced March 2025)."""+ return await proxy.handle_openai_responses(request)++ # OpenAI Batch API endpoints (with compression!)+ @app.post("/v1/batches")+ async def create_batch(request: Request):+ """Create a batch with automatic compression of messages."""+ return await proxy.handle_batch_create(request)++ @app.get("/v1/batches")+ async def list_batches(request: Request):+ """List batches (passthrough to OpenAI)."""+ return await proxy.handle_batch_list(request)++ @app.get("/v1/batches/{batch_id}")+ async def get_batch(request: Request, batch_id: str):+ """Get batch details (passthrough to OpenAI)."""+ return await proxy.handle_batch_get(request, batch_id)++ @app.post("/v1/batches/{batch_id}/cancel")+ async def cancel_batch(request: Request, batch_id: str):+ """Cancel a batch (passthrough to OpenAI)."""+ return await proxy.handle_batch_cancel(request, batch_id)++ # Gemini native endpoints+ @app.post("/v1beta/models/{model}:generateContent")+ async def gemini_generate_content(request: Request, model: str):+ """Gemini native generateContent API."""+ return await proxy.handle_gemini_generate_content(request, model)++ @app.post("/v1beta/models/{model}:streamGenerateContent")+ async def gemini_stream_generate_content(request: Request, model: str):+ """Gemini native streaming generateContent API."""+ return await proxy.handle_gemini_stream_generate_content(request, model)++ @app.post("/v1beta/models/{model}:countTokens")+ async def gemini_count_tokens(request: Request, model: str):+ """Gemini countTokens API with compression applied."""+ return await proxy.handle_gemini_count_tokens(request, model)++ # =========================================================================+ # Passthrough Endpoints (no compression needed)+ # =========================================================================++ # --- OpenAI Passthrough Endpoints ---++ @app.get("/v1/models")+ async def list_models(request: Request):+ """List models - route based on auth header.++ - x-api-key header present -> Anthropic+ - Authorization: Bearer header -> OpenAI+ """+ if request.headers.get("x-api-key"):+ return await proxy.handle_passthrough(+ request, proxy.ANTHROPIC_API_URL, "models", "anthropic"+ )+ return await proxy.handle_passthrough(request, proxy.OPENAI_API_URL, "models", "openai")++ @app.get("/v1/models/{model_id}")+ async def get_model(request: Request, model_id: str):+ """Get model details - route based on auth header.++ - x-api-key header present -> Anthropic+ - Authorization: Bearer header -> OpenAI+ """+ if request.headers.get("x-api-key"):+ return await proxy.handle_passthrough(+ request, proxy.ANTHROPIC_API_URL, "models", "anthropic"+ )+ return await proxy.handle_passthrough(request, proxy.OPENAI_API_URL, "models", "openai")++ @app.post("/v1/embeddings")+ async def openai_embeddings(request: Request):+ """OpenAI embeddings API - passthrough."""+ return await proxy.handle_passthrough(request, proxy.OPENAI_API_URL, "embeddings", "openai")++ @app.post("/v1/moderations")+ async def openai_moderations(request: Request):+ """OpenAI moderations API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.OPENAI_API_URL, "moderations", "openai"+ )++ @app.post("/v1/images/generations")+ async def openai_images_generations(request: Request):+ """OpenAI image generation API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.OPENAI_API_URL, "images/generations", "openai"+ )++ @app.post("/v1/audio/transcriptions")+ async def openai_audio_transcriptions(request: Request):+ """OpenAI audio transcription API (multipart/form-data) - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.OPENAI_API_URL, "audio/transcriptions", "openai"+ )++ @app.post("/v1/audio/speech")+ async def openai_audio_speech(request: Request):+ """OpenAI text-to-speech API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.OPENAI_API_URL, "audio/speech", "openai"+ )++ # --- Gemini Passthrough Endpoints ---++ @app.get("/v1beta/models")+ async def gemini_list_models(request: Request):+ """Gemini list models API - passthrough."""+ return await proxy.handle_passthrough(request, proxy.GEMINI_API_URL, "models", "gemini")++ @app.get("/v1beta/models/{model_name}")+ async def gemini_get_model(request: Request, model_name: str):+ """Gemini get model API - passthrough.++ Note: This handles GET /v1beta/models/{model_name} but NOT :countTokens+ which is handled by a separate POST route above.+ """+ return await proxy.handle_passthrough(request, proxy.GEMINI_API_URL, "models", "gemini")++ @app.post("/v1beta/models/{model}:embedContent")+ async def gemini_embed_content(request: Request, model: str):+ """Gemini embedding API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.GEMINI_API_URL, "embedContent", "gemini"+ )++ @app.post("/v1beta/models/{model}:batchEmbedContents")+ async def gemini_batch_embed_contents(request: Request, model: str):+ """Gemini batch embeddings API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.GEMINI_API_URL, "batchEmbedContents", "gemini"+ )++ # Google/Gemini Batch API endpoints (with compression!)+ @app.post("/v1beta/models/{model}:batchGenerateContent")+ async def gemini_batch_create(request: Request, model: str):+ """Create a Gemini batch with compression applied to all requests."""+ return await proxy.handle_google_batch_create(request, model)++ @app.get("/v1beta/batches/{batch_name}")+ async def gemini_batch_get(request: Request, batch_name: str):+ """Get a specific Gemini batch with CCR post-processing."""+ return await proxy.handle_google_batch_results(request, batch_name)++ @app.post("/v1beta/batches/{batch_name}:cancel")+ async def gemini_batch_cancel(request: Request, batch_name: str):+ """Cancel a Gemini batch (passthrough)."""+ return await proxy.handle_google_batch_passthrough(request, batch_name)++ @app.delete("/v1beta/batches/{batch_name}")+ async def gemini_batch_delete(request: Request, batch_name: str):+ """Delete a Gemini batch (passthrough)."""+ return await proxy.handle_google_batch_passthrough(request, batch_name)++ @app.post("/v1beta/cachedContents")+ async def gemini_create_cached_content(request: Request):+ """Gemini create cached content API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.GEMINI_API_URL, "cachedContents", "gemini"+ )++ @app.get("/v1beta/cachedContents")+ async def gemini_list_cached_contents(request: Request):+ """Gemini list cached contents API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.GEMINI_API_URL, "cachedContents", "gemini"+ )++ @app.get("/v1beta/cachedContents/{cache_id}")+ async def gemini_get_cached_content(request: Request, cache_id: str):+ """Gemini get cached content API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.GEMINI_API_URL, "cachedContents", "gemini"+ )++ @app.delete("/v1beta/cachedContents/{cache_id}")+ async def gemini_delete_cached_content(request: Request, cache_id: str):+ """Gemini delete cached content API - passthrough."""+ return await proxy.handle_passthrough(+ request, proxy.GEMINI_API_URL, "cachedContents", "gemini"+ )++ # =========================================================================+ # Catch-all Passthrough+ # =========================================================================+# Passthrough - route to correct backend based on headers@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE"])async def passthrough(request: Request, path: str):diff --git a/tests/test_ccr_tool_injection.py b/tests/test_ccr_tool_injection.pyindex 40c2558db..6d3f6a023 100644--- a/tests/test_ccr_tool_injection.py+++ b/tests/test_ccr_tool_injection.py@@ -344,3 +344,87 @@ class TestSystemInstructions:assert "hash0" in instructionsassert "hash4" in instructionsassert "..." in instructions+++class TestAlternativeMarkerFormats:+ """Test CCR marker detection for different compressor formats.++ Different compressors use slightly different marker formats:+ - SmartCrusher: [N items compressed to M. Retrieve more: hash=xxx]+ - TextCompressor: [N lines compressed to M. Retrieve more: hash=xxx]+ - LogCompressor: [N lines compressed to M. Retrieve more: hash=xxx]+ - SearchCompressor: [N matches compressed to M. Retrieve more: hash=xxx]+ - LLMLingua: [N items compressed to M. Retrieve more: hash=xxx]++ The CCRToolInjector should detect all these formats.+ """++ def test_textcompressor_format(self):+ """Detects TextCompressor marker format (lines)."""+ messages = [+ {+ "role": "assistant",+ "content": "Build output:\n[500 lines compressed to 50. Retrieve more: hash=aabbccddeeff00112233]",+ },+ ]++ injector = CCRToolInjector()+ hashes = injector.scan_for_markers(messages)++ assert len(hashes) == 1+ assert "aabbccddeeff00112233" in hashes++ def test_searchcompressor_format(self):+ """Detects SearchCompressor marker format (matches)."""+ messages = [+ {+ "role": "assistant",+ "content": "Search results:\n[100 matches compressed to 10. Retrieve more: hash=1122334455667788]",+ },+ ]++ injector = CCRToolInjector()+ hashes = injector.scan_for_markers(messages)++ assert len(hashes) == 1+ assert "1122334455667788" in hashes++ def test_mixed_compressor_formats(self):+ """Detects multiple marker formats in same conversation."""+ messages = [+ {+ "role": "assistant",+ "content": "Search results:\n[50 matches compressed to 5. Retrieve more: hash=aaaa11111111]",+ },+ {+ "role": "assistant",+ "content": "Build logs:\n[200 lines compressed to 20. Retrieve more: hash=bbbb22222222]",+ },+ {+ "role": "assistant",+ "content": "Database:\n[1000 items compressed to 100. Retrieve more: hash=cccc33333333]",+ },+ ]++ injector = CCRToolInjector()+ hashes = injector.scan_for_markers(messages)++ assert len(hashes) == 3+ assert "aaaa11111111" in hashes+ assert "bbbb22222222" in hashes+ assert "cccc33333333" in hashes++ def test_generic_compressed_marker(self):+ """Detects generic compression markers via fallback pattern."""+ messages = [+ {+ "role": "assistant",+ "content": "Data:\n[Content compressed for efficiency. hash=fedcba9876543210fedcba98]",+ },+ ]++ injector = CCRToolInjector()+ hashes = injector.scan_for_markers(messages)++ assert len(hashes) == 1+ assert "fedcba9876543210fedcba98" in hashesdiff --git a/tests/test_proxy_batch_integration.py b/tests/test_proxy_batch_integration.pynew file mode 100644index 000000000..fa4bc33f0--- /dev/null+++ b/tests/test_proxy_batch_integration.py@@ -0,0 +1,522 @@+"""Integration tests for proxy batch APIs with compression.++These tests verify that batch endpoints work correctly with real API calls+and compression enabled, testing token savings tracking.++Required environment variables:+- OPENAI_API_KEY: For OpenAI /v1/batches endpoint+- ANTHROPIC_API_KEY: For Anthropic /v1/messages/batches endpoint++IMPORTANT: Batch API tests create real batch jobs which may incur costs.+Use sparingly and clean up resources after testing.++Run with:+ OPENAI_API_KEY=... ANTHROPIC_API_KEY=... pytest tests/test_proxy_batch_integration.py -v+"""++import json+import os++import pytest++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++from fastapi.testclient import TestClient++from headroom.proxy.server import ProxyConfig, create_app++# =============================================================================+# Fixtures+# =============================================================================+++def openai_batch_client():+ """Create test client for OpenAI batch API with compression enabled."""+ config = ProxyConfig(+ optimize=True, # Enable compression for batch+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def anthropic_batch_client():+ """Create test client for Anthropic batch API with compression enabled."""+ config = ProxyConfig(+ optimize=True, # Enable compression for batch+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def openai_api_key():+ """Get OpenAI API key from environment."""+ return os.environ.get("OPENAI_API_KEY")+++def anthropic_api_key():+ """Get Anthropic API key from environment."""+ return os.environ.get("ANTHROPIC_API_KEY")+++def create_large_messages(num_items: int = 50) -> list[dict]:+ """Create messages with large JSON data for compression testing."""+ # Create a list of items that will be compressible+ items = [+ {+ "id": i,+ "name": f"Item number {i}",+ "description": f"This is a detailed description for item {i}. It contains additional information.",+ "status": "active" if i % 2 == 0 else "inactive",+ "metadata": {+ "created_at": f"2024-01-{(i % 28) + 1:02d}",+ "updated_at": f"2024-06-{(i % 28) + 1:02d}",+ "tags": [f"tag{i % 5}", f"category{i % 3}"],+ },+ }+ for i in range(num_items)+ ]+ large_json = json.dumps(items, indent=2)++ return [+ {"role": "system", "content": "You are a helpful data analyst assistant."},+ {"role": "user", "content": "I have some data I need you to analyze."},+ {"role": "assistant", "content": f"I've received your data:\n\n{large_json}"},+ {"role": "user", "content": "How many items have status 'active'?"},+ ]+++# =============================================================================+# OpenAI Batch API Tests+# =============================================================================++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestOpenAIBatchCreate:+ """Test OpenAI /v1/batches create endpoint with compression."""++ def test_batch_create_validation_missing_input_file(self, openai_batch_client, openai_api_key):+ """POST /v1/batches without input_file_id returns validation error."""+ response = openai_batch_client.post(+ "/v1/batches",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "endpoint": "/v1/chat/completions",+ "completion_window": "24h",+ },+ )+ assert response.status_code == 400+ data = response.json()+ assert "error" in data+ assert "input_file_id" in data["error"]["message"].lower()++ def test_batch_create_validation_missing_endpoint(self, openai_batch_client, openai_api_key):+ """POST /v1/batches without endpoint returns validation error."""+ response = openai_batch_client.post(+ "/v1/batches",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "input_file_id": "file-abc123",+ "completion_window": "24h",+ },+ )+ assert response.status_code == 400+ data = response.json()+ assert "error" in data+ assert "endpoint" in data["error"]["message"].lower()++ def test_batch_create_with_compression(self, openai_batch_client, openai_api_key):+ """Full batch creation flow with compression.++ This test:+ 1. Creates a JSONL file with compressible content+ 2. Uploads it to OpenAI+ 3. Creates a batch with compression enabled+ 4. Verifies compression stats are tracked+ 5. Cancels the batch to avoid costs+ """+ # Step 1: Create JSONL content with compressible messages+ messages = create_large_messages(num_items=30)+ jsonl_lines = [+ json.dumps(+ {+ "custom_id": f"request-{i}",+ "method": "POST",+ "url": "/v1/chat/completions",+ "body": {+ "model": "gpt-4o-mini",+ "messages": messages,+ "max_tokens": 100,+ },+ }+ )+ for i in range(3) # 3 requests in batch+ ]+ jsonl_content = "\n".join(jsonl_lines)++ # Step 2: Upload the JSONL file directly to OpenAI+ import httpx++ upload_response = httpx.post(+ "https://api.openai.com/v1/files",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ files={"file": ("batch_input.jsonl", jsonl_content.encode(), "application/jsonl")},+ data={"purpose": "batch"},+ )+ assert upload_response.status_code == 200, f"File upload failed: {upload_response.text}"+ file_data = upload_response.json()+ input_file_id = file_data["id"]++ try:+ # Step 3: Create batch through proxy with compression+ response = openai_batch_client.post(+ "/v1/batches",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "input_file_id": input_file_id,+ "endpoint": "/v1/chat/completions",+ "completion_window": "24h",+ "metadata": {"test": "compression_integration"},+ },+ )+ assert response.status_code == 200, f"Batch creation failed: {response.text}"+ batch_data = response.json()++ # Verify batch was created+ assert "id" in batch_data+ assert batch_data["object"] == "batch"+ batch_id = batch_data["id"]++ # Verify compression stats in response headers+ if "x-headroom-tokens-saved" in response.headers:+ tokens_saved = int(response.headers["x-headroom-tokens-saved"])+ assert tokens_saved >= 0++ if "x-headroom-savings-percent" in response.headers:+ savings_percent = float(response.headers["x-headroom-savings-percent"])+ assert 0 <= savings_percent <= 100++ # Verify compression metadata was added+ metadata = batch_data.get("metadata", {})+ if metadata.get("headroom_compressed") == "true":+ # Compression was applied+ assert "headroom_tokens_saved" in metadata+ assert "headroom_original_tokens" in metadata+ assert "headroom_compressed_tokens" in metadata+ tokens_saved = int(metadata["headroom_tokens_saved"])+ assert tokens_saved >= 0++ # Step 4: Cancel the batch to avoid costs+ cancel_response = openai_batch_client.post(+ f"/v1/batches/{batch_id}/cancel",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ )+ # Cancel may succeed or fail if batch already completed/cancelled+ assert cancel_response.status_code in [200, 400]++ finally:+ # Cleanup: Delete the uploaded file+ httpx.delete(+ f"https://api.openai.com/v1/files/{input_file_id}",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ )++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestOpenAIBatchList:+ """Test OpenAI /v1/batches list endpoint passthrough."""++ def test_list_batches(self, openai_batch_client, openai_api_key):+ """GET /v1/batches returns list of batches."""+ response = openai_batch_client.get(+ "/v1/batches",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify list response format+ assert "data" in data+ assert "object" in data+ assert data["object"] == "list"++ def test_list_batches_with_limit(self, openai_batch_client, openai_api_key):+ """GET /v1/batches with limit parameter."""+ response = openai_batch_client.get(+ "/v1/batches?limit=5",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ )+ assert response.status_code == 200+ data = response.json()++ assert len(data["data"]) <= 5+++# =============================================================================+# Anthropic Batch API Tests+# =============================================================================++[email protected](not os.environ.get("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not set")+class TestAnthropicBatchCreate:+ """Test Anthropic /v1/messages/batches create endpoint with compression."""++ def test_batch_create_validation_missing_requests(+ self, anthropic_batch_client, anthropic_api_key+ ):+ """POST /v1/messages/batches without requests returns validation error."""+ response = anthropic_batch_client.post(+ "/v1/messages/batches",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ json={},+ )+ assert response.status_code == 400+ data = response.json()+ assert "error" in data++ def test_batch_create_validation_empty_requests(+ self, anthropic_batch_client, anthropic_api_key+ ):+ """POST /v1/messages/batches with empty requests list returns error."""+ response = anthropic_batch_client.post(+ "/v1/messages/batches",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ json={"requests": []},+ )+ assert response.status_code == 400+ data = response.json()+ assert "error" in data++ def test_batch_create_with_compression(self, anthropic_batch_client, anthropic_api_key):+ """Create Anthropic batch with compression.++ This test:+ 1. Creates a batch request with compressible messages+ 2. Verifies the batch is created successfully+ 3. Checks that compression stats are tracked+ 4. Cancels the batch to avoid costs+ """+ # Create messages with compressible content+ messages = create_large_messages(num_items=25)++ # Create batch request in Anthropic format+ batch_requests = [+ {+ "custom_id": f"req-{i}",+ "params": {+ "model": "claude-3-5-haiku-20241022",+ "max_tokens": 100,+ "messages": messages,+ },+ }+ for i in range(2) # 2 requests in batch+ ]++ response = anthropic_batch_client.post(+ "/v1/messages/batches",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ "content-type": "application/json",+ },+ json={"requests": batch_requests},+ )+ assert response.status_code == 200, f"Batch creation failed: {response.text}"+ batch_data = response.json()++ # Verify batch was created+ assert "id" in batch_data+ assert batch_data["type"] == "message_batch"+ batch_id = batch_data["id"]++ # Verify processing status+ assert "processing_status" in batch_data+ assert batch_data["processing_status"] in ["in_progress", "ended", "canceling"]++ # Check proxy stats for compression+ stats_response = anthropic_batch_client.get("/stats")+ stats = stats_response.json()+ # Batch requests should be tracked+ assert stats["requests"]["total"] >= 1++ # Cancel the batch to avoid costs+ cancel_response = anthropic_batch_client.post(+ f"/v1/messages/batches/{batch_id}/cancel",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ )+ # Cancel may succeed or return error if already processed+ assert cancel_response.status_code in [200, 400, 409]++[email protected](not os.environ.get("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not set")+class TestAnthropicBatchList:+ """Test Anthropic /v1/messages/batches list endpoint passthrough."""++ def test_list_batches(self, anthropic_batch_client, anthropic_api_key):+ """GET /v1/messages/batches returns list of batches."""+ response = anthropic_batch_client.get(+ "/v1/messages/batches",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Verify list response format+ assert "data" in data++ def test_list_batches_with_limit(self, anthropic_batch_client, anthropic_api_key):+ """GET /v1/messages/batches with limit parameter."""+ response = anthropic_batch_client.get(+ "/v1/messages/batches?limit=5",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ )+ assert response.status_code == 200+ data = response.json()++ assert len(data.get("data", [])) <= 5+++# =============================================================================+# Compression Verification Tests+# =============================================================================++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestBatchCompressionStats:+ """Test that batch compression stats are properly tracked."""++ def test_stats_track_batch_requests(self, openai_batch_client, openai_api_key):+ """Verify batch requests update proxy stats correctly."""+ # Get initial stats+ initial_stats = openai_batch_client.get("/stats").json()+ initial_requests = initial_stats["requests"]["total"]++ # Make a batch list request (passthrough)+ openai_batch_client.get(+ "/v1/batches",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ )++ # Verify stats updated+ updated_stats = openai_batch_client.get("/stats").json()+ assert updated_stats["requests"]["total"] >= initial_requests++[email protected](not os.environ.get("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not set")+class TestAnthropicBatchCompressionStats:+ """Test Anthropic batch compression stats tracking."""++ def test_stats_track_anthropic_batch_requests(self, anthropic_batch_client, anthropic_api_key):+ """Verify Anthropic batch requests update proxy stats."""+ # Get initial stats+ initial_stats = anthropic_batch_client.get("/stats").json()+ initial_requests = initial_stats["requests"]["total"]++ # Make a batch list request+ anthropic_batch_client.get(+ "/v1/messages/batches",+ headers={+ "x-api-key": anthropic_api_key,+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ )++ # Verify stats updated+ updated_stats = anthropic_batch_client.get("/stats").json()+ assert updated_stats["requests"]["total"] >= initial_requests+++# =============================================================================+# Error Handling Tests+# =============================================================================+++class TestBatchErrorHandling:+ """Test error handling for batch endpoints."""++ @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+ def test_openai_batch_invalid_file_id(self, openai_batch_client, openai_api_key):+ """Invalid file ID returns appropriate error."""+ response = openai_batch_client.post(+ "/v1/batches",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "input_file_id": "file-nonexistent12345",+ "endpoint": "/v1/chat/completions",+ "completion_window": "24h",+ },+ )+ # Should return error for non-existent file+ assert response.status_code in [400, 404]++ def test_openai_batch_missing_auth(self, openai_batch_client):+ """Missing authentication returns error (401 or 404 depending on routing)."""+ response = openai_batch_client.post(+ "/v1/batches",+ json={+ "input_file_id": "file-abc123",+ "endpoint": "/v1/chat/completions",+ },+ )+ # Proxy may return 404 (no route match) or 401 (auth error)+ assert response.status_code in [401, 404]++ def test_anthropic_batch_missing_auth(self, anthropic_batch_client):+ """Missing authentication returns error (401 or 400 depending on validation)."""+ response = anthropic_batch_client.post(+ "/v1/messages/batches",+ headers={+ "anthropic-version": "2023-06-01",+ "anthropic-beta": "message-batches-2024-09-24",+ },+ json={"requests": []},+ )+ # Proxy may return 400 (validation) or 401 (auth error)+ assert response.status_code in [400, 401]++ @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+ def test_openai_batch_invalid_json(self, openai_batch_client, openai_api_key):+ """Invalid JSON body returns 400."""+ response = openai_batch_client.post(+ "/v1/batches",+ headers={+ "Authorization": f"Bearer {openai_api_key}",+ "Content-Type": "application/json",+ },+ content=b"not valid json",+ )+ assert response.status_code == 400diff --git a/tests/test_proxy_count_tokens_integration.py b/tests/test_proxy_count_tokens_integration.pynew file mode 100644index 000000000..40f53399d--- /dev/null+++ b/tests/test_proxy_count_tokens_integration.py@@ -0,0 +1,500 @@+"""Integration tests for Gemini countTokens endpoint with compression.++These tests verify that the Gemini /v1beta/models/{model}:countTokens endpoint+works correctly with compression enabled, properly counting tokens after+compression is applied.++Required environment variables:+- GEMINI_API_KEY: For Gemini countTokens endpoint++Run with:+ GEMINI_API_KEY=... pytest tests/test_proxy_count_tokens_integration.py -v+"""++import json+import os++import pytest++# Skip entire module if no API key+pytestmark = pytest.mark.skipif(+ not os.environ.get("GEMINI_API_KEY"), reason="GEMINI_API_KEY not set"+)++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++# =============================================================================+# Fixtures+# =============================================================================+++def gemini_client_optimized():+ """Create test client with optimization enabled for Gemini."""+ config = ProxyConfig(+ optimize=True, # Enable compression+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def gemini_client_passthrough():+ """Create test client with optimization disabled (passthrough mode)."""+ config = ProxyConfig(+ optimize=False, # Disable compression+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def api_key():+ """Get Gemini API key from environment."""+ return os.environ.get("GEMINI_API_KEY")+++def create_large_content(num_items: int = 50) -> list[dict]:+ """Create Gemini-format contents with large compressible data."""+ # Create JSON data that can be compressed+ items = [+ {+ "id": i,+ "name": f"Product Item {i}",+ "description": f"This is a detailed description for product item {i}. "+ f"It includes various specifications and features.",+ "price": 99.99 + i * 0.5,+ "category": f"category_{i % 5}",+ "in_stock": i % 2 == 0,+ "metadata": {+ "sku": f"SKU-{i:05d}",+ "weight": f"{i * 0.1:.2f}kg",+ "dimensions": f"{10 + i}x{15 + i}x{5 + i}cm",+ },+ }+ for i in range(num_items)+ ]+ large_json = json.dumps(items, indent=2)++ return [+ {+ "role": "user",+ "parts": [{"text": "I have product data to analyze."}],+ },+ {+ "role": "model",+ "parts": [{"text": f"Here is the product data:\n\n{large_json}"}],+ },+ {+ "role": "user",+ "parts": [{"text": "How many products are in stock?"}],+ },+ ]+++def create_simple_content() -> list[dict]:+ """Create simple Gemini-format contents for basic testing."""+ return [+ {+ "role": "user",+ "parts": [{"text": "What is 2 + 2?"}],+ }+ ]+++# =============================================================================+# Basic countTokens Tests+# =============================================================================+++class TestGeminiCountTokensBasic:+ """Test basic Gemini countTokens functionality."""++ def test_count_tokens_simple_content(self, gemini_client_optimized, api_key):+ """Basic token counting works correctly."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify response format+ assert "totalTokens" in data+ assert isinstance(data["totalTokens"], int)+ assert data["totalTokens"] > 0++ def test_count_tokens_with_system_instruction(self, gemini_client_optimized, api_key):+ """Token counting includes system instruction."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={+ "contents": create_simple_content(),+ "systemInstruction": {"parts": [{"text": "You are a helpful math assistant."}]},+ },+ )+ # Note: systemInstruction may not be supported by all models/versions+ # Accept both success and 400 (if not supported)+ assert response.status_code in [200, 400]+ if response.status_code == 200:+ data = response.json()+ assert "totalTokens" in data+ assert data["totalTokens"] > 0++ def test_count_tokens_multi_turn(self, gemini_client_optimized, api_key):+ """Token counting for multi-turn conversation."""+ contents = [+ {"role": "user", "parts": [{"text": "Hello, my name is Alice."}]},+ {"role": "model", "parts": [{"text": "Nice to meet you, Alice!"}]},+ {"role": "user", "parts": [{"text": "What is my name?"}]},+ ]++ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": contents},+ )+ assert response.status_code == 200+ data = response.json()++ assert data["totalTokens"] > 0+++# =============================================================================+# Compression Tests+# =============================================================================+++class TestGeminiCountTokensCompression:+ """Test that compression reduces token count."""++ def test_compression_reduces_token_count(+ self, gemini_client_optimized, gemini_client_passthrough, api_key+ ):+ """Verify compression reduces token count for large content.++ This test compares token counts between:+ - Passthrough mode (no compression)+ - Optimized mode (compression enabled)+ """+ large_contents = create_large_content(num_items=40)++ # Get token count without compression+ passthrough_response = gemini_client_passthrough.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": large_contents},+ )+ assert passthrough_response.status_code == 200+ passthrough_tokens = passthrough_response.json()["totalTokens"]++ # Get token count with compression+ optimized_response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": large_contents},+ )+ assert optimized_response.status_code == 200+ optimized_tokens = optimized_response.json()["totalTokens"]++ # Compression should reduce token count (or at least not increase it)+ # Note: compression effect depends on content and may vary+ assert optimized_tokens <= passthrough_tokens * 1.1 # Allow 10% margin++ # For large content, we expect some savings+ if passthrough_tokens > 1000:+ assert optimized_tokens < passthrough_tokens, (+ f"Expected compression to reduce tokens from {passthrough_tokens} "+ f"but got {optimized_tokens}"+ )++ def test_compression_stats_tracked(self, gemini_client_optimized, api_key):+ """Verify compression stats are tracked in proxy stats."""+ large_contents = create_large_content(num_items=30)++ # Make countTokens request with large content+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": large_contents},+ )+ assert response.status_code == 200++ # Check proxy stats+ stats_response = gemini_client_optimized.get("/stats")+ assert stats_response.status_code == 200+ stats = stats_response.json()++ # Verify Gemini requests are tracked+ assert stats["requests"]["total"] >= 1+ assert "gemini" in stats["requests"]["by_provider"]+++class TestGeminiCountTokensLargeContent:+ """Test countTokens with large content that benefits from compression."""++ def test_very_large_json_content(self, gemini_client_optimized, api_key):+ """Token counting handles very large JSON content."""+ large_contents = create_large_content(num_items=100)++ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": large_contents},+ )+ assert response.status_code == 200+ data = response.json()++ assert "totalTokens" in data+ assert data["totalTokens"] > 0++ def test_repeated_data_compression(self, gemini_client_optimized, api_key):+ """Content with repeated patterns compresses well."""+ # Create content with highly repetitive data+ repeated_items = [{"id": i, "status": "active", "type": "item"} for i in range(200)]+ repeated_json = json.dumps(repeated_items)++ contents = [+ {"role": "user", "parts": [{"text": "Analyze this data."}]},+ {"role": "model", "parts": [{"text": f"Data:\n{repeated_json}"}]},+ {"role": "user", "parts": [{"text": "Count the items."}]},+ ]++ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": contents},+ )+ assert response.status_code == 200+ data = response.json()++ assert data["totalTokens"] > 0++ def test_code_content_compression(self, gemini_client_optimized, api_key):+ """Token counting handles code content."""+ code_sample = '''+def calculate_statistics(data):+ """Calculate statistics for the given data."""+ if not data:+ return {"count": 0, "sum": 0, "average": 0}++ count = len(data)+ total = sum(data)+ average = total / count++ return {+ "count": count,+ "sum": total,+ "average": average,+ "min": min(data),+ "max": max(data),+ }++# Example usage+numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]+result = calculate_statistics(numbers)+print(result)+'''++ contents = [+ {"role": "user", "parts": [{"text": "Can you explain this code?"}]},+ {+ "role": "model",+ "parts": [{"text": f"Here's the code:\n\n```python\n{code_sample}\n```"}],+ },+ {"role": "user", "parts": [{"text": "What does calculate_statistics return?"}]},+ ]++ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": contents},+ )+ assert response.status_code == 200+ data = response.json()++ assert data["totalTokens"] > 0+++# =============================================================================+# Model Variant Tests+# =============================================================================+++class TestGeminiCountTokensModels:+ """Test countTokens with different Gemini models."""++ def test_gemini_flash_model(self, gemini_client_optimized, api_key):+ """countTokens works with gemini-2.0-flash model."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )+ assert response.status_code == 200+ assert "totalTokens" in response.json()++ def test_gemini_flash_lite_model(self, gemini_client_optimized, api_key):+ """countTokens works with gemini-2.0-flash-lite model."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash-lite:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )+ # Model may or may not be available+ assert response.status_code in [200, 404]+ if response.status_code == 200:+ assert "totalTokens" in response.json()+++# =============================================================================+# Error Handling Tests+# =============================================================================+++class TestGeminiCountTokensErrors:+ """Test error handling for countTokens endpoint."""++ def test_invalid_api_key(self, gemini_client_optimized):+ """Invalid API key returns authentication error."""+ response = gemini_client_optimized.post(+ "/v1beta/models/gemini-2.0-flash:countTokens?key=invalid-key-12345",+ json={"contents": create_simple_content()},+ )+ assert response.status_code in [400, 401, 403]++ def test_invalid_model(self, gemini_client_optimized, api_key):+ """Invalid model name returns error."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/nonexistent-model-xyz:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )+ assert response.status_code >= 400++ def test_empty_contents(self, gemini_client_optimized, api_key):+ """Empty contents may return error or zero tokens."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": []},+ )+ # May return error or success with 0 tokens+ if response.status_code == 200:+ data = response.json()+ assert "totalTokens" in data++ def test_invalid_json_body(self, gemini_client_optimized, api_key):+ """Invalid JSON body returns 400 error."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ headers={"Content-Type": "application/json"},+ content=b"not valid json",+ )+ assert response.status_code == 400++ def test_missing_contents_field(self, gemini_client_optimized, api_key):+ """Missing contents field handled gracefully."""+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={},+ )+ # May return error or handle empty contents+ assert response.status_code in [200, 400]+++# =============================================================================+# Stats Tracking Tests+# =============================================================================+++class TestGeminiCountTokensStats:+ """Test proxy stats tracking for countTokens requests."""++ def test_stats_track_gemini_provider(self, gemini_client_optimized, api_key):+ """Stats correctly track Gemini provider."""+ # Clear stats by getting a fresh client+ gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )++ stats = gemini_client_optimized.get("/stats").json()+ assert "gemini" in stats["requests"]["by_provider"]+ assert stats["requests"]["by_provider"]["gemini"] >= 1++ def test_stats_track_model(self, gemini_client_optimized, api_key):+ """Stats correctly track model used."""+ gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )++ stats = gemini_client_optimized.get("/stats").json()+ # Model should be tracked in by_model+ assert len(stats["requests"]["by_model"]) >= 1++ def test_stats_track_tokens_saved(self, gemini_client_optimized, api_key):+ """Stats track tokens saved from compression."""+ # Make request with large compressible content+ large_contents = create_large_content(num_items=30)+ gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": large_contents},+ )++ stats = gemini_client_optimized.get("/stats").json()+ # tokens.saved should be tracked (may be 0 if content wasn't compressed)+ assert "tokens" in stats+ assert "saved" in stats["tokens"]+++# =============================================================================+# Integration Tests+# =============================================================================+++class TestGeminiCountTokensIntegration:+ """Integration tests combining multiple features."""++ def test_full_workflow(self, gemini_client_optimized, api_key):+ """Test complete workflow: count tokens, verify compression, check stats."""+ # Step 1: Count tokens with large content+ large_contents = create_large_content(num_items=35)++ initial_stats = gemini_client_optimized.get("/stats").json()+ initial_tokens_saved = initial_stats["tokens"]["saved"]++ # Step 2: Make countTokens request+ response = gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": large_contents},+ )+ assert response.status_code == 200+ token_count = response.json()["totalTokens"]+ assert token_count > 0++ # Step 3: Verify stats updated+ updated_stats = gemini_client_optimized.get("/stats").json()+ assert updated_stats["requests"]["total"] > initial_stats["requests"]["total"]++ # Step 4: Verify tokens saved is tracked (may be negative for small overhead)+ # Allow for some compression overhead+ assert updated_stats["tokens"]["saved"] >= initial_tokens_saved - 100++ def test_multiple_requests_accumulate_stats(self, gemini_client_optimized, api_key):+ """Multiple requests correctly accumulate stats."""+ initial_stats = gemini_client_optimized.get("/stats").json()+ initial_total = initial_stats["requests"]["total"]++ # Make several requests+ for _ in range(3):+ gemini_client_optimized.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": create_simple_content()},+ )++ updated_stats = gemini_client_optimized.get("/stats").json()+ assert updated_stats["requests"]["total"] >= initial_total + 3diff --git a/tests/test_proxy_gemini_integration.py b/tests/test_proxy_gemini_integration.pynew file mode 100644index 000000000..0aba8d09d--- /dev/null+++ b/tests/test_proxy_gemini_integration.py@@ -0,0 +1,254 @@+"""Integration tests for the proxy with real Gemini API calls.++These tests require a valid GEMINI_API_KEY environment variable.+They test the actual /v1/chat/completions endpoint with real API calls.++Run with:+ GEMINI_API_KEY=your-key pytest tests/test_proxy_gemini_integration.py -v+"""++import json+import os++import pytest++# Skip entire module if no API key+pytestmark = pytest.mark.skipif(+ not os.environ.get("GEMINI_API_KEY"), reason="GEMINI_API_KEY not set"+)++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai"+++def gemini_client():+ """Create test client configured to forward to Gemini."""+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ openai_api_url=GEMINI_BASE_URL,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def api_key():+ """Get Gemini API key from environment."""+ return os.environ.get("GEMINI_API_KEY")+++class TestGeminiChatCompletions:+ """Test /v1/chat/completions with real Gemini API."""++ def test_basic_completion(self, gemini_client, api_key):+ """Basic chat completion works."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gemini-2.0-flash",+ "messages": [+ {"role": "user", "content": "What is 2+2? Reply with just the number."}+ ],+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Verify OpenAI-compatible response format+ assert "choices" in data+ assert len(data["choices"]) > 0+ assert "message" in data["choices"][0]+ assert "content" in data["choices"][0]["message"]+ assert "4" in data["choices"][0]["message"]["content"]++ # Verify usage stats+ assert "usage" in data+ assert "prompt_tokens" in data["usage"]+ assert "completion_tokens" in data["usage"]++ def test_multi_turn_conversation(self, gemini_client, api_key):+ """Multi-turn conversations maintain context."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gemini-2.0-flash",+ "messages": [+ {"role": "system", "content": "You are a helpful assistant. Be concise."},+ {"role": "user", "content": "My name is TestUser123."},+ {"role": "assistant", "content": "Nice to meet you, TestUser123!"},+ {"role": "user", "content": "What is my name?"},+ ],+ },+ )+ assert response.status_code == 200+ data = response.json()++ content = data["choices"][0]["message"]["content"].lower()+ assert "testuser123" in content++ def test_streaming(self, gemini_client, api_key):+ """Streaming responses work correctly."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gemini-2.0-flash",+ "stream": True,+ "messages": [{"role": "user", "content": "Count from 1 to 3."}],+ },+ )+ assert response.status_code == 200++ # Parse SSE stream+ chunks = []+ for line in response.text.strip().split("\n"):+ if line.startswith("data: ") and line != "data: [DONE]":+ chunk = json.loads(line[6:])+ chunks.append(chunk)++ assert len(chunks) > 0++ # Verify chunk format+ for chunk in chunks:+ assert "choices" in chunk+ assert "delta" in chunk["choices"][0]+ assert chunk["object"] == "chat.completion.chunk"++ def test_function_calling(self, gemini_client, api_key):+ """Function calling / tools work correctly."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gemini-2.0-flash",+ "messages": [{"role": "user", "content": "What is the weather in Paris?"}],+ "tools": [+ {+ "type": "function",+ "function": {+ "name": "get_weather",+ "description": "Get the weather for a location",+ "parameters": {+ "type": "object",+ "properties": {+ "location": {"type": "string", "description": "City name"}+ },+ "required": ["location"],+ },+ },+ }+ ],+ "tool_choice": "auto",+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Verify tool call response+ message = data["choices"][0]["message"]+ assert "tool_calls" in message+ assert len(message["tool_calls"]) > 0++ tool_call = message["tool_calls"][0]+ assert tool_call["function"]["name"] == "get_weather"+ assert "paris" in tool_call["function"]["arguments"].lower()++ def test_json_mode(self, gemini_client, api_key):+ """JSON response format works."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gemini-2.0-flash",+ "messages": [+ {+ "role": "user",+ "content": "Return a JSON object with keys 'name' and 'age' for a person named Alice who is 30 years old.",+ }+ ],+ "response_format": {"type": "json_object"},+ },+ )+ assert response.status_code == 200+ data = response.json()++ content = data["choices"][0]["message"]["content"]+ # Parse the response as JSON+ parsed = json.loads(content)+ assert "name" in parsed or "Name" in parsed+ assert "age" in parsed or "Age" in parsed+++class TestGeminiModels:+ """Test /v1/models endpoint with Gemini."""++ def test_list_models(self, gemini_client, api_key):+ """Can list available models."""+ response = gemini_client.get("/v1/models", headers={"Authorization": f"Bearer {api_key}"})+ # This goes through passthrough handler+ assert response.status_code == 200+ data = response.json()++ assert "data" in data or "object" in data+++class TestProxyStats:+ """Test that proxy stats track Gemini requests correctly."""++ def test_stats_track_requests(self, gemini_client, api_key):+ """Proxy stats track Gemini requests."""+ # Make a request+ gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={"model": "gemini-2.0-flash", "messages": [{"role": "user", "content": "Hi"}]},+ )++ # Check stats+ stats_response = gemini_client.get("/stats")+ assert stats_response.status_code == 200+ stats = stats_response.json()++ assert stats["requests"]["total"] >= 1+ assert stats["requests"]["by_provider"]["openai"] >= 1+ assert "gemini" in str(stats["requests"]["by_model"]).lower()+++class TestErrorHandling:+ """Test error handling with Gemini."""++ def test_invalid_api_key(self, gemini_client):+ """Invalid API key returns appropriate error."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": "Bearer invalid-key-123"},+ json={"model": "gemini-2.0-flash", "messages": [{"role": "user", "content": "Hi"}]},+ )+ # Should return 4xx error+ assert response.status_code >= 400++ def test_invalid_model(self, gemini_client, api_key):+ """Invalid model returns appropriate error."""+ response = gemini_client.post(+ "/v1/chat/completions",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "nonexistent-model-xyz",+ "messages": [{"role": "user", "content": "Hi"}],+ },+ )+ # Should return 4xx error+ assert response.status_code >= 400diff --git a/tests/test_proxy_gemini_native_integration.py b/tests/test_proxy_gemini_native_integration.pynew file mode 100644index 000000000..ca05a56af--- /dev/null+++ b/tests/test_proxy_gemini_native_integration.py@@ -0,0 +1,374 @@+"""Integration tests for Gemini native API endpoint with real API calls.++These tests require a valid GEMINI_API_KEY environment variable.+They test the /v1beta/models/{model}:generateContent endpoint with compression.++Run with:+ GEMINI_API_KEY=your-key pytest tests/test_proxy_gemini_native_integration.py -v+"""++import json+import os++import pytest++# Skip entire module if no API key+pytestmark = pytest.mark.skipif(+ not os.environ.get("GEMINI_API_KEY"), reason="GEMINI_API_KEY not set"+)++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402+++def gemini_native_client():+ """Create test client for Gemini native API with optimization enabled."""+ config = ProxyConfig(+ optimize=True, # Enable compression+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def api_key():+ """Get Gemini API key from environment."""+ return os.environ.get("GEMINI_API_KEY")+++class TestGeminiNativeGenerateContent:+ """Test /v1beta/models/{model}:generateContent endpoint."""++ def test_basic_generation(self, gemini_native_client, api_key):+ """Basic text generation works."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={"contents": [{"parts": [{"text": "What is 2+2? Reply with just the number."}]}]},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify Gemini native response format+ assert "candidates" in data+ assert len(data["candidates"]) > 0+ assert "content" in data["candidates"][0]+ assert "parts" in data["candidates"][0]["content"]+ text = data["candidates"][0]["content"]["parts"][0]["text"]+ assert "4" in text++ # Verify usage metadata+ assert "usageMetadata" in data+ assert "promptTokenCount" in data["usageMetadata"]++ def test_with_system_instruction(self, gemini_native_client, api_key):+ """System instruction works correctly."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={+ "contents": [{"parts": [{"text": "Hello"}]}],+ "systemInstruction": {"parts": [{"text": "Always respond with exactly one word."}]},+ },+ )+ assert response.status_code == 200+ data = response.json()+ text = data["candidates"][0]["content"]["parts"][0]["text"]+ # Should be a short response due to system instruction+ assert len(text.split()) <= 3++ def test_multi_turn_conversation(self, gemini_native_client, api_key):+ """Multi-turn conversations maintain context."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={+ "contents": [+ {"role": "user", "parts": [{"text": "My name is TestUser456."}]},+ {"role": "model", "parts": [{"text": "Nice to meet you, TestUser456!"}]},+ {"role": "user", "parts": [{"text": "What is my name?"}]},+ ]+ },+ )+ assert response.status_code == 200+ data = response.json()+ text = data["candidates"][0]["content"]["parts"][0]["text"].lower()+ assert "testuser456" in text++ def test_function_calling(self, gemini_native_client, api_key):+ """Function calling / tools work correctly."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={+ "contents": [{"parts": [{"text": "What is the weather in Tokyo?"}]}],+ "tools": [+ {+ "functionDeclarations": [+ {+ "name": "get_weather",+ "description": "Get current weather for a location",+ "parameters": {+ "type": "object",+ "properties": {+ "location": {"type": "string", "description": "City name"}+ },+ "required": ["location"],+ },+ }+ ]+ }+ ],+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Verify function call response+ parts = data["candidates"][0]["content"]["parts"]+ function_call = None+ for part in parts:+ if "functionCall" in part:+ function_call = part["functionCall"]+ break++ assert function_call is not None+ assert function_call["name"] == "get_weather"+ assert "tokyo" in function_call["args"]["location"].lower()++ def test_generation_config(self, gemini_native_client, api_key):+ """Generation config parameters are respected."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={+ "contents": [{"parts": [{"text": "Write a very short poem about AI."}]}],+ "generationConfig": {"maxOutputTokens": 50, "temperature": 0.1},+ },+ )+ assert response.status_code == 200+ data = response.json()+ # Response should be limited by maxOutputTokens+ assert data["usageMetadata"]["candidatesTokenCount"] <= 60 # Some buffer+++class TestGeminiNativeCompression:+ """Test that compression works with Gemini native API."""++ def test_compression_on_model_message(self, gemini_native_client, api_key):+ """Large data in model message gets compressed."""+ # Create large JSON data (simulating tool output)+ items = [+ {"id": i, "name": f"Item {i}", "desc": f"Description for item {i}"} for i in range(100)+ ]+ tool_output = json.dumps(items)++ # Send as model message (like tool returning data)+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={+ "contents": [+ {"role": "user", "parts": [{"text": "Get items from database"}]},+ {"role": "model", "parts": [{"text": f"Here are the results:\n{tool_output}"}]},+ {"role": "user", "parts": [{"text": "How many items are there?"}]},+ ]+ },+ )+ assert response.status_code == 200+ data = response.json()+ text = data["candidates"][0]["content"]["parts"][0]["text"]+ # Model should correctly count the items+ assert "100" in text++ # Check that compression happened via stats+ stats = gemini_native_client.get("/stats").json()+ # At least some tokens should have been saved+ assert stats["tokens"]["saved"] >= 0 # May or may not compress depending on size++ def test_user_messages_protected(self, gemini_native_client, api_key):+ """User messages are not compressed (by design)."""+ # Large data in user message+ items = [{"id": i} for i in range(50)]+ user_data = json.dumps(items)++ # First request with data in user message+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={+ "contents": [+ {"role": "user", "parts": [{"text": f"Analyze this data: {user_data}"}]}+ ]+ },+ )+ assert response.status_code == 200+ # The request should succeed - user messages are protected from compression+++class TestGeminiNativeStats:+ """Test that proxy stats track Gemini native requests correctly."""++ def test_stats_track_gemini_provider(self, gemini_native_client, api_key):+ """Stats show requests under 'gemini' provider."""+ # Make a request+ gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={"contents": [{"parts": [{"text": "Hi"}]}]},+ )++ stats = gemini_native_client.get("/stats").json()+ assert "gemini" in stats["requests"]["by_provider"]+ assert stats["requests"]["by_provider"]["gemini"] >= 1++ def test_stats_track_model(self, gemini_native_client, api_key):+ """Stats track the specific model used."""+ gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}",+ json={"contents": [{"parts": [{"text": "Hi"}]}]},+ )++ stats = gemini_native_client.get("/stats").json()+ assert "gemini-2.0-flash" in stats["requests"]["by_model"]+++class TestGeminiNativeErrorHandling:+ """Test error handling for Gemini native API."""++ def test_invalid_api_key(self, gemini_native_client):+ """Invalid API key returns appropriate error."""+ response = gemini_native_client.post(+ "/v1beta/models/gemini-2.0-flash:generateContent?key=invalid-key-123",+ json={"contents": [{"parts": [{"text": "Hi"}]}]},+ )+ assert response.status_code >= 400++ def test_invalid_model(self, gemini_native_client, api_key):+ """Invalid model returns appropriate error."""+ response = gemini_native_client.post(+ f"/v1beta/models/nonexistent-model-xyz:generateContent?key={api_key}",+ json={"contents": [{"parts": [{"text": "Hi"}]}]},+ )+ assert response.status_code >= 400++ def test_empty_contents(self, gemini_native_client, api_key):+ """Empty contents handled gracefully."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:generateContent?key={api_key}", json={"contents": []}+ )+ # Should either return error or handle gracefully+ assert response.status_code in [200, 400]+++class TestGeminiNativeHeaderAuth:+ """Test authentication via x-goog-api-key header."""++ def test_header_auth(self, gemini_native_client, api_key):+ """API key in header works."""+ response = gemini_native_client.post(+ "/v1beta/models/gemini-2.0-flash:generateContent",+ headers={"x-goog-api-key": api_key},+ json={"contents": [{"parts": [{"text": "Hi"}]}]},+ )+ assert response.status_code == 200+++class TestGeminiNativeCountTokens:+ """Test /v1beta/models/{model}:countTokens endpoint with compression."""++ def test_count_tokens_basic(self, gemini_native_client, api_key):+ """Basic token counting works."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={"contents": [{"parts": [{"text": "Hello, world!"}]}]},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify response format+ assert "totalTokens" in data+ assert isinstance(data["totalTokens"], int)+ assert data["totalTokens"] > 0++ def test_count_tokens_with_system_instruction(self, gemini_native_client, api_key):+ """Token counting includes system instruction."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={+ "contents": [{"parts": [{"text": "Hello"}]}],+ "systemInstruction": {"parts": [{"text": "You are a helpful assistant."}]},+ },+ )+ # Note: systemInstruction may not be supported by countTokens in all versions+ assert response.status_code in [200, 400]+ if response.status_code == 200:+ data = response.json()+ assert "totalTokens" in data+ assert data["totalTokens"] > 0++ def test_count_tokens_reflects_compression(self, gemini_native_client, api_key):+ """Token count reflects compressed content size."""+ # Create large repetitive JSON data that should compress+ items = [+ {+ "id": i,+ "name": f"Item {i}",+ "description": f"This is the description for item number {i}",+ }+ for i in range(100)+ ]+ tool_output = json.dumps(items)++ # Count tokens with large data in model message (which gets compressed)+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={+ "contents": [+ {"role": "user", "parts": [{"text": "Get items from database"}]},+ {"role": "model", "parts": [{"text": f"Here are the results:\n{tool_output}"}]},+ {"role": "user", "parts": [{"text": "Summarize these items"}]},+ ]+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Verify we got a token count+ assert "totalTokens" in data+ compressed_tokens = data["totalTokens"]+ assert compressed_tokens > 0++ # Check stats to verify compression was applied+ stats = gemini_native_client.get("/stats").json()+ # The request should have been tracked+ assert stats["requests"]["by_provider"].get("gemini", 0) >= 1++ def test_count_tokens_multi_turn(self, gemini_native_client, api_key):+ """Token counting works for multi-turn conversations."""+ response = gemini_native_client.post(+ f"/v1beta/models/gemini-2.0-flash:countTokens?key={api_key}",+ json={+ "contents": [+ {"role": "user", "parts": [{"text": "My name is Alice."}]},+ {"role": "model", "parts": [{"text": "Nice to meet you, Alice!"}]},+ {"role": "user", "parts": [{"text": "What is my name?"}]},+ ]+ },+ )+ assert response.status_code == 200+ data = response.json()+ assert "totalTokens" in data+ assert data["totalTokens"] > 0++ def test_count_tokens_header_auth(self, gemini_native_client, api_key):+ """API key in header works for countTokens."""+ response = gemini_native_client.post(+ "/v1beta/models/gemini-2.0-flash:countTokens",+ headers={"x-goog-api-key": api_key},+ json={"contents": [{"parts": [{"text": "Hello"}]}]},+ )+ assert response.status_code == 200+ data = response.json()+ assert "totalTokens" in datadiff --git a/tests/test_proxy_openai_responses_integration.py b/tests/test_proxy_openai_responses_integration.pynew file mode 100644index 000000000..74f8a8504--- /dev/null+++ b/tests/test_proxy_openai_responses_integration.py@@ -0,0 +1,280 @@+"""Integration tests for OpenAI /v1/responses endpoint with real API calls.++These tests require a valid OPENAI_API_KEY environment variable.+They test the /v1/responses endpoint (introduced March 2025) with compression.++Run with:+ OPENAI_API_KEY=your-key pytest tests/test_proxy_openai_responses_integration.py -v+"""++import json+import os++import pytest++# Skip entire module if no API key+pytestmark = pytest.mark.skipif(+ not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set"+)++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.proxy.server import ProxyConfig, create_app # noqa: E402+++def openai_responses_client():+ """Create test client for OpenAI responses API with optimization enabled."""+ config = ProxyConfig(+ optimize=True, # Enable compression+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def api_key():+ """Get OpenAI API key from environment."""+ return os.environ.get("OPENAI_API_KEY")+++class TestOpenAIResponsesBasic:+ """Test /v1/responses endpoint basic functionality."""++ def test_basic_generation(self, openai_responses_client, api_key):+ """Basic text generation works."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={"model": "gpt-4o-mini", "input": "What is 2+2? Reply with just the number."},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify responses API format+ assert "id" in data+ assert "output" in data+ assert len(data["output"]) > 0+ assert data["output"][0]["type"] == "message"+ assert data["output"][0]["role"] == "assistant"++ # Get the text content+ content = data["output"][0]["content"]+ assert len(content) > 0+ text = content[0].get("text", "")+ assert "4" in text++ # Verify usage metadata+ assert "usage" in data++ def test_with_instructions(self, openai_responses_client, api_key):+ """System instructions work correctly."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gpt-4o-mini",+ "input": "Hello",+ "instructions": "Always respond with exactly one word.",+ },+ )+ assert response.status_code == 200+ data = response.json()++ content = data["output"][0]["content"]+ text = content[0].get("text", "")+ # Should be a short response due to instructions+ assert len(text.split()) <= 3++ def test_input_as_array(self, openai_responses_client, api_key):+ """Input can be an array of messages."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gpt-4o-mini",+ "input": [+ {"role": "user", "content": "My name is TestUser789."},+ {"role": "assistant", "content": "Nice to meet you, TestUser789!"},+ {"role": "user", "content": "What is my name?"},+ ],+ },+ )+ assert response.status_code == 200+ data = response.json()++ content = data["output"][0]["content"]+ text = content[0].get("text", "").lower()+ assert "testuser789" in text++ def test_generation_parameters(self, openai_responses_client, api_key):+ """Generation parameters are respected."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gpt-4o-mini",+ "input": "Write a very short poem about AI.",+ "max_output_tokens": 50,+ "temperature": 0.1,+ },+ )+ assert response.status_code == 200+ data = response.json()+ # Response should be limited by max_output_tokens+ assert data["usage"]["output_tokens"] <= 60 # Some buffer+++class TestOpenAIResponsesTools:+ """Test function calling / tools with /v1/responses endpoint."""++ def test_function_calling(self, openai_responses_client, api_key):+ """Function calling works correctly."""+ # Note: /v1/responses uses a different tools format than /v1/chat/completions+ # - name, description, parameters are at top level, not nested under "function"+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gpt-4o-mini",+ "input": "What is the weather in Tokyo?",+ "tools": [+ {+ "type": "function",+ "name": "get_weather",+ "description": "Get current weather for a location",+ "parameters": {+ "type": "object",+ "properties": {+ "location": {"type": "string", "description": "City name"}+ },+ "required": ["location"],+ },+ }+ ],+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Find tool call in output+ output = data["output"]+ tool_call_found = False+ for item in output:+ if item.get("type") == "function_call":+ tool_call_found = True+ assert item["name"] == "get_weather"+ args = (+ json.loads(item["arguments"])+ if isinstance(item["arguments"], str)+ else item["arguments"]+ )+ assert "tokyo" in args.get("location", "").lower()+ break++ assert tool_call_found, "Expected function_call in output"+++class TestOpenAIResponsesCompression:+ """Test that compression works with /v1/responses endpoint."""++ def test_compression_on_assistant_message(self, openai_responses_client, api_key):+ """Large data in assistant message gets compressed."""+ # Create large JSON data (simulating tool output)+ items = [+ {"id": i, "name": f"Item {i}", "desc": f"Description for item {i}"} for i in range(100)+ ]+ tool_output = json.dumps(items)++ # Send as multi-turn with assistant message containing data+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={+ "model": "gpt-4o-mini",+ "input": [+ {"role": "user", "content": "Get items from database"},+ {"role": "assistant", "content": f"Here are the results:\n{tool_output}"},+ {"role": "user", "content": "How many items are there?"},+ ],+ },+ )+ assert response.status_code == 200+ data = response.json()++ content = data["output"][0]["content"]+ text = content[0].get("text", "")+ # Model should correctly count the items+ assert "100" in text++ # Check that compression happened via stats+ stats = openai_responses_client.get("/stats").json()+ # At least some tokens should have been saved+ assert stats["tokens"]["saved"] >= 0 # May or may not compress depending on size+++class TestOpenAIResponsesStats:+ """Test that proxy stats track /v1/responses requests correctly."""++ def test_stats_track_openai_provider(self, openai_responses_client, api_key):+ """Stats show requests under 'openai' provider."""+ # Make a request+ openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={"model": "gpt-4o-mini", "input": "Hi"},+ )++ stats = openai_responses_client.get("/stats").json()+ assert "openai" in stats["requests"]["by_provider"]+ assert stats["requests"]["by_provider"]["openai"] >= 1++ def test_stats_track_model(self, openai_responses_client, api_key):+ """Stats track the specific model used."""+ openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={"model": "gpt-4o-mini", "input": "Hi"},+ )++ stats = openai_responses_client.get("/stats").json()+ assert "gpt-4o-mini" in stats["requests"]["by_model"]+++class TestOpenAIResponsesErrorHandling:+ """Test error handling for /v1/responses endpoint."""++ def test_invalid_api_key(self, openai_responses_client):+ """Invalid API key returns appropriate error."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": "Bearer invalid-key-123"},+ json={"model": "gpt-4o-mini", "input": "Hi"},+ )+ assert response.status_code >= 400++ def test_invalid_model(self, openai_responses_client, api_key):+ """Invalid model returns appropriate error."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={"model": "nonexistent-model-xyz", "input": "Hi"},+ )+ assert response.status_code >= 400++ def test_missing_input(self, openai_responses_client, api_key):+ """Missing input handled gracefully."""+ response = openai_responses_client.post(+ "/v1/responses",+ headers={"Authorization": f"Bearer {api_key}"},+ json={"model": "gpt-4o-mini"},+ )+ # Should either return error or handle gracefully+ assert response.status_code in [200, 400, 422]diff --git a/tests/test_proxy_passthrough_integration.py b/tests/test_proxy_passthrough_integration.pynew file mode 100644index 000000000..872d4e87a--- /dev/null+++ b/tests/test_proxy_passthrough_integration.py@@ -0,0 +1,486 @@+"""Integration tests for proxy passthrough endpoints with real API calls.++These tests verify that passthrough endpoints work correctly with real API calls+to OpenAI, Gemini, and Anthropic APIs.++Required environment variables:+- OPENAI_API_KEY: For OpenAI /v1/models, /v1/embeddings, /v1/moderations+- GEMINI_API_KEY: For Gemini /v1beta/models, :embedContent+- ANTHROPIC_API_KEY: For Anthropic /v1/models++Run with:+ OPENAI_API_KEY=... GEMINI_API_KEY=... ANTHROPIC_API_KEY=... pytest tests/test_proxy_passthrough_integration.py -v+"""++import os++import pytest++pytest.importorskip("fastapi")+pytest.importorskip("httpx")++from fastapi.testclient import TestClient++from headroom.proxy.server import ProxyConfig, create_app++# =============================================================================+# Fixtures+# =============================================================================+++def openai_client():+ """Create test client configured for OpenAI passthrough."""+ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def gemini_client():+ """Create test client configured for Gemini passthrough."""+ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def anthropic_client():+ """Create test client configured for Anthropic passthrough."""+ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ )+ app = create_app(config)+ with TestClient(app) as client:+ yield client+++def openai_api_key():+ """Get OpenAI API key from environment."""+ return os.environ.get("OPENAI_API_KEY")+++def gemini_api_key():+ """Get Gemini API key from environment."""+ return os.environ.get("GEMINI_API_KEY")+++def anthropic_api_key():+ """Get Anthropic API key from environment."""+ return os.environ.get("ANTHROPIC_API_KEY")+++# =============================================================================+# OpenAI Passthrough Tests+# =============================================================================++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestOpenAIModels:+ """Test OpenAI /v1/models endpoint passthrough."""++ def test_list_models(self, openai_client, openai_api_key):+ """GET /v1/models returns list of available models."""+ response = openai_client.get(+ "/v1/models", headers={"Authorization": f"Bearer {openai_api_key}"}+ )+ assert response.status_code == 200+ data = response.json()++ # Verify OpenAI models list format+ assert "data" in data+ assert "object" in data+ assert data["object"] == "list"+ assert len(data["data"]) > 0++ # Verify model object structure+ model = data["data"][0]+ assert "id" in model+ assert "object" in model+ assert model["object"] == "model"++ def test_get_specific_model(self, openai_client, openai_api_key):+ """GET /v1/models/{model_id} returns model details."""+ response = openai_client.get(+ "/v1/models/gpt-4o-mini", headers={"Authorization": f"Bearer {openai_api_key}"}+ )+ assert response.status_code == 200+ data = response.json()++ assert data["id"] == "gpt-4o-mini"+ assert data["object"] == "model"++ def test_invalid_api_key(self, openai_client):+ """Invalid API key returns authentication error."""+ response = openai_client.get(+ "/v1/models", headers={"Authorization": "Bearer invalid-key-12345"}+ )+ assert response.status_code == 401++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestOpenAIEmbeddings:+ """Test OpenAI /v1/embeddings endpoint passthrough."""++ def test_create_embedding(self, openai_client, openai_api_key):+ """POST /v1/embeddings creates embeddings successfully."""+ response = openai_client.post(+ "/v1/embeddings",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "model": "text-embedding-3-small",+ "input": "The quick brown fox jumps over the lazy dog.",+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Verify embedding response format+ assert "data" in data+ assert "model" in data+ assert "usage" in data+ assert data["object"] == "list"++ # Verify embedding data+ embedding = data["data"][0]+ assert "embedding" in embedding+ assert "index" in embedding+ assert embedding["object"] == "embedding"+ assert isinstance(embedding["embedding"], list)+ assert len(embedding["embedding"]) > 0++ # Verify usage+ assert "prompt_tokens" in data["usage"]+ assert "total_tokens" in data["usage"]++ def test_create_embedding_batch(self, openai_client, openai_api_key):+ """POST /v1/embeddings with multiple inputs creates batch embeddings."""+ response = openai_client.post(+ "/v1/embeddings",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "model": "text-embedding-3-small",+ "input": ["First text to embed", "Second text to embed", "Third text to embed"],+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Should return 3 embeddings+ assert len(data["data"]) == 3+ for i, embedding in enumerate(data["data"]):+ assert embedding["index"] == i+ assert isinstance(embedding["embedding"], list)++ def test_embedding_invalid_model(self, openai_client, openai_api_key):+ """Invalid model returns appropriate error."""+ response = openai_client.post(+ "/v1/embeddings",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={"model": "nonexistent-embedding-model", "input": "Test text"},+ )+ assert response.status_code >= 400++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestOpenAIModerations:+ """Test OpenAI /v1/moderations endpoint passthrough."""++ def test_moderation_safe_content(self, openai_client, openai_api_key):+ """POST /v1/moderations on safe content returns no flags."""+ response = openai_client.post(+ "/v1/moderations",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={"input": "I love sunny days and playing with my dog in the park."},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify moderation response format+ assert "id" in data+ assert "model" in data+ assert "results" in data++ # Safe content should not be flagged+ result = data["results"][0]+ assert "flagged" in result+ assert "categories" in result+ assert "category_scores" in result+ # Safe content should generally not be flagged+ # (though model may have false positives occasionally)++ def test_moderation_batch(self, openai_client, openai_api_key):+ """POST /v1/moderations with multiple inputs."""+ response = openai_client.post(+ "/v1/moderations",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={+ "input": [+ "Hello, how are you today?",+ "What a beautiful sunset!",+ "I enjoy reading books.",+ ]+ },+ )+ assert response.status_code == 200+ data = response.json()++ # Should return 3 moderation results+ assert len(data["results"]) == 3+++# =============================================================================+# Gemini Passthrough Tests+# =============================================================================++[email protected](not os.environ.get("GEMINI_API_KEY"), reason="GEMINI_API_KEY not set")+class TestGeminiModels:+ """Test Gemini /v1beta/models endpoint passthrough."""++ def test_list_models(self, gemini_client, gemini_api_key):+ """GET /v1beta/models returns list of available models."""+ response = gemini_client.get(f"/v1beta/models?key={gemini_api_key}")+ assert response.status_code == 200+ data = response.json()++ # Verify Gemini models list format+ assert "models" in data+ assert len(data["models"]) > 0++ # Verify model object structure+ model = data["models"][0]+ assert "name" in model+ assert "displayName" in model or "description" in model++ def test_get_specific_model(self, gemini_client, gemini_api_key):+ """GET /v1beta/models/{model} returns model details."""+ response = gemini_client.get(f"/v1beta/models/gemini-2.0-flash?key={gemini_api_key}")+ assert response.status_code == 200+ data = response.json()++ assert "name" in data+ assert "gemini" in data["name"].lower()++[email protected](not os.environ.get("GEMINI_API_KEY"), reason="GEMINI_API_KEY not set")+class TestGeminiEmbedContent:+ """Test Gemini /v1beta/models/{model}:embedContent endpoint passthrough."""++ def test_embed_content(self, gemini_client, gemini_api_key):+ """POST :embedContent creates embeddings successfully."""+ response = gemini_client.post(+ f"/v1beta/models/text-embedding-004:embedContent?key={gemini_api_key}",+ json={"content": {"parts": [{"text": "The quick brown fox jumps over the lazy dog."}]}},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify embedding response format+ assert "embedding" in data+ assert "values" in data["embedding"]+ assert isinstance(data["embedding"]["values"], list)+ assert len(data["embedding"]["values"]) > 0++ def test_embed_content_with_task_type(self, gemini_client, gemini_api_key):+ """POST :embedContent with task type specified."""+ response = gemini_client.post(+ f"/v1beta/models/text-embedding-004:embedContent?key={gemini_api_key}",+ json={+ "content": {"parts": [{"text": "What is the capital of France?"}]},+ "taskType": "RETRIEVAL_QUERY",+ },+ )+ assert response.status_code == 200+ data = response.json()++ assert "embedding" in data+ assert "values" in data["embedding"]++[email protected](not os.environ.get("GEMINI_API_KEY"), reason="GEMINI_API_KEY not set")+class TestGeminiBatchEmbedContents:+ """Test Gemini /v1beta/models/{model}:batchEmbedContents endpoint passthrough."""++ def test_batch_embed_contents(self, gemini_client, gemini_api_key):+ """POST :batchEmbedContents creates batch embeddings."""+ # Note: batchEmbedContents requires model field in each request+ response = gemini_client.post(+ f"/v1beta/models/text-embedding-004:batchEmbedContents?key={gemini_api_key}",+ json={+ "requests": [+ {+ "model": "models/text-embedding-004",+ "content": {"parts": [{"text": "First document to embed"}]},+ },+ {+ "model": "models/text-embedding-004",+ "content": {"parts": [{"text": "Second document to embed"}]},+ },+ {+ "model": "models/text-embedding-004",+ "content": {"parts": [{"text": "Third document to embed"}]},+ },+ ]+ },+ )+ # May return 400 if format changed, or 200 on success+ assert response.status_code in [200, 400]+ if response.status_code == 200:+ data = response.json()+ # Verify batch embedding response format+ assert "embeddings" in data+ assert len(data["embeddings"]) == 3++ for embedding in data["embeddings"]:+ assert "values" in embedding+ assert isinstance(embedding["values"], list)+++# =============================================================================+# Anthropic Passthrough Tests+# =============================================================================++[email protected](not os.environ.get("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not set")+class TestAnthropicModels:+ """Test Anthropic /v1/models endpoint passthrough."""++ def test_list_models(self, anthropic_client, anthropic_api_key):+ """GET /v1/models returns list of available models with x-api-key header."""+ response = anthropic_client.get(+ "/v1/models",+ headers={"x-api-key": anthropic_api_key, "anthropic-version": "2023-06-01"},+ )+ assert response.status_code == 200+ data = response.json()++ # Verify Anthropic models list format+ assert "data" in data+ assert len(data["data"]) > 0++ # Verify model object structure+ model = data["data"][0]+ assert "id" in model+ assert "type" in model++ def test_get_specific_model(self, anthropic_client, anthropic_api_key):+ """GET /v1/models/{model_id} returns model details."""+ # First get the list to find a valid model ID+ list_response = anthropic_client.get(+ "/v1/models",+ headers={"x-api-key": anthropic_api_key, "anthropic-version": "2023-06-01"},+ )+ assert list_response.status_code == 200+ models = list_response.json().get("data", [])++ if not models:+ pytest.skip("No models available")++ # Use the first available model+ model_id = models[0]["id"]++ response = anthropic_client.get(+ f"/v1/models/{model_id}",+ headers={"x-api-key": anthropic_api_key, "anthropic-version": "2023-06-01"},+ )+ assert response.status_code == 200+ data = response.json()++ assert "id" in data+ assert data["id"] == model_id++ def test_invalid_api_key(self, anthropic_client):+ """Invalid API key returns authentication error."""+ response = anthropic_client.get(+ "/v1/models",+ headers={"x-api-key": "invalid-key-12345", "anthropic-version": "2023-06-01"},+ )+ assert response.status_code == 401+++# =============================================================================+# Proxy Stats Tests+# =============================================================================++[email protected](not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+class TestPassthroughStats:+ """Test that passthrough requests are tracked in proxy stats."""++ def test_stats_track_passthrough_requests(self, openai_client, openai_api_key):+ """Verify passthrough requests are tracked in stats."""+ # Make a passthrough request+ openai_client.get("/v1/models", headers={"Authorization": f"Bearer {openai_api_key}"})++ # Check stats+ stats_response = openai_client.get("/stats")+ assert stats_response.status_code == 200+ stats = stats_response.json()++ # Verify stats structure+ assert "requests" in stats+ assert "total" in stats["requests"]+ assert stats["requests"]["total"] >= 1++ def test_stats_track_embeddings_requests(self, openai_client, openai_api_key):+ """Verify embeddings passthrough requests are tracked."""+ # Make an embeddings request+ openai_client.post(+ "/v1/embeddings",+ headers={"Authorization": f"Bearer {openai_api_key}"},+ json={"model": "text-embedding-3-small", "input": "Test embedding"},+ )++ # Check stats+ stats_response = openai_client.get("/stats")+ stats = stats_response.json()++ # Should track embeddings under openai provider+ assert "openai" in stats["requests"]["by_provider"]+++# =============================================================================+# Error Handling Tests+# =============================================================================+++class TestPassthroughErrorHandling:+ """Test error handling for passthrough endpoints."""++ def test_missing_auth_header_openai(self, openai_client):+ """Missing auth header returns appropriate error."""+ response = openai_client.get("/v1/models")+ # OpenAI requires authentication+ assert response.status_code >= 400++ @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")+ def test_invalid_json_body(self, openai_client, openai_api_key):+ """Invalid JSON body returns 400 error."""+ response = openai_client.post(+ "/v1/embeddings",+ headers={+ "Authorization": f"Bearer {openai_api_key}",+ "Content-Type": "application/json",+ },+ content=b"not valid json",+ )+ assert response.status_code >= 400