Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat: introduce canonical pipeline lifecycle contract

JerrettDaviscd9c2e1d012b
diff --git a/README.md b/README.md
index e5afcdbfd..7090a50c8 100644
--- a/README.md
+++ b/README.md
@@ -112,6 +112,17 @@ Bundles the [RTK](https://github.com/rtk-ai/rtk) binary for shell-output rewriti
→ [Architecture](https://headroom-docs.vercel.app/docs/architecture) · [CCR reversible compression](https://headroom-docs.vercel.app/docs/ccr) · [Kompress-base model card](https://huggingface.co/chopratejas/kompress-base)
+### Canonical pipeline lifecycle
+
+Headroom now exposes one stable request lifecycle across `compress()`, the SDK, and the proxy:
+
+`Setup` → `Pre-Start` → `Post-Start` → `Input Received` → `Input Cached` → `Input Routed` → `Input Compressed` → `Input Remembered` → `Pre-Send` → `Post-Send` → `Response Received`
+
+- **Transforms** still do the work: CacheAligner, ContentRouter, SmartCrusher, CodeCompressor, Kompress-base, IntelligentContext / RollingWindow.
+- **Pipeline extensions** observe or customize those lifecycle stages via `on_pipeline_event(...)`.
+- **Compression hooks** still work and now sit alongside the canonical lifecycle instead of being the only extension seam.
+- **Proxy extensions** remain the server/app integration seam for ASGI middleware, routes, and startup policy.
+
---
## Proof
diff --git a/headroom/__init__.py b/headroom/__init__.py
index 8fe042c22..1bc2ef41a 100644
--- a/headroom/__init__.py
+++ b/headroom/__init__.py
@@ -175,6 +175,11 @@ __all__ = [
"CompressionHooks",
"CompressContext",
"CompressEvent",
+ # Canonical pipeline
+ "PipelineStage",
+ "PipelineEvent",
+ "PipelineExtensionManager",
+ "CANONICAL_PIPELINE_STAGES",
# Shared context for multi-agent workflows
"SharedContext",
]
@@ -268,6 +273,11 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
"CompressionHooks": ("headroom.hooks", "CompressionHooks"),
"CompressContext": ("headroom.hooks", "CompressContext"),
"CompressEvent": ("headroom.hooks", "CompressEvent"),
+ # Canonical pipeline
+ "PipelineStage": ("headroom.pipeline", "PipelineStage"),
+ "PipelineEvent": ("headroom.pipeline", "PipelineEvent"),
+ "PipelineExtensionManager": ("headroom.pipeline", "PipelineExtensionManager"),
+ "CANONICAL_PIPELINE_STAGES": ("headroom.pipeline", "CANONICAL_PIPELINE_STAGES"),
# Shared context
"SharedContext": ("headroom.shared_context", "SharedContext"),
}
diff --git a/headroom/client.py b/headroom/client.py
index ba23ed273..38812fc30 100644
--- a/headroom/client.py
+++ b/headroom/client.py
@@ -20,6 +20,7 @@ from .config import (
SimulationResult,
)
from .parser import parse_messages
+from .pipeline import PipelineExtensionManager, PipelineStage, summarize_routing_markers
from .providers.base import Provider
from .storage import create_storage
from .tokenizer import Tokenizer
@@ -324,6 +325,10 @@ class HeadroomClient:
# Initialize transform pipeline
self._pipeline = TransformPipeline(self._config, provider=self._provider)
+ self._pipeline_extensions = PipelineExtensionManager(
+ extensions=self._config.pipeline_extensions,
+ discover=self._config.discover_pipeline_extensions,
+ )
# Initialize cache optimizer
self._cache_optimizer: BaseCacheOptimizer | None = None
@@ -357,6 +362,16 @@ class HeadroomClient:
self.chat = type("Chat", (), {"completions": ChatCompletions(self)})()
# Public API - Anthropic style
self.messages = Messages(self)
+ self._pipeline_extensions.emit(
+ PipelineStage.SETUP,
+ operation="sdk.setup",
+ provider=self._provider.name.lower(),
+ metadata={
+ "default_mode": self._default_mode.value,
+ "cache_optimizer_enabled": enable_cache_optimizer,
+ "semantic_cache_enabled": enable_semantic_cache,
+ },
+ )
def _get_tokenizer(self, model: str) -> Tokenizer:
"""Get tokenizer for model using provider."""
@@ -391,6 +406,18 @@ class HeadroomClient:
timestamp = datetime.now(timezone.utc).replace(tzinfo=None)
mode = HeadroomMode(headroom_mode) if headroom_mode else self._default_mode
+ input_event = self._pipeline_extensions.emit(
+ PipelineStage.INPUT_RECEIVED,
+ operation="sdk.request",
+ request_id=request_id,
+ provider=self._provider.name.lower(),
+ model=model,
+ messages=messages,
+ metadata={"api_style": api_style, "stream": stream, "mode": mode.value},
+ )
+ if input_event.messages is not None:
+ messages = input_event.messages
+
tokenizer = self._get_tokenizer(model)
# Analyze original messages
@@ -433,6 +460,23 @@ class HeadroomClient:
tokens_after = result.tokens_after
transforms_applied = result.transforms_applied
+ routing_markers = summarize_routing_markers(transforms_applied)
+ if routing_markers:
+ routed_event = self._pipeline_extensions.emit(
+ PipelineStage.INPUT_ROUTED,
+ operation="sdk.request",
+ request_id=request_id,
+ provider=self._provider.name.lower(),
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "routing_markers": routing_markers,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if routed_event.messages is not None:
+ optimized_messages = routed_event.messages
+
# Apply provider-specific cache optimization
if self._cache_optimizer is not None or self._semantic_cache_layer is not None:
cache_context = OptimizationContext(
@@ -481,6 +525,23 @@ class HeadroomClient:
f"cache_optimizer:{t}" for t in (cache_result.transforms_applied or [])
)
+ compressed_event = self._pipeline_extensions.emit(
+ PipelineStage.INPUT_COMPRESSED,
+ operation="sdk.request",
+ request_id=request_id,
+ provider=self._provider.name.lower(),
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "tokens_before": tokens_before,
+ "tokens_after": tokens_after,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if compressed_event.messages is not None:
+ optimized_messages = compressed_event.messages
+ tokens_after = tokenizer.count_messages(optimized_messages)
+
# Recalculate prefix hash after optimization
stable_prefix_hash = compute_prefix_hash(optimized_messages)
else:
@@ -489,6 +550,20 @@ class HeadroomClient:
tokens_after = tokens_before
transforms_applied = []
+ presend_event = self._pipeline_extensions.emit(
+ PipelineStage.PRE_SEND,
+ operation="sdk.request",
+ request_id=request_id,
+ provider=self._provider.name.lower(),
+ model=model,
+ messages=optimized_messages,
+ metadata={"api_style": api_style, "stream": stream},
+ )
+ if presend_event.messages is not None:
+ optimized_messages = presend_event.messages
+ tokens_after = tokenizer.count_messages(optimized_messages)
+ stable_prefix_hash = compute_prefix_hash(optimized_messages)
+
# Create metrics
metrics = RequestMetrics(
request_id=request_id,
@@ -530,7 +605,7 @@ class HeadroomClient:
# Call underlying client based on API style
try:
if api_style == "anthropic":
- return self._call_anthropic(
+ response = self._call_anthropic(
model=model,
messages=optimized_messages,
stream=stream,
@@ -538,7 +613,7 @@ class HeadroomClient:
**kwargs,
)
else:
- return self._call_openai(
+ response = self._call_openai(
model=model,
messages=optimized_messages,
stream=stream,
@@ -546,6 +621,27 @@ class HeadroomClient:
**kwargs,
)
+ self._pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="sdk.request",
+ request_id=request_id,
+ provider=self._provider.name.lower(),
+ model=model,
+ messages=optimized_messages,
+ response=response,
+ metadata={"api_style": api_style, "stream": stream},
+ )
+ self._pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="sdk.request",
+ request_id=request_id,
+ provider=self._provider.name.lower(),
+ model=model,
+ response=response,
+ metadata={"api_style": api_style, "stream": stream},
+ )
+ return response
+
except Exception as e:
metrics.error = str(e)
self._storage.save(metrics)
diff --git a/headroom/config.py b/headroom/config.py
index c18a130e5..1668df18b 100644
--- a/headroom/config.py
+++ b/headroom/config.py
@@ -636,6 +636,10 @@ class HeadroomConfig:
# Debugging - opt-in diff artifact generation
generate_diff_artifact: bool = False # Enable to get detailed transform diffs
+ # Canonical pipeline lifecycle extensions
+ pipeline_extensions: list[Any] = field(default_factory=list)
+ discover_pipeline_extensions: bool = True
+
def get_context_limit(self, model: str) -> int | None:
"""
Get context limit for a model from user overrides.
diff --git a/headroom/hooks.py b/headroom/hooks.py
index dc1f7454b..4f3cf82ce 100644
--- a/headroom/hooks.py
+++ b/headroom/hooks.py
@@ -1,4 +1,4 @@
-"""Compression Hooks — extension points for customizing compression behavior.
+"""Compression hooks and pipeline lifecycle events.
Three hooks at well-defined pipeline stages:
@@ -6,6 +6,10 @@ Three hooks at well-defined pipeline stages:
2. compute_biases: set per-message compression aggressiveness (position-aware, phase-aware)
3. post_compress: observe results after compression (learning, analytics, logging)
+The canonical pipeline also emits lifecycle events through ``on_pipeline_event``.
+That gives extensions one stable contract across SDK, ``compress()``, and proxy
+request flow without replacing the existing compression hooks.
+
Default implementation is no-op — OSS behavior unchanged. Override these
in a subclass to customize (e.g., Headroom SaaS implements position-aware
compression and cross-turn deduplication via these hooks).
@@ -30,6 +34,8 @@ from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
+from .pipeline import PipelineEvent
+
@dataclass
class CompressContext:
@@ -135,3 +141,11 @@ class CompressionHooks:
event: Full compression event with before/after metrics.
"""
pass
+
+ def on_pipeline_event(self, event: PipelineEvent) -> PipelineEvent | None:
+ """Observe canonical pipeline lifecycle events.
+
+ Override when the integration needs stable lifecycle notifications beyond
+ the three legacy compression-specific hooks.
+ """
+ return None
diff --git a/headroom/pipeline.py b/headroom/pipeline.py
new file mode 100644
index 000000000..868943cab
--- /dev/null
+++ b/headroom/pipeline.py
@@ -0,0 +1,178 @@
+"""Canonical Headroom pipeline lifecycle and extension contracts."""
+
+from __future__ import annotations
+
+import importlib.metadata
+import logging
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Protocol
+
+log = logging.getLogger(__name__)
+
+ENTRY_POINT_GROUP = "headroom.pipeline_extension"
+
+
+class PipelineStage(str, Enum):
+ """Stable lifecycle stages for the canonical Headroom pipeline."""
+
+ SETUP = "setup"
+ PRE_START = "pre_start"
+ POST_START = "post_start"
+ INPUT_RECEIVED = "input_received"
+ INPUT_CACHED = "input_cached"
+ INPUT_ROUTED = "input_routed"
+ INPUT_COMPRESSED = "input_compressed"
+ INPUT_REMEMBERED = "input_remembered"
+ PRE_SEND = "pre_send"
+ POST_SEND = "post_send"
+ RESPONSE_RECEIVED = "response_received"
+
+
+CANONICAL_PIPELINE_STAGES: tuple[PipelineStage, ...] = (
+ PipelineStage.SETUP,
+ PipelineStage.PRE_START,
+ PipelineStage.POST_START,
+ PipelineStage.INPUT_RECEIVED,
+ PipelineStage.INPUT_CACHED,
+ PipelineStage.INPUT_ROUTED,
+ PipelineStage.INPUT_COMPRESSED,
+ PipelineStage.INPUT_REMEMBERED,
+ PipelineStage.PRE_SEND,
+ PipelineStage.POST_SEND,
+ PipelineStage.RESPONSE_RECEIVED,
+)
+
+
+@dataclass
+class PipelineEvent:
+ """Event emitted at a canonical pipeline stage.
+
+ Extensions may mutate ``messages``, ``tools``, ``headers``, or ``metadata`` in
+ place, or return a replacement ``PipelineEvent`` from ``on_pipeline_event``.
+ """
+
+ stage: PipelineStage
+ operation: str
+ request_id: str = ""
+ provider: str = ""
+ model: str = ""
+ messages: list[dict[str, Any]] | None = None
+ tools: list[dict[str, Any]] | None = None
+ headers: dict[str, str] | None = None
+ response: Any = None
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class PipelineExtension(Protocol):
+ """Request lifecycle extension contract for the canonical pipeline."""
+
+ def on_pipeline_event(self, event: PipelineEvent) -> PipelineEvent | None:
+ """Handle a canonical pipeline event."""
+
+
+def discover_pipeline_extensions() -> list[PipelineExtension]:
+ """Load registered pipeline extensions from Python entry points."""
+
+ discovered: list[PipelineExtension] = []
+ try:
+ entries = importlib.metadata.entry_points(group=ENTRY_POINT_GROUP)
+ except Exception as exc: # noqa: BLE001 - importlib metadata varies by runtime
+ log.debug("pipeline extensions: entry-point enumeration failed: %s", exc)
+ return discovered
+
+ for entry in entries:
+ try:
+ extension = entry.load()
+ except Exception as exc: # noqa: BLE001 - third-party load failures are isolated
+ log.warning("pipeline extension %r failed to load: %s", entry.name, exc)
+ continue
+
+ if isinstance(extension, type):
+ try:
+ extension = extension()
+ except Exception as exc: # noqa: BLE001
+ log.warning("pipeline extension %r failed to initialize: %s", entry.name, exc)
+ continue
+
+ discovered.append(extension)
+
+ return discovered
+
+
+def summarize_routing_markers(transforms_applied: list[str]) -> list[str]:
+ """Return the routed transform markers emitted by ContentRouter."""
+
+ return [item for item in transforms_applied if item.startswith("router:")]
+
+
+class PipelineExtensionManager:
+ """Dispatch canonical pipeline events to configured extensions."""
+
+ def __init__(
+ self,
+ *,
+ hooks: Any = None,
+ extensions: list[Any] | None = None,
+ discover: bool = True,
+ ) -> None:
+ resolved: list[Any] = []
+ if hooks is not None and callable(getattr(hooks, "on_pipeline_event", None)):
+ resolved.append(hooks)
+ if extensions:
+ resolved.extend(extensions)
+ if discover:
+ resolved.extend(discover_pipeline_extensions())
+ self._extensions = resolved
+
+ @property
+ def enabled(self) -> bool:
+ return bool(self._extensions)
+
+ def emit(
+ self,
+ stage: PipelineStage,
+ *,
+ operation: str,
+ request_id: str = "",
+ provider: str = "",
+ model: str = "",
+ messages: list[dict[str, Any]] | None = None,
+ tools: list[dict[str, Any]] | None = None,
+ headers: dict[str, str] | None = None,
+ response: Any = None,
+ metadata: dict[str, Any] | None = None,
+ ) -> PipelineEvent:
+ """Emit a canonical lifecycle event and return the final event state."""
+
+ event = PipelineEvent(
+ stage=stage,
+ operation=operation,
+ request_id=request_id,
+ provider=provider,
+ model=model,
+ messages=messages,
+ tools=tools,
+ headers=headers,
+ response=response,
+ metadata=metadata or {},
+ )
+
+ for extension in self._extensions:
+ handler = getattr(extension, "on_pipeline_event", None)
+ if not callable(handler):
+ continue
+ try:
+ updated = handler(event)
+ except Exception as exc: # noqa: BLE001 - preserve hook fail-open behavior
+ log.warning(
+ "pipeline extension %r failed during %s: %s",
+ type(extension).__name__,
+ stage.value,
+ exc,
+ )
+ continue
+ if isinstance(updated, PipelineEvent):
+ event = updated
+
+ return event
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index a9625672c..036ff83c6 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -23,6 +23,8 @@ if TYPE_CHECKING:
import httpx
+from headroom.pipeline import PipelineStage, summarize_routing_markers
+
logger = logging.getLogger("headroom.proxy")
@@ -297,6 +299,11 @@ class AnthropicHandlerMixin:
request: Request,
) -> Response | StreamingResponse:
"""Handle Anthropic /v1/messages endpoint."""
+ if not hasattr(self, "pipeline_extensions"):
+ from headroom.pipeline import PipelineExtensionManager
+
+ self.pipeline_extensions = PipelineExtensionManager(discover=False)
+
from fastapi import HTTPException
from fastapi.responses import JSONResponse, Response
@@ -466,6 +473,22 @@ class AnthropicHandlerMixin:
messages = body.get("messages", [])
with stage_timer.measure("deep_copy"):
original_client_messages = copy.deepcopy(messages)
+ input_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=messages,
+ tools=body.get("tools"),
+ metadata={"path": "/v1/messages", "stream": body.get("stream", False)},
+ )
+ if input_event.messages is not None:
+ messages = input_event.messages
+ with stage_timer.measure("deep_copy"):
+ original_client_messages = copy.deepcopy(messages)
+ if input_event.tools is not None:
+ body["tools"] = input_event.tools
# Validate message array size
if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
@@ -570,6 +593,15 @@ class AnthropicHandlerMixin:
cached = await self.cache.get(messages, model)
if cached:
cache_hit = True
+ self.pipeline_extensions.emit(
+ PipelineStage.INPUT_CACHED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=messages,
+ metadata={"cache_hit": True, "path": "/v1/messages"},
+ )
optimization_latency = (time.time() - start_time) * 1000
await self.metrics.record_request(
@@ -857,6 +889,43 @@ class AnthropicHandlerMixin:
tokens_saved = max(0, original_tokens - optimized_tokens)
optimization_latency = (time.time() - start_time) * 1000
+ routing_markers = summarize_routing_markers(transforms_applied)
+ if routing_markers:
+ routed_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_ROUTED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "routing_markers": routing_markers,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if routed_event.messages is not None:
+ optimized_messages = routed_event.messages
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+
+ compressed_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_COMPRESSED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "tokens_before": original_tokens,
+ "tokens_after": optimized_tokens,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if compressed_event.messages is not None:
+ optimized_messages = compressed_event.messages
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+
# Hook: post_compress — let hooks observe compression results
if self.config.hooks and tokens_saved > 0:
from headroom.hooks import CompressEvent
@@ -1005,6 +1074,8 @@ class AnthropicHandlerMixin:
logger.debug(f"[{request_id}] Traffic learner: {e}")
# Memory: Inject context and tools
+ memory_context_injected = False
+ memory_tools_injected = False
if self.memory_handler and memory_user_id:
# Search and inject memory context
if self.memory_handler.config.inject_context:
@@ -1040,6 +1111,7 @@ class AnthropicHandlerMixin:
frozen_message_count=frozen_message_count,
)
)
+ memory_context_injected = True
logger.info(
f"[{request_id}] Memory: Appended {len(memory_context)} chars "
f"to latest non-frozen user turn (prefix cache-safe)"
@@ -1048,6 +1120,7 @@ class AnthropicHandlerMixin:
optimized_messages = self._inject_system_context(
optimized_messages, memory_context, body=body
)
+ memory_context_injected = True
logger.info(
f"[{request_id}] Memory: Injected {len(memory_context)} chars of context"
)
@@ -1058,6 +1131,7 @@ class AnthropicHandlerMixin:
if self.memory_handler.config.inject_tools:
tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "anthropic")
if mem_tools_injected:
+ memory_tools_injected = True
tool_names = [
t.get("name") or t.get("type", "")
for t in tools
@@ -1080,6 +1154,28 @@ class AnthropicHandlerMixin:
f"[{request_id}] Memory: Added beta header: {key}={headers[key]}"
)
+ if memory_context_injected or memory_tools_injected:
+ remembered_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_REMEMBERED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=optimized_messages,
+ tools=tools,
+ headers=headers,
+ metadata={
+ "memory_context_injected": memory_context_injected,
+ "memory_tools_injected": memory_tools_injected,
+ },
+ )
+ if remembered_event.messages is not None:
+ optimized_messages = remembered_event.messages
+ if remembered_event.tools is not None:
+ tools = remembered_event.tools
+ if remembered_event.headers is not None:
+ headers = remembered_event.headers
+
# Query Echo: disabled — hurts prefix caching in long conversations.
# The echo changes every turn, invalidating the cached prefix.
# To re-enable, uncomment and set query_echo_enabled on ProxyConfig.
@@ -1090,6 +1186,28 @@ class AnthropicHandlerMixin:
tools = self._sort_tools_deterministically(tools)
body["tools"] = tools
+ presend_event = self.pipeline_extensions.emit(
+ PipelineStage.PRE_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=optimized_messages,
+ tools=tools,
+ headers=headers,
+ metadata={"path": "/v1/messages", "stream": stream},
+ )
+ if presend_event.messages is not None:
+ optimized_messages = presend_event.messages
+ body["messages"] = optimized_messages
+ if presend_event.tools is not None:
+ tools = self._sort_tools_deterministically(presend_event.tools)
+ body["tools"] = tools
+ if presend_event.headers is not None:
+ headers = presend_event.headers
+ optimized_tokens = tokenizer.count_messages(body["messages"])
+ tokens_saved = max(0, original_tokens - optimized_tokens)
+
# Unit 2: mark end of pre-upstream phase. Everything after this
# point is upstream I/O or post-response bookkeeping.
stage_timer.record(
@@ -1102,6 +1220,16 @@ class AnthropicHandlerMixin:
# Route through Bedrock backend
try:
if stream:
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ metadata={"path": "/v1/messages", "stream": True},
+ )
await _finalize_pre_upstream()
return await self._stream_response_bedrock(
body,
@@ -1122,6 +1250,34 @@ class AnthropicHandlerMixin:
backend_response = await self.anthropic_backend.send_message(
body, headers
)
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ response=backend_response.body,
+ metadata={
+ "path": "/v1/messages",
+ "stream": False,
+ "status_code": backend_response.status_code,
+ },
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ response=backend_response.body,
+ metadata={
+ "path": "/v1/messages",
+ "stream": False,
+ "status_code": backend_response.status_code,
+ },
+ )
# Non-stream: first-byte and connect are effectively
# the same horizon — ``send_message`` awaits until
# the response body is fully buffered.
@@ -1210,6 +1366,16 @@ class AnthropicHandlerMixin:
try:
if stream:
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ metadata={"path": "/v1/messages", "stream": True},
+ )
await _finalize_pre_upstream()
return await self._stream_response(
url,
@@ -1232,6 +1398,34 @@ class AnthropicHandlerMixin:
else:
async with stage_timer.measure("upstream_connect"):
response = await self._retry_request("POST", url, headers, body)
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ response=response,
+ metadata={
+ "path": "/v1/messages",
+ "stream": False,
+ "status_code": response.status_code,
+ },
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="anthropic",
+ model=model,
+ response=response,
+ metadata={
+ "path": "/v1/messages",
+ "stream": False,
+ "status_code": response.status_code,
+ },
+ )
if (
"upstream_first_byte" not in stage_timer
and "upstream_connect" in stage_timer
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index e80dbc5de..49938c1ae 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -1,2558 +1,2746 @@
-"""OpenAI handler mixin for HeadroomProxy.
-
-Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import base64
-import contextlib
-import copy
-import json
-import logging
-import os
-import time
-import uuid
-from datetime import datetime
-from typing import TYPE_CHECKING, Any
-
-from headroom.proxy.helpers import jitter_delay_ms
-from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
-from headroom.proxy.ws_session_registry import (
- TerminationCause,
- WebSocketSessionRegistry,
- WSSessionHandle,
-)
-
-if TYPE_CHECKING:
- from fastapi import Request, WebSocket
- from fastapi.responses import JSONResponse, Response, StreamingResponse
-
-import httpx
-
-from headroom.copilot_auth import apply_copilot_api_auth
-
-logger = logging.getLogger("headroom.proxy")
-
-
-# Interactive Responses turns are latency-sensitive. Fail open quickly rather
-# than holding the session hostage on memory lookup.
-RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0
-
-# Cap the wait for the first client frame after the WS handshake completes.
-# A zombie or malicious client that accepts the upgrade but never sends the
-# first response.create frame would otherwise hold a slot indefinitely and
-# starve the session registry. 60 s is generous for real clients (Codex
-# typically sends the first frame within a few hundred milliseconds of the
-# accept) but short enough to bound the damage from a hung peer.
-WS_FIRST_FRAME_TIMEOUT_SECONDS = 60.0
-
-
-def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:
- """Best-effort decode of an OpenAI OAuth bearer token payload.
-
- OpenClaw's Codex OAuth flow may forward only the bearer token after the
- provider base URL is overridden. In that case the explicit
- ``ChatGPT-Account-ID`` header can be missing even though the JWT still
- carries the account id we need to route to the ChatGPT Codex backend.
- """
- auth = headers.get("authorization") or headers.get("Authorization")
- if not auth:
- return None
-
- scheme, _, token = auth.partition(" ")
- if scheme.lower() != "bearer" or token.count(".") < 2:
- return None
-
- payload = token.split(".", 2)[1]
- payload += "=" * (-len(payload) % 4)
- # Intentionally no signature verification here: this is only a best-effort
- # routing hint extractor. Upstream still performs the actual auth/authz checks.
- try:
- decoded = base64.urlsafe_b64decode(payload.encode("ascii"))
- data = json.loads(decoded.decode("utf-8"))
- except (ValueError, UnicodeDecodeError):
- return None
-
- return data if isinstance(data, dict) else None
-
-
-def _resolve_codex_routing_headers(headers: dict[str, str]) -> tuple[dict[str, str], bool]:
- """Resolve ChatGPT Codex routing hints from explicit headers or OAuth JWT."""
- resolved = dict(headers)
- lower_lookup = {k.lower(): k for k in resolved}
-
- if "chatgpt-account-id" in lower_lookup:
- return resolved, True
-
- payload = _decode_openai_bearer_payload(resolved)
- auth_claims = payload.get("https://api.openai.com/auth") if isinstance(payload, dict) else None
- account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None
- if isinstance(account_id, str) and account_id.strip():
- resolved["ChatGPT-Account-ID"] = account_id.strip()
- return resolved, True
-
- return resolved, False
-
-
-class OpenAIHandlerMixin:
- """Mixin providing OpenAI API handler methods for HeadroomProxy."""
-
- @staticmethod
- def _strict_previous_turn_frozen_count(
- messages: list[dict[str, Any]],
- base_frozen_count: int,
- ) -> int:
- """Freeze all prior turns in cache mode; only final user turn is mutable."""
- if not messages:
- return base_frozen_count
- final_idx = len(messages) - 1
- if messages[final_idx].get("role") == "user":
- return max(base_frozen_count, final_idx)
- return len(messages)
-
- @staticmethod
- def _restore_frozen_prefix(
- original_messages: list[dict[str, Any]],
- candidate_messages: list[dict[str, Any]],
- *,
- frozen_message_count: int,
- ) -> tuple[list[dict[str, Any]], int]:
- """Force frozen prefix bytes to match original request exactly."""
- if frozen_message_count <= 0 or not original_messages:
- return candidate_messages, 0
-
- frozen = min(frozen_message_count, len(original_messages))
- restored = list(candidate_messages)
-
- if len(restored) < frozen:
- return list(original_messages[:frozen]) + restored, frozen
-
- changed = 0
- for idx in range(frozen):
- if restored[idx] != original_messages[idx]:
- restored[idx] = original_messages[idx]
- changed += 1
- return restored, changed
-
- async def handle_openai_chat(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle OpenAI /v1/chat/completions endpoint."""
- from fastapi import HTTPException
- from fastapi.responses import JSONResponse, Response
-
- from headroom.ccr import CCRToolInjector
- from headroom.proxy.helpers import (
- COMPRESSION_TIMEOUT_SECONDS,
- MAX_MESSAGE_ARRAY_LENGTH,
- MAX_REQUEST_BODY_SIZE,
- _read_request_json,
- )
- from headroom.proxy.modes import is_cache_mode, is_token_mode
- from headroom.tokenizers import get_tokenizer
- from headroom.utils import extract_user_query
-
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "type": "invalid_request_error",
- "code": "request_too_large",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
- model = body.get("model", "unknown")
- messages = body.get("messages", [])
- original_client_messages = copy.deepcopy(messages)
-
- # Validate message array size
- if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Message array too large ({len(messages)} messages). "
- f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
- "type": "invalid_request_error",
- "code": "invalid_request",
- }
- },
- )
-
- stream = body.get("stream", False)
-
- # Bypass: skip ALL compression for explicit opt-out
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- if _bypass:
- logger.info(f"[{request_id}] Bypass: skipping compression (header)")
-
- # Image compression: tile alignment + ML-based technique routing
- if self.config.image_optimize and messages and not _bypass:
- from headroom.proxy.helpers import _get_image_compressor
-
- compressor = _get_image_compressor()
- if compressor and compressor.has_images(messages):
- messages = compressor.compress(messages, provider="openai")
- if compressor.last_result:
- logger.info(
- f"[{request_id}] Image: {compressor.last_result.technique.value} "
- f"({compressor.last_result.savings_percent:.0f}% saved, "
- f"{compressor.last_result.original_tokens} → "
- f"{compressor.last_result.compressed_tokens} tokens)"
- )
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- # Strip accept-encoding so httpx negotiates its own encoding.
- # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
- # if httpx lacks brotli support the response body is undecipherable → 502.
- headers.pop("accept-encoding", None)
- tags = self._extract_tags(headers)
-
- # Memory: Get user ID when memory is enabled
- memory_user_id: str | None = None
- if self.memory_handler:
- memory_user_id = headers.get(
- "x-headroom-user-id",
- os.environ.get("USER", os.environ.get("USERNAME", "default")),
- )
-
- # Rate limiting
- if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited(provider="openai")
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Check cache
- if self.cache and not stream:
- cached = await self.cache.get(messages, model)
- if cached:
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0, # Savings already counted when response was cached
- latency_ms=(time.time() - start_time) * 1000,
- cached=True,
- )
-
- # Remove compression headers from cached response
- response_headers = dict(cached.response_headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(content=cached.response_body, headers=response_headers)
-
- # Token counting
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Hook: pre_compress
- _hook_biases = None
- if self.config.hooks:
- from headroom.hooks import CompressContext
-
- _hook_ctx = CompressContext(model=model, provider="openai")
- try:
- messages = self.config.hooks.pre_compress(messages, _hook_ctx)
- _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
- except Exception as e:
- logger.debug(f"[{request_id}] Hook error: {e}")
-
- # Optimization
- transforms_applied = []
- pipeline_timing: dict[str, float] = {}
- waste_signals_dict: dict[str, int] | None = None
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- # Get prefix cache tracker for this session
- openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
- openai_prefix_tracker = self.session_tracker_store.get_or_create(
- openai_session_id, "openai"
- )
- openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
- if is_cache_mode(self.config.mode):
- openai_frozen_count = self._strict_previous_turn_frozen_count(
- original_client_messages,
- openai_frozen_count,
- )
-
- _compression_failed = False
- original_messages = messages # Preserve for 400-retry fallback
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
- if self.config.optimize and messages and not _bypass and _license_ok:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
-
- if is_token_mode(self.config.mode):
- comp_cache = self._get_compression_cache(openai_session_id)
-
- # Zone 1: Swap cached compressed versions
- working_messages = comp_cache.apply_cached(messages)
-
- # Re-freeze boundary
- openai_frozen_count = comp_cache.compute_frozen_count(messages)
-
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=working_messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(working_messages),
- frozen_message_count=openai_frozen_count,
- biases=_hook_biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != working_messages:
- comp_cache.update_from_result(messages, result.messages)
-
- # Always use pipeline result in token mode
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- # Keep original_tokens as the REAL original (pre-Zone-1-swap)
- # so tokens_saved captures both Zone 1 + Zone 2 savings.
- optimized_tokens = result.tokens_after
- else:
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- frozen_message_count=openai_frozen_count,
- biases=_hook_biases,
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- pipeline_timing = result.timing
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
-
- if result.waste_signals:
- waste_signals_dict = result.waste_signals.to_dict()
- except Exception as e:
- logger.warning(f"Optimization failed: {e}")
- # Flag compression failure for observability
- _compression_failed = True
-
- # Guard: if "optimization" inflated tokens, revert to originals
- if optimized_tokens > original_tokens:
- logger.warning(
- f"[{request_id}] Optimization inflated tokens "
- f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
- )
- optimized_messages = original_messages
- optimized_tokens = original_tokens
- transforms_applied = []
-
- tokens_saved = original_tokens - optimized_tokens
- optimization_latency = (time.time() - start_time) * 1000
-
- # Hook: post_compress
- if self.config.hooks and tokens_saved > 0:
- from headroom.hooks import CompressEvent
-
- try:
- self.config.hooks.post_compress(
- CompressEvent(
- tokens_before=original_tokens,
- tokens_after=optimized_tokens,
- tokens_saved=tokens_saved,
- compression_ratio=tokens_saved / original_tokens
- if original_tokens > 0
- else 0,
- transforms_applied=transforms_applied,
- model=model,
- provider="openai",
- )
- )
- except Exception as e:
- logger.debug(f"[{request_id}] post_compress hook error: {e}")
-
- # CCR Tool Injection: Inject retrieval tool if compression occurred
- tools = body.get("tools")
- _original_tools = tools # Preserve for diagnostic / future retry
- if (
- self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
- ) and not _bypass:
- injector = CCRToolInjector(
- provider="openai",
- inject_tool=self.config.ccr_inject_tool,
- inject_system_instructions=self.config.ccr_inject_system_instructions,
- )
- optimized_messages, tools, was_injected = injector.process_request(
- optimized_messages, tools
- )
-
- if injector.has_compressed_content:
- if was_injected:
- logger.debug(
- f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
- )
- else:
- logger.debug(
- f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
- )
-
- # Query Echo: disabled — hurts prefix caching in long conversations.
- if is_cache_mode(self.config.mode):
- optimized_messages, restored_count = self._restore_frozen_prefix(
- original_client_messages,
- optimized_messages,
- frozen_message_count=openai_frozen_count,
- )
- if restored_count > 0:
- logger.warning(
- f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
- "to preserve cache stability (openai)"
- )
-
- # Memory: inject context and tools for OpenAI requests
- if self.memory_handler and memory_user_id:
- try:
- # Inject memory context (search similar memories, add as system message)
- if self.memory_handler.config.inject_context:
- memory_context = await self.memory_handler.search_and_format_context(
- memory_user_id, optimized_messages
- )
- if memory_context:
- # Prepend as system message for OpenAI format
- optimized_messages = [
- {"role": "system", "content": memory_context},
- *optimized_messages,
- ]
- logger.info(
- f"[{request_id}] Memory: Injected {len(memory_context)} chars "
- f"of context for user {memory_user_id}"
- )
-
- # Inject memory tools
- if self.memory_handler.config.inject_tools:
- tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "openai")
- if mem_tools_injected:
- logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")
- except Exception as e:
- logger.warning(f"[{request_id}] Memory injection failed: {e}")
-
- body["messages"] = optimized_messages
- if tools is not None:
- body["tools"] = tools
-
- # Route through LiteLLM/any-llm backend if configured
- if self.anthropic_backend is not None:
- try:
- if stream:
- # Streaming: use stream_openai_message() → SSE events
- return await self._stream_openai_via_backend(
- body,
- headers,
- model,
- request_id,
- start_time,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- )
- else:
- # Non-streaming: use send_openai_message() → JSON
- backend_response = await self.anthropic_backend.send_openai_message(
- body, headers
- )
-
- if backend_response.error:
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
-
- # Track metrics
- total_latency = (time.time() - start_time) * 1000
- usage = backend_response.body.get("usage", {})
- output_tokens = usage.get("completion_tokens", 0)
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
-
- await self.metrics.record_request(
- provider=self.anthropic_backend.name,
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- cached=False,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"
- )
-
- return JSONResponse(
- status_code=backend_response.status_code,
- content=backend_response.body,
- )
- except Exception as e:
- logger.error(f"[{request_id}] Backend error: {e}")
- return JSONResponse(
- status_code=500,
- content={
- "error": {
- "message": str(e),
- "type": "api_error",
- "code": "backend_error",
- }
- },
- )
-
- # Direct OpenAI API (no backend configured)
- url = f"{self.OPENAI_API_URL}/v1/chat/completions"
-
- try:
- if stream:
- # Inject stream_options to get usage stats in streaming response
- # This allows accurate token counting instead of byte-based estimation
- if "stream_options" not in body:
- body["stream_options"] = {"include_usage": True}
- elif isinstance(body.get("stream_options"), dict):
- body["stream_options"]["include_usage"] = True
-
- return await self._stream_response(
- url,
- headers,
- body,
- "openai",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- pipeline_timing=pipeline_timing,
- prefix_tracker=openai_prefix_tracker,
- )
- else:
- headers = await apply_copilot_api_auth(headers, url=url)
- response = await self._retry_request("POST", url, headers, body)
-
- # Full diagnostic dump on upstream errors (OpenAI handler)
- if response.status_code >= 400:
- try:
- err_body = response.json()
- err_msg = err_body.get("error", {}).get("message", "")
- err_type = err_body.get("error", {}).get("type", "")
- except Exception:
- err_body = {"raw": response.text[:2000]}
- err_msg = str(response.text[:500])
- err_type = "parse_error"
-
- logger.warning(
- f"[{request_id}] UPSTREAM_ERROR "
- f"status={response.status_code} "
- f"error_type={err_type} "
- f"error_msg={err_msg!r} "
- f"model={model} "
- f"compressed={'yes' if transforms_applied else 'no'} "
- f"transforms={transforms_applied} "
- f"original_tokens={original_tokens} "
- f"optimized_tokens={optimized_tokens} "
- f"message_count={len(body.get('messages', []))} "
- f"stream={stream}"
- )
-
- try:
- from headroom import paths as _hr_paths
-
- debug_dir = _hr_paths.debug_400_dir()
- debug_dir.mkdir(parents=True, exist_ok=True)
- ts = datetime.now().strftime("%Y%m%d_%H%M%S")
- debug_file = debug_dir / f"{ts}_{request_id}.json"
-
- safe_headers = {}
- for k, v in headers.items():
- if k.lower() in ("x-api-key", "authorization"):
- safe_headers[k] = v[:12] + "..." if v else ""
- else:
- safe_headers[k] = v
-
- debug_payload = {
- "request_id": request_id,
- "timestamp": datetime.now().isoformat(),
- "status_code": response.status_code,
- "error_response": err_body,
- "model": model,
- "stream": stream,
- "headers": safe_headers,
- "compression": {
- "was_compressed": bool(transforms_applied),
- "transforms": transforms_applied,
- "original_tokens": original_tokens,
- "optimized_tokens": optimized_tokens,
- "tokens_saved": tokens_saved,
- "compression_failed": _compression_failed,
- },
- "tools_sent": body.get("tools"),
- "tool_count": len(body.get("tools") or []),
- "original_tool_count": len(_original_tools or []),
- "messages_sent": body.get("messages"),
- "message_count": len(body.get("messages", [])),
- "original_messages": (
- original_messages
- if original_messages is not body.get("messages")
- else "__same_as_sent__"
- ),
- "original_message_count": len(original_messages),
- "system_prompt": body.get("system"),
- }
-
- with open(debug_file, "w") as f:
- json.dump(debug_payload, f, indent=2, default=str)
-
- logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
- except Exception as dump_err:
- logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
-
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = optimized_tokens # fallback
- output_tokens = 0
- cache_read_tokens = 0
- resp_json = None
- try:
- resp_json = response.json()
- usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
- output_tokens = usage.get("completion_tokens", 0)
- # OpenAI returns cached_tokens in prompt_tokens_details
- # These are charged at 50% of the input price
- prompt_details = usage.get("prompt_tokens_details") or {}
- cache_read_tokens = prompt_details.get("cached_tokens", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
- )
-
- # Update prefix cache tracker for next turn
- openai_prefix_tracker.update_from_response(
- cache_read_tokens=cache_read_tokens,
- cache_write_tokens=0, # OpenAI doesn't report write tokens
- messages=optimized_messages,
- )
-
- # OpenAI has no write penalty — uncached = total - cached
- uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(
- model,
- tokens_saved,
- optimized_tokens,
- cache_read_tokens=cache_read_tokens,
- uncached_tokens=uncached_input_tokens,
- )
-
- # Memory: handle memory tool calls in OpenAI Chat Completions response.
- # After executing tools, send a continuation request so the model
- # can produce a final user-facing response (not just tool_calls).
- if (
- self.memory_handler
- and memory_user_id
- and resp_json
- and response.status_code == 200
- and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
- ):
- try:
- tool_results = await self.memory_handler.handle_memory_tool_calls(
- resp_json, memory_user_id, "openai"
- )
- if tool_results:
- # Build continuation: original messages + assistant tool_calls + tool results
- assistant_msg = resp_json.get("choices", [{}])[0].get("message", {})
- continuation_messages = list(optimized_messages)
- continuation_messages.append(assistant_msg)
- continuation_messages.extend(tool_results)
-
- continuation_body = {
- **body,
- "messages": continuation_messages,
- }
-
- cont_response = await self._retry_request(
- "POST", url, headers, continuation_body
- )
- if cont_response.status_code == 200:
- resp_json = cont_response.json()
- response = cont_response
-
- logger.info(
- f"[{request_id}] Memory: Handled {len(tool_results)} "
- f"tool call(s) with continuation for user {memory_user_id}"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Memory tool handling failed: {e}")
-
- # Cache
- if self.cache and response.status_code == 200:
- await self.cache.set(
- messages,
- model,
- response.content,
- dict(response.headers),
- tokens_saved,
- )
-
- # Capture Codex rate-limit window data from response headers
- from headroom.subscription.codex_rate_limits import (
- get_codex_rate_limit_state,
- )
-
- get_codex_rate_limit_state().update_from_headers(dict(response.headers))
-
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- pipeline_timing=pipeline_timing,
- waste_signals=waste_signals_dict,
- cache_read_tokens=cache_read_tokens,
- uncached_input_tokens=uncached_input_tokens,
- )
-
- if tokens_saved > 0:
- logger.info(
- f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
- f"(saved {tokens_saved:,} tokens)"
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Inject Headroom compression metrics (for SaaS metering)
- response_headers["x-headroom-tokens-before"] = str(original_tokens)
- response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
- response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
- response_headers["x-headroom-model"] = model
- if transforms_applied:
- response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
- if cache_read_tokens > 0:
- response_headers["x-headroom-cached"] = "true"
- if _compression_failed:
- response_headers["x-headroom-compression-failed"] = "true"
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed(provider="openai")
- # Log full error details internally for debugging
- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
- # Return sanitized error message to client (don't expose internal details)
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "type": "server_error",
- "code": "proxy_error",
- }
- },
- )
-
- async def handle_openai_responses(
- self,
- request: Request,
- ) -> Response | StreamingResponse:
- """Handle OpenAI /v1/responses endpoint (new Responses API).
-
- The Responses API differs from /v1/chat/completions:
- - Input: `input` (string or array) instead of `messages`
- - System: `instructions` instead of system message
- - Output: `output[]` array instead of `choices[].message`
- - State: `previous_response_id` for multi-turn
- - Built-in tools: web_search, file_search, code_interpreter
- """
- from fastapi import HTTPException
- from fastapi.responses import JSONResponse, Response
-
- from headroom.proxy.helpers import (
- COMPRESSION_TIMEOUT_SECONDS,
- MAX_REQUEST_BODY_SIZE,
- _read_request_json,
- )
- from headroom.proxy.responses_converter import (
- messages_to_responses_items,
- responses_items_to_messages,
- )
- from headroom.tokenizers import get_tokenizer
- from headroom.utils import extract_user_query
-
- start_time = time.time()
- request_id = await self._next_request_id()
-
- # Check request body size
- content_length = request.headers.get("content-length")
- if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
- return JSONResponse(
- status_code=413,
- content={
- "error": {
- "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
- "type": "invalid_request_error",
- "code": "request_too_large",
- }
- },
- )
-
- # Parse request
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid request body: {e!s}",
- "type": "invalid_request_error",
- "code": "invalid_json",
- }
- },
- )
-
- model = body.get("model", "unknown")
- stream = body.get("stream", False)
-
- # Convert Responses API input to messages format for optimization.
- # The Responses API uses a different item model (function_call,
- # function_call_output, reasoning as top-level items) — we convert to
- # Chat Completions messages for the pipeline, then convert back.
-
- input_data = body.get("input", "")
- instructions = body.get("instructions")
- previous_response_id = body.get("previous_response_id")
-
- messages: list[dict[str, Any]] = []
- original_items: list[dict[str, Any]] | None = None
- preserved_indices: list[int] = []
-
- if instructions:
- messages.append({"role": "system", "content": instructions})
-
- if isinstance(input_data, str):
- messages.append({"role": "user", "content": input_data})
- elif isinstance(input_data, list):
- original_items = input_data
- converted, preserved_indices = responses_items_to_messages(input_data)
- messages.extend(converted)
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("content-length", None)
- # Strip accept-encoding so httpx negotiates its own encoding.
- # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
- # if httpx lacks brotli support the response body is undecipherable → 502.
- headers.pop("accept-encoding", None)
- tags = self._extract_tags(headers)
-
- # Memory: Get user ID when memory is enabled
- memory_user_id: str | None = None
- if self.memory_handler:
- memory_user_id = headers.get(
- "x-headroom-user-id",
- os.environ.get("USER", os.environ.get("USERNAME", "default")),
- )
-
- # Rate limiting
- if self.rate_limiter:
- rate_key = headers.get("authorization", "default")[:20]
- allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
- if not allowed:
- await self.metrics.record_rate_limited(provider="openai")
- raise HTTPException(
- status_code=429,
- detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
- )
-
- # Token counting on converted messages
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- # Optimize: convert items → compress → convert back
- tokens_saved = 0
- transforms_applied: list[str] = []
- optimized_messages = messages
- optimized_tokens = original_tokens
-
- _bypass = (
- request.headers.get("x-headroom-bypass", "").lower() == "true"
- or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
- )
- _should_compress = (
- self.config.optimize
- and original_items is not None
- and not previous_response_id
- and not _bypass
- and len(messages) > 1
- )
- _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
-
- if _should_compress and _license_ok:
- try:
- context_limit = self.openai_provider.get_context_limit(model)
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
- if result.messages != messages:
- optimized_messages = result.messages
- transforms_applied = result.transforms_applied
- original_tokens = result.tokens_before
- optimized_tokens = result.tokens_after
- except Exception as e:
- logger.warning(f"[{request_id}] Responses API optimization failed: {e}")
-
- # Guard: if "optimization" inflated tokens, revert to originals
- if optimized_tokens > original_tokens:
- logger.warning(
- f"[{request_id}] Optimization inflated tokens "
- f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
- )
- optimized_messages = messages
- optimized_tokens = original_tokens
- transforms_applied = []
-
- tokens_saved = original_tokens - optimized_tokens
- optimization_latency = (time.time() - start_time) * 1000
-
- # Convert compressed messages back to Responses API items
- if optimized_messages is not messages and original_items is not None:
- opt_msgs = optimized_messages
- # Strip system message (instructions) — it's separate in Responses API
- if instructions and opt_msgs and opt_msgs[0].get("role") == "system":
- body["instructions"] = opt_msgs[0]["content"]
- opt_msgs = opt_msgs[1:]
-
- body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)
-
- # Memory: inject context and tools for Responses API requests
- if self.memory_handler and memory_user_id:
- try:
- # Inject memory context into instructions (Responses API system prompt)
- if self.memory_handler.config.inject_context:
- try:
- memory_context = await asyncio.wait_for(
- self.memory_handler.search_and_format_context(
- memory_user_id, optimized_messages
- ),
- timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- memory_context = None
- logger.info(
- f"[{request_id}] Memory context lookup exceeded "
- f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"
- )
- if memory_context:
- existing_instructions = body.get("instructions") or ""
- if existing_instructions:
- body["instructions"] = f"{existing_instructions}\n\n{memory_context}"
- else:
- body["instructions"] = memory_context
- logger.info(
- f"[{request_id}] Memory: Injected {len(memory_context)} chars "
- f"of context into instructions for user {memory_user_id}"
- )
-
- # Inject memory tools (Responses API format)
- if self.memory_handler.config.inject_tools:
- resp_tools = body.get("tools") or []
- resp_tools, mem_tools_injected = self.memory_handler.inject_tools(
- resp_tools, "openai"
- )
- if mem_tools_injected:
- # Convert Chat Completions format to Responses API format
- converted_tools = []
- for t in resp_tools:
- if t.get("type") == "function" and "function" in t:
- fn = t["function"]
- converted_tools.append(
- {
- "type": "function",
- "name": fn.get("name"),
- "description": fn.get("description", ""),
- "parameters": fn.get("parameters", {}),
- }
- )
- else:
- converted_tools.append(t)
- body["tools"] = converted_tools
- logger.info(
- f"[{request_id}] Memory: Injected memory tools (openai/responses)"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] Memory injection failed (responses): {e}")
-
- # /v1/responses is OpenAI-specific (Codex) — always routes direct.
- # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.
- if self.anthropic_backend is not None:
- logger.debug(
- f"[{request_id}] /v1/responses always routes to OpenAI direct "
- f"(backend '{self.anthropic_backend.name}' not used for Responses API)"
- )
-
- headers, is_chatgpt_auth = _resolve_codex_routing_headers(headers)
-
- # Route to correct endpoint based on auth mode.
- # ChatGPT session auth (codex login) uses chatgpt.com, not api.openai.com.
- if is_chatgpt_auth:
- url = "https://chatgpt.com/backend-api/codex/responses"
- else:
- url = f"{self.OPENAI_API_URL}/v1/responses"
-
- try:
- if stream:
- # Streaming for Responses API uses semantic events
- return await self._stream_response(
- url,
- headers,
- body,
- "openai",
- model,
- request_id,
- original_tokens,
- optimized_tokens,
- tokens_saved,
- transforms_applied,
- tags,
- optimization_latency,
- memory_user_id=memory_user_id,
- )
- else:
- headers = await apply_copilot_api_auth(headers, url=url)
- response = await self._retry_request("POST", url, headers, body)
- total_latency = (time.time() - start_time) * 1000
-
- total_input_tokens = original_tokens # fallback
- output_tokens = 0
- try:
- resp_json = response.json()
- usage = resp_json.get("usage", {})
- total_input_tokens = usage.get("input_tokens", original_tokens)
- output_tokens = usage.get("output_tokens", 0)
- except (KeyError, TypeError, AttributeError) as e:
- logger.debug(
- f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"
- )
-
- # Memory: handle memory tool calls in Responses API response
- if (
- self.memory_handler
- and memory_user_id
- and resp_json
- and response.status_code == 200
- and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
- ):
- try:
- # Extract function_call items from output
- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
-
- output_items = resp_json.get("output", [])
- memory_fc_items = [
- item
- for item in output_items
- if isinstance(item, dict)
- and item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ]
-
- # Execute memory tool calls
- tool_outputs: list[dict[str, Any]] = []
- for fc in memory_fc_items:
- call_id = fc.get("call_id", fc.get("id", ""))
- name = fc.get("name", "")
- args_str = fc.get("arguments", "{}")
- try:
- args = json.loads(args_str)
- except json.JSONDecodeError:
- args = {}
-
- await self.memory_handler._ensure_initialized()
- if self.memory_handler._backend:
- result = await self.memory_handler._execute_memory_tool(
- name, args, memory_user_id, "openai"
- )
- else:
- result = json.dumps({"error": "Memory backend not initialized"})
-
- tool_outputs.append(
- {
- "type": "function_call_output",
- "call_id": call_id,
- "output": result,
- }
- )
-
- if tool_outputs:
- # Make continuation request with tool results
- response_id = resp_json.get("id")
- continuation_body = {
- "model": model,
- "input": tool_outputs,
- }
- if response_id:
- continuation_body["previous_response_id"] = response_id
- existing_tools = body.get("tools")
- if existing_tools:
- continuation_body["tools"] = existing_tools
-
- cont_response = await self._retry_request(
- "POST", url, headers, continuation_body
- )
- resp_json = cont_response.json()
- response = cont_response
- logger.info(
- f"[{request_id}] Memory: Handled {len(tool_outputs)} "
- f"tool call(s) with continuation for user {memory_user_id} (responses)"
- )
- except Exception as e:
- logger.warning(
- f"[{request_id}] Memory tool handling failed (responses): {e}"
- )
-
- if self.cost_tracker:
- self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)
-
- await self.metrics.record_request(
- provider="openai",
- model=model,
- input_tokens=total_input_tokens,
- output_tokens=output_tokens,
- tokens_saved=tokens_saved,
- latency_ms=total_latency,
- overhead_ms=optimization_latency,
- )
-
- logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")
-
- # Capture Codex rate-limit window data from response headers
- from headroom.subscription.codex_rate_limits import (
- get_codex_rate_limit_state,
- )
-
- get_codex_rate_limit_state().update_from_headers(dict(response.headers))
-
- # Remove compression headers
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None)
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
- except Exception as e:
- await self.metrics.record_failed(provider="openai")
- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
- return JSONResponse(
- status_code=502,
- content={
- "error": {
- "message": "An error occurred while processing your request. Please try again.",
- "type": "server_error",
- "code": "proxy_error",
- }
- },
- )
-
- async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:
- """WebSocket proxy for /v1/responses (Codex gpt-5.4+).
-
- Newer Codex versions use WebSocket instead of HTTP POST for the
- Responses API. This handler:
- 1. Accepts the client WebSocket
- 2. Receives the first message (``response.create`` request)
- 3. Compresses the ``input`` array using the existing pipeline
- 4. Opens an upstream WebSocket to OpenAI
- 5. Sends the compressed request upstream
- 6. Relays all subsequent messages bidirectionally
- """
- from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
- from headroom.tokenizers import get_tokenizer
- from headroom.utils import extract_user_query
-
- try:
- import websockets
- except ImportError:
- await websocket.accept()
- await websocket.close(
- code=1011,
- reason="websockets package not installed. pip install websockets",
- )
- return
-
- request_id = await self._next_request_id()
- session_id = uuid.uuid4().hex
-
- # Stage-timer — captures per-stage durations for the structured
- # log emitted on session close. Unit 2 instrumentation.
- stage_timer = StageTimer()
- session_started_at = time.perf_counter()
-
- # Unit 3: initialize registry variables *before* accept so the
- # outermost ``finally`` can rely on them existing even if
- # registration itself fails for some reason.
- ws_sessions: WebSocketSessionRegistry | None = getattr(self, "ws_sessions", None)
- session_handle: WSSessionHandle | None = None
- termination_cause: TerminationCause = "unknown"
-
- # Forward client headers to upstream, adding required OpenAI-Beta header
- ws_headers = dict(websocket.headers)
-
- # Extract subprotocol from client — this is an application-level negotiation
- # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
- # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.
- client_subprotocols: list[str] = []
- raw_protocol = ws_headers.get("sec-websocket-protocol", "")
- if raw_protocol:
- client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]
-
- # Accept client connection with the requested subprotocol
- async with stage_timer.measure("accept"):
- if client_subprotocols:
- await websocket.accept(subprotocol=client_subprotocols[0])
- else:
- await websocket.accept()
-
- # --- Unit 3: register the session as soon as accept succeeds ---
- client_addr: str | None = None
- client_info = getattr(websocket, "client", None)
- if client_info is not None:
- host = getattr(client_info, "host", None)
- port = getattr(client_info, "port", None)
- if host is not None and port is not None:
- client_addr = f"{host}:{port}"
- elif host is not None:
- client_addr = str(host)
- if ws_sessions is not None:
- session_handle = WSSessionHandle(
- session_id=session_id,
- request_id=request_id,
- client_addr=client_addr,
- upstream_url=None, # set below once upstream_url is computed
- )
- ws_sessions.register(session_handle)
- metrics = getattr(self, "metrics", None)
- if metrics is not None and hasattr(metrics, "inc_active_ws_sessions"):
- try:
- metrics.inc_active_ws_sessions()
- except Exception: # pragma: no cover - defensive
- pass
-
- # Forward all client headers except hop-by-hop / per-connection headers.
- # These are WebSocket handshake mechanics that the `websockets` library
- # generates fresh for the upstream connection — forwarding them would conflict.
- # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.
- _skip_headers = frozenset(
- {
- "host", # must match upstream, not local proxy
- "connection", # hop-by-hop
- "upgrade", # hop-by-hop
- "sec-websocket-key", # per-connection cryptographic nonce
- "sec-websocket-version", # protocol version (websockets lib sets this)
- "sec-websocket-extensions", # per-connection negotiation
- "sec-websocket-accept", # server-side only
- "sec-websocket-protocol", # handled via subprotocols param below
- "content-length", # hop-by-hop
- "transfer-encoding", # hop-by-hop
- }
- )
- upstream_headers: dict[str, str] = {}
- for k, v in ws_headers.items():
- if k.lower() not in _skip_headers:
- upstream_headers[k] = v
-
- upstream_headers, is_chatgpt_auth = _resolve_codex_routing_headers(upstream_headers)
- _lower_headers = {k.lower(): v for k, v in upstream_headers.items()}
-
- # Build upstream WebSocket URL based on auth mode
- if is_chatgpt_auth:
- # ChatGPT session auth → route to chatgpt.com backend
- upstream_url = "wss://chatgpt.com/backend-api/codex/responses"
- logger.debug(
- f"[{request_id}] WS: ChatGPT session auth detected, routing to chatgpt.com"
- )
- else:
- # API key auth → route to configured OpenAI API URL
- base = self.OPENAI_API_URL
- ws_base = base.replace("https://", "wss://").replace("http://", "ws://")
- upstream_url = f"{ws_base}/v1/responses"
-
- # Unit 3: attach the resolved upstream URL to the session handle.
- if session_handle is not None:
- session_handle.upstream_url = upstream_url
-
- # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.
- # Safety net for clients that don't forward auth headers via WebSocket upgrade.
- if "authorization" not in _lower_headers:
- api_key = os.environ.get("OPENAI_API_KEY")
- if api_key:
- upstream_headers["Authorization"] = f"Bearer {api_key}"
- logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")
- else:
- logger.warning(
- f"[{request_id}] WS: no Authorization header from client and "
- f"OPENAI_API_KEY not set — upstream will likely reject"
- )
-
- upstream_headers = await apply_copilot_api_auth(upstream_headers, url=upstream_url)
-
- # Ensure the required beta header is present — OpenAI returns 500 without it.
- # Codex sends `responses_websockets=2026-02-06`; only inject if missing entirely.
- if "openai-beta" not in _lower_headers:
- upstream_headers["OpenAI-Beta"] = "responses_websockets=2026-02-06"
-
- logger.debug(
- f"[{request_id}] WS upstream headers: "
- f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "
- f"subprotocols={client_subprotocols}"
- )
-
- try:
- # Receive the first message from client (the response.create request).
- # Bound the wait with WS_FIRST_FRAME_TIMEOUT_SECONDS so a zombie
- # client that opens the WS but never sends a frame cannot hold a
- # session slot indefinitely. The StageTimer measurement still
- # captures the elapsed time up to the timeout so operators can
- # see the slow-client pattern in the stage-timings log.
- try:
- async with stage_timer.measure("first_client_frame"):
- first_msg_raw = await asyncio.wait_for(
- websocket.receive_text(),
- timeout=WS_FIRST_FRAME_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- logger.info(
- f"[{request_id}] WS first-frame timeout after "
- f"{WS_FIRST_FRAME_TIMEOUT_SECONDS:.0f}s; closing session "
- f"{session_id} (no client data)"
- )
- termination_cause = "client_timeout"
- with contextlib.suppress(Exception):
- # 1001 (going away): server is cleanly terminating a slow
- # client, not an internal error.
- await websocket.close(code=1001, reason="first-frame timeout")
- # Exit the outer try so the session-lifecycle ``finally`` runs
- # deregister / metrics / stage-timings emission as usual.
- return
-
- # --- Optional: compress the input in the first message ---
- body: dict[str, Any] = {}
- try:
- body = json.loads(first_msg_raw)
- tokens_saved = 0
- ws_request_body = body.get("response", body)
- input_data = (
- ws_request_body.get("input") if isinstance(ws_request_body, dict) else None
- )
-
- should_compress = (
- self.config.optimize
- and isinstance(input_data, list)
- and len(input_data) > 1
- and not (
- ws_request_body.get("previous_response_id")
- if isinstance(ws_request_body, dict)
- else None
- )
- )
- if should_compress:
- try:
- from headroom.proxy.responses_converter import (
- messages_to_responses_items,
- responses_items_to_messages,
- )
-
- model = ws_request_body.get("model", "gpt-4o")
- converted, preserved = responses_items_to_messages(input_data)
-
- messages: list[dict[str, Any]] = []
- instructions = ws_request_body.get("instructions")
- if instructions:
- messages.append({"role": "system", "content": instructions})
- messages.extend(converted)
-
- tokenizer = get_tokenizer(model)
- original_tokens = tokenizer.count_messages(messages)
-
- context_limit = self.openai_provider.get_context_limit(model)
- async with stage_timer.measure("compression"):
- result = await asyncio.wait_for(
- asyncio.to_thread(
- lambda: self.openai_pipeline.apply(
- messages=messages,
- model=model,
- model_limit=context_limit,
- context=extract_user_query(messages),
- )
- ),
- timeout=COMPRESSION_TIMEOUT_SECONDS,
- )
-
- if result.messages != messages:
- opt = result.messages
- if instructions and opt and opt[0].get("role") == "system":
- ws_request_body["instructions"] = opt[0]["content"]
- opt = opt[1:]
- if result.tokens_after <= original_tokens:
- ws_request_body["input"] = messages_to_responses_items(
- opt, input_data, preserved
- )
- else:
- logger.warning(
- f"[{request_id}] WS optimization inflated tokens "
- f"({original_tokens} -> {result.tokens_after}), reverting"
- )
- tokens_saved = max(0, original_tokens - result.tokens_after)
- if "response" in body and isinstance(body["response"], dict):
- body["response"] = ws_request_body
- else:
- body = ws_request_body
- first_msg_raw = json.dumps(body)
- logger.info(
- f"[{request_id}] WS /v1/responses compressed: "
- f"saved {tokens_saved} tokens"
- )
- except Exception as e:
- logger.warning(f"[{request_id}] WS compression failed: {e}")
-
- except json.JSONDecodeError:
- # Not JSON — pass through as-is
- tokens_saved = 0
-
- # --- Memory: inject context, tools, and instructions ---
- memory_user_id: str | None = None
- if self.memory_handler and body:
- memory_user_id = ws_headers.get(
- "x-headroom-user-id",
- os.environ.get("USER", os.environ.get("USERNAME", "default")),
- )
- try:
- # Unwrap response.create envelope to access the response body
- ws_response_body = body.get("response", body)
-
- # Debug: log what Codex sends so we can see the full tool list
- existing_tool_names = [
- t.get("name") or t.get("function", {}).get("name", "?")
- for t in (ws_response_body.get("tools") or [])
- ]
- instr_preview = (ws_response_body.get("instructions") or "")[:200]
- logger.info(
- f"[{request_id}] WS Memory: Codex tools={existing_tool_names}, "
- f"instructions_len={len(ws_response_body.get('instructions') or '')}, "
- f"instructions_preview={instr_preview!r}"
- )
-
- # Inject memory context into instructions
- if self.memory_handler.config.inject_context:
- ws_input = ws_response_body.get("input", "")
- ws_instructions = ws_response_body.get("instructions")
- ws_msgs: list[dict[str, Any]] = []
- if ws_instructions:
- ws_msgs.append({"role": "system", "content": ws_instructions})
- if isinstance(ws_input, str) and ws_input:
- ws_msgs.append({"role": "user", "content": ws_input})
- elif isinstance(ws_input, list):
- from headroom.proxy.responses_converter import (
- responses_items_to_messages,
- )
-
- converted_msgs, _ = responses_items_to_messages(ws_input)
- ws_msgs.extend(converted_msgs)
-
- try:
- async with stage_timer.measure("memory_context"):
- memory_context = await asyncio.wait_for(
- self.memory_handler.search_and_format_context(
- memory_user_id, ws_msgs
- ),
- timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
- )
- except asyncio.TimeoutError:
- memory_context = None
- logger.info(
- f"[{request_id}] WS Memory: Context lookup exceeded "
- f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "
- f"continuing without it"
- )
- if memory_context:
- existing = ws_response_body.get("instructions") or ""
- if existing:
- ws_response_body["instructions"] = f"{existing}\n\n{memory_context}"
- else:
- ws_response_body["instructions"] = memory_context
- logger.info(
- f"[{request_id}] WS Memory: Injected {len(memory_context)} chars "
- f"of context into instructions"
- )
-
- # Inject memory tools (Responses API format)
- if self.memory_handler.config.inject_tools:
- ws_tools = ws_response_body.get("tools") or []
- ws_tools, mem_injected = self.memory_handler.inject_tools(
- ws_tools, "openai"
- )
- if mem_injected:
- converted_tools = []
- for t in ws_tools:
- if t.get("type") == "function" and "function" in t:
- fn = t["function"]
- converted_tools.append(
- {
- "type": "function",
- "name": fn.get("name"),
- "description": fn.get("description", ""),
- "parameters": fn.get("parameters", {}),
- }
- )
- else:
- converted_tools.append(t)
- ws_response_body["tools"] = converted_tools
-
- # Add memory instruction so the model uses
- # memory tools as persistent cross-session knowledge.
- mem_instruction = (
- "\n\n## Memory\n"
- "You have persistent memory via memory_search and "
- "memory_save tools. Memory stores knowledge across "
- "sessions — user info, project details, org context, "
- "decisions, architecture, conventions, anything worth "
- "remembering.\n\n"
- "- ALWAYS call memory_search BEFORE searching files "
- "when the user asks a question that could be answered "
- "from prior knowledge.\n"
- "- Call memory_save to store important facts, decisions, "
- "or context that would be useful in future sessions.\n"
- "- Memory is your first source of truth for anything "
- "not visible in the current conversation."
- )
- existing_instr = ws_response_body.get("instructions") or ""
- ws_response_body["instructions"] = existing_instr + mem_instruction
- logger.info(
- f"[{request_id}] WS Memory: Injected memory tools + instruction"
- )
-
- # Write back into envelope if it was wrapped
- if "response" in body and isinstance(body["response"], dict):
- body["response"] = ws_response_body
- else:
- body = ws_response_body
-
- first_msg_raw = json.dumps(body)
- except Exception as e:
- logger.warning(f"[{request_id}] WS Memory injection failed: {e}")
-
- # --- Connect to upstream OpenAI WebSocket ---
- logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")
-
- # Use ssl=True to let the websockets library handle SSL natively.
- # Manual ssl.create_default_context() + certifi doesn't load the
- # Windows system cert store, causing HTTP 500 on wss:// connections.
- use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
-
- ws_connected = False
- ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
- ws_last_err: Exception | None = None
- _upstream_connect_started = time.perf_counter()
- _upstream_connect_recorded = False
- _upstream_first_event_started: float | None = None
-
- for ws_attempt in range(ws_connect_attempts):
- try:
- async with websockets.connect(
- upstream_url,
- additional_headers=upstream_headers,
- subprotocols=(
- [websockets.Subprotocol(p) for p in client_subprotocols]
- if client_subprotocols and hasattr(websockets, "Subprotocol")
- else client_subprotocols or None
- ),
- ssl=use_ssl,
- open_timeout=max(30, self.config.connect_timeout_seconds * 3),
- close_timeout=10,
- ping_interval=20,
- ping_timeout=20,
- ) as upstream:
- ws_connected = True
- if not _upstream_connect_recorded:
- stage_timer.record(
- "upstream_connect",
- (time.perf_counter() - _upstream_connect_started) * 1000.0,
- )
- _upstream_connect_recorded = True
- _upstream_first_event_started = time.perf_counter()
- await upstream.send(first_msg_raw)
-
- # Unit 3: flag the upstream side flips on seeing
- # ``response.completed`` so the outer cause
- # classifier can prefer it over the raw
- # "upstream iterator ended" default.
- response_completed_seen = False
- # Captures the first exception surfaced by the
- # inner relay ``except`` blocks so the outer
- # classifier can still tell ``upstream_error``
- # from ``upstream_disconnect`` / ``response_completed``
- # even though the halves swallow and log.
- upstream_relay_error: BaseException | None = None
- client_relay_error: BaseException | None = None
-
- async def _client_to_upstream() -> None:
- nonlocal client_relay_error
- try:
- while True:
- msg = await websocket.receive_text()
- await upstream.send(msg)
- except asyncio.CancelledError:
- # Explicit cancel from the outer
- # orchestrator — re-raise so
- # ``t.cancelled()`` and ``t.exception()``
- # behave correctly in the caller.
- raise
- except Exception as relay_err:
- # Surface real errors to the classifier
- # without re-raising (existing fork
- # behavior: log and return so the
- # partner task can be cancelled
- # deterministically).
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- client_relay_error = relay_err
- logger.debug(
- f"[{request_id}] WS client→upstream relay ended: {relay_err}"
- )
- with contextlib.suppress(Exception):
- await upstream.close()
-
- async def _upstream_to_client() -> None:
- """Relay upstream→client with transparent memory tool handling.
-
- Uses a buffer-then-decide approach:
- 1. Buffer events until first output item arrives
- 2. If first output is a memory tool → suppress entire response,
- execute tools silently, send continuation upstream
- 3. If first output is non-memory → flush buffer, stream normally
- 4. Continuation response events are relayed to Codex seamlessly
-
- This prevents orphaned response.created events from confusing Codex.
- """
- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
-
- # Unit 3: surface response.completed observation
- # to the outer scope so the termination-cause
- # classifier can prefer ``response_completed``
- # over ``upstream_disconnect``.
- nonlocal response_completed_seen
- nonlocal upstream_relay_error
-
- memory_enabled = bool(self.memory_handler and memory_user_id)
-
- # Per-response state (reset after each response.completed)
- event_buffer: list[str] = []
- decided = False
- suppress_response = False
- pending_fcs: list[dict[str, Any]] = []
- resp_id: str | None = None
-
- def _reset() -> None:
- nonlocal decided, suppress_response, resp_id
- event_buffer.clear()
- decided = False
- suppress_response = False
- pending_fcs.clear()
- resp_id = None
-
- # The retry-loop variable is safe to close over here:
- # ``_upstream_to_client`` is defined and awaited within
- # a single iteration and never escapes.
- _first_event_started_at = _upstream_first_event_started # noqa: B023
-
- try:
- async for msg in upstream:
- if (
- _first_event_started_at is not None
- and "upstream_first_event" not in stage_timer
- ):
- stage_timer.record(
- "upstream_first_event",
- (time.perf_counter() - _first_event_started_at)
- * 1000.0,
- )
- if isinstance(msg, bytes):
- await websocket.send_bytes(msg)
- continue
- msg_str = msg if isinstance(msg, str) else str(msg)
-
- if not memory_enabled:
- await websocket.send_text(msg_str)
- continue
-
- # Parse event
- try:
- event = json.loads(msg_str)
- except (json.JSONDecodeError, TypeError):
- await websocket.send_text(msg_str)
- continue
-
- event_type = event.get("type", "")
-
- # --- Phase 1: Buffer until first output item ---
- if not decided:
- event_buffer.append(msg_str)
-
- if event_type == "response.output_item.added":
- item = event.get("item", {})
- if (
- item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ):
- # Memory tool first → suppress entire response
- suppress_response = True
- decided = True
- event_buffer.clear()
- logger.info(
- f"[{request_id}] WS Memory: Detected "
- f"{item.get('name')} — suppressing response"
- )
- else:
- # Non-memory first → flush buffer, pass through
- decided = True
- for buf in event_buffer:
- await websocket.send_text(buf)
- event_buffer.clear()
-
- elif event_type == "response.completed":
- # No output items at all — flush
- decided = True
- for buf in event_buffer:
- await websocket.send_text(buf)
- event_buffer.clear()
- _reset()
- response_completed_seen = True
-
- continue
-
- # --- Phase 2a: Suppress mode (memory response) ---
- if suppress_response:
- if event_type == "response.output_item.done":
- item = event.get("item", {})
- if (
- item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ):
- pending_fcs.append(item)
-
- elif event_type == "response.completed":
- response_completed_seen = True
- resp = event.get("response", {})
- resp_id = resp.get("id")
-
- if pending_fcs:
- logger.info(
- f"[{request_id}] WS Memory: Executing "
- f"{len(pending_fcs)} tool(s) transparently"
- )
-
- # Execute memory tool calls
- tool_outputs: list[dict[str, Any]] = []
- for fc in pending_fcs:
- call_id = fc.get("call_id", fc.get("id", ""))
- fc_name = fc.get("name", "")
- args_str = fc.get("arguments", "{}")
- try:
- fc_args = json.loads(args_str)
- except json.JSONDecodeError:
- fc_args = {}
-
- await self.memory_handler._ensure_initialized()
- if self.memory_handler._backend:
- result = await self.memory_handler._execute_memory_tool(
- fc_name,
- fc_args,
- memory_user_id,
- "openai",
- )
- else:
- result = json.dumps(
- {"error": "backend not ready"}
- )
-
- tool_outputs.append(
- {
- "type": "function_call_output",
- "call_id": call_id,
- "output": result,
- }
- )
- logger.info(
- f"[{request_id}] WS Memory: Executed "
- f"{fc_name} for user {memory_user_id}"
- )
-
- # Send continuation upstream
- cont: dict[str, Any] = {
- "type": "response.create",
- "response": {"input": tool_outputs},
- }
- if resp_id:
- cont["response"]["previous_response_id"] = (
- resp_id
- )
- await upstream.send(json.dumps(cont))
- logger.info(
- f"[{request_id}] WS Memory: Sent continuation "
- f"with {len(tool_outputs)} result(s)"
- )
-
- _reset()
- # All events suppressed in this mode
- continue
-
- # --- Phase 2b: Pass-through mode ---
- await websocket.send_text(msg_str)
-
- except asyncio.CancelledError:
- raise
- except Exception as relay_err:
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- # Capture for the outer classifier
- # so ``upstream_error`` can be
- # distinguished from a clean
- # upstream disconnect.
- upstream_relay_error = relay_err
- logger.debug(
- f"[{request_id}] WS upstream→client relay ended: {relay_err}"
- )
- finally:
- with contextlib.suppress(Exception):
- await websocket.close()
-
- # --- Unit 3: deterministic relay-task cancellation ---
- # Spawn each half as a named task so we can:
- # (a) attach them to the session registry for
- # ``/debug/ws-sessions``,
- # (b) cancel the survivor explicitly when the
- # first one exits, and
- # (c) classify the termination cause for the
- # duration histogram.
- client_task = asyncio.create_task(
- _client_to_upstream(),
- name=f"codex-ws-c2u-{session_id}",
- )
- upstream_task = asyncio.create_task(
- _upstream_to_client(),
- name=f"codex-ws-u2c-{session_id}",
- )
- relay_tasks = [client_task, upstream_task]
- if ws_sessions is not None:
- ws_sessions.attach_tasks(session_id, relay_tasks)
- metrics_for_tasks = getattr(self, "metrics", None)
- if metrics_for_tasks is not None and hasattr(
- metrics_for_tasks, "inc_active_relay_tasks"
- ):
- try:
- metrics_for_tasks.inc_active_relay_tasks(len(relay_tasks))
- except Exception: # pragma: no cover - defensive
- pass
-
- try:
- done, pending = await asyncio.wait(
- {client_task, upstream_task},
- return_when=asyncio.FIRST_COMPLETED,
- )
- # Cancel the survivor so we don't leak the
- # partner task. Suppress the CancelledError
- # we just raised ourselves — any *other*
- # exception from the cancelled task is
- # already logged inside its own try/except.
- for t in pending:
- t.cancel()
- if pending:
- with contextlib.suppress(asyncio.CancelledError):
- await asyncio.gather(*pending, return_exceptions=True)
-
- # Classify termination cause from whichever
- # task completed first. ``CancelledError``
- # can show up on the "done" side if the
- # handler itself was cancelled from outside
- # (e.g. server shutdown).
- for t in done:
- exc = None
- # Cancelled tasks raise CancelledError from
- # .exception(); surface it explicitly so the
- # downstream ``isinstance(exc, CancelledError)``
- # branches actually run. For any other
- # unexpected state (``InvalidStateError`` if
- # the task somehow isn't done — shouldn't
- # happen post-gather but defensive), we
- # suppress and leave ``exc`` as ``None``.
- if t.cancelled():
- exc = asyncio.CancelledError()
- else:
- with contextlib.suppress(asyncio.InvalidStateError):
- exc = t.exception()
- task_name = t.get_name() or ""
- if t is client_task:
- if client_relay_error is not None:
- termination_cause = "client_error"
- elif exc is None:
- termination_cause = "client_disconnect"
- elif isinstance(exc, asyncio.CancelledError):
- termination_cause = "client_disconnect"
- else:
- # Distinguish legitimate client
- # disconnect exceptions from
- # real errors: WebSocketDisconnect
- # is a normal client exit.
- if "WebSocketDisconnect" in type(exc).__name__:
- termination_cause = "client_disconnect"
- else:
- termination_cause = "client_error"
- elif t is upstream_task:
- if upstream_relay_error is not None:
- termination_cause = "upstream_error"
- logger.debug(
- f"[{request_id}] WS relay {task_name} "
- f"raised: {upstream_relay_error!r}"
- )
- elif exc is None:
- termination_cause = (
- "response_completed"
- if response_completed_seen
- else "upstream_disconnect"
- )
- elif isinstance(exc, asyncio.CancelledError):
- termination_cause = "upstream_disconnect"
- else:
- termination_cause = "upstream_error"
- logger.debug(
- f"[{request_id}] WS relay {task_name} raised: {exc!r}"
- )
- finally:
- # In case anything above raised before the
- # cancel-and-await loop ran.
- for t in relay_tasks:
- if not t.done():
- t.cancel()
- with contextlib.suppress(asyncio.CancelledError):
- await asyncio.gather(*relay_tasks, return_exceptions=True)
-
- logger.info(
- f"[{request_id}] WS /v1/responses completed "
- f"(tokens_saved={tokens_saved}, cause={termination_cause})"
- )
- break
- except Exception as ws_err:
- if ws_connected:
- # WS was established but broke mid-stream — re-raise
- raise
-
- ws_last_err = ws_err
- if ws_attempt >= ws_connect_attempts - 1:
- break
-
- delay_with_jitter = jitter_delay_ms(
- self.config.retry_base_delay_ms,
- self.config.retry_max_delay_ms,
- ws_attempt,
- )
- logger.warning(
- f"[{request_id}] WS upstream connect failed "
- f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "
- f"retrying in {delay_with_jitter:.0f}ms"
- )
- await asyncio.sleep(delay_with_jitter / 1000)
-
- if not ws_connected:
- # WS upgrade failed (HTTP 500 from OpenAI is common).
- # Fall back to HTTP POST streaming and relay SSE events
- # back over the client WebSocket transparently.
- ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")
- _ws_detail = str(ws_err)
- if hasattr(ws_err, "response"):
- resp_body = getattr(getattr(ws_err, "response", None), "body", b"")
- if resp_body:
- _ws_detail += f" | {resp_body[:300].decode('utf-8', errors='replace')}"
- logger.warning(
- f"[{request_id}] WS upstream failed ({_ws_detail}), "
- f"falling back to HTTP POST streaming"
- )
- await self._ws_http_fallback(
- websocket, body, first_msg_raw, upstream_headers, request_id
- )
-
- # Record metrics
- if tokens_saved > 0:
- model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"
- await self.metrics.record_request(
- provider="openai",
- model=model_name,
- input_tokens=0,
- output_tokens=0,
- tokens_saved=tokens_saved,
- latency_ms=0,
- )
-
- except Exception as e:
- if "WebSocketDisconnect" in type(e).__name__:
- # Unit 3: client dropped the socket before or during
- # relay. The registry classifier may already have set
- # ``client_disconnect`` via the relay task exit path;
- # preserve that, otherwise set it here.
- if termination_cause == "unknown":
- termination_cause = "client_disconnect"
- else:
- # Extract response body from websockets InvalidStatus for better debugging
- error_detail = str(e)
- if hasattr(e, "response"):
- try:
- resp = e.response
- body_bytes = getattr(resp, "body", None) or b""
- if body_bytes:
- error_detail += (
- f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"
- )
- except Exception:
- pass
- logger.error(f"[{request_id}] WS proxy error: {error_detail}")
- if termination_cause == "unknown":
- termination_cause = "client_error"
- with contextlib.suppress(Exception):
- await websocket.close(code=1011, reason=str(e)[:120])
- finally:
- # Unit 2: emit structured per-session stage timings.
- stage_timer.record(
- "total_session",
- (time.perf_counter() - session_started_at) * 1000.0,
- )
- # Unit 3: deregister the session before (or independently
- # of) the stage-timings log so a failure there cannot leak
- # the registry entry. ``deregister`` is idempotent, so a
- # session that never registered is a no-op.
- if ws_sessions is not None and session_handle is not None:
- # Use deregister_and_count so the handle pop and the
- # relay-task count are read atomically inside the
- # registry. Capturing ``len(session_handle.relay_tasks)``
- # separately before ``deregister`` would risk drift if
- # the registry's bookkeeping ever changes.
- _deregistered, released_tasks = ws_sessions.deregister_and_count(
- session_id, cause=termination_cause
- )
- session_duration_ms = (time.perf_counter() - session_started_at) * 1000.0
- metrics_for_close = getattr(self, "metrics", None)
- if metrics_for_close is not None:
- with contextlib.suppress(Exception):
- if hasattr(metrics_for_close, "dec_active_ws_sessions"):
- metrics_for_close.dec_active_ws_sessions()
- if released_tasks and hasattr(metrics_for_close, "dec_active_relay_tasks"):
- metrics_for_close.dec_active_relay_tasks(released_tasks)
- if hasattr(metrics_for_close, "record_ws_session_duration"):
- metrics_for_close.record_ws_session_duration(
- session_duration_ms, termination_cause
- )
- await emit_stage_timings_log(
- path="openai_responses_ws",
- request_id=request_id,
- session_id=session_id,
- stage_timer=stage_timer,
- expected_stages=(
- "accept",
- "first_client_frame",
- "upstream_connect",
- "upstream_first_event",
- "memory_context",
- "compression",
- "total_session",
- ),
- metrics=getattr(self, "metrics", None),
- )
-
- async def _ws_http_fallback(
- self,
- websocket: WebSocket,
- body: dict[str, Any],
- first_msg_raw: str,
- upstream_headers: dict[str, str],
- request_id: str,
- ) -> None:
- """Fall back to HTTP POST streaming when upstream WS fails.
-
- Converts the WS ``response.create`` message to an HTTP POST to
- ``/v1/responses?stream=true``, reads SSE events, and relays each
- ``data:`` line as a WS text message to the client. This makes
- Codex work immediately instead of exhausting its WS retry budget.
- """
- # Route to correct endpoint based on auth mode
- _lower = {k.lower() for k in upstream_headers}
- if "chatgpt-account-id" in _lower:
- http_url = "https://chatgpt.com/backend-api/codex/responses"
- else:
- http_url = f"{self.OPENAI_API_URL}/v1/responses"
-
- # Build HTTP body from the WS response.create payload.
- # WS messages use {"type": "response.create", "response": {...}} wrapper.
- # The HTTP POST endpoint expects the inner response object directly.
- http_body: dict[str, Any]
- try:
- parsed = json.loads(first_msg_raw) if isinstance(first_msg_raw, str) else body
- except (json.JSONDecodeError, TypeError):
- parsed = body
-
- # Normalize WebSocket response.create payload into the HTTP request body.
- # Codex may send either:
- # 1. {"type":"response.create","response":{...}}
- # 2. {"type":"response.create", ...response fields...}
- if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):
- http_body = dict(parsed["response"])
- elif isinstance(parsed, dict):
- http_body = dict(parsed)
- if http_body.get("type") == "response.create":
- http_body.pop("type", None)
- else:
- http_body = body if isinstance(body, dict) else {}
-
- # Some clients include response-ish metadata that the HTTP endpoint rejects.
- if http_body.get("type") in {"response.create", "response"}:
- http_body.pop("type", None)
-
- # Ensure streaming is enabled so we get SSE events
- http_body["stream"] = True
-
- # Build HTTP headers from the upstream headers (already stripped of WS
- # hop-by-hop headers by the caller).
- http_headers = dict(upstream_headers)
- http_headers["content-type"] = "application/json"
-
- logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")
-
- try:
- retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
- for http_attempt in range(retry_attempts):
- try:
- async with self.http_client.stream(
- "POST",
- http_url,
- headers=http_headers,
- json=http_body,
- timeout=120.0,
- ) as response:
- if response.status_code != 200:
- error_body = b""
- async for chunk in response.aiter_bytes():
- error_body += chunk
- if len(error_body) > 2000:
- break
- error_text = error_body.decode("utf-8", errors="replace")
- logger.error(
- f"[{request_id}] WS→HTTP fallback got {response.status_code}: "
- f"{error_text[:500]}"
- )
- # Send error as WS message so client sees it
- error_event = {
- "type": "error",
- "error": {
- "type": "server_error",
- "message": f"Upstream returned {response.status_code}",
- },
- }
- await websocket.send_text(json.dumps(error_event))
- return
-
- # Relay SSE events as WS text messages
- buffer = ""
- async for chunk in response.aiter_text():
- buffer += chunk
- while "\n" in buffer:
- line, buffer = buffer.split("\n", 1)
- line = line.strip()
- if not line:
- continue
- if line.startswith("data: "):
- data = line[6:]
- if data == "[DONE]":
- continue
- try:
- await websocket.send_text(data)
- except Exception:
- return
- elif line.startswith("event: "):
- # SSE event type — skip, the data line contains the type
- continue
-
- # Flush any remaining data in buffer
- for line in buffer.strip().splitlines():
- line = line.strip()
- if line.startswith("data: ") and line[6:] != "[DONE]":
- with contextlib.suppress(Exception):
- await websocket.send_text(line[6:])
- return
- except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:
- if http_attempt >= retry_attempts - 1:
- raise
-
- delay_with_jitter = jitter_delay_ms(
- self.config.retry_base_delay_ms,
- self.config.retry_max_delay_ms,
- http_attempt,
- )
- logger.warning(
- f"[{request_id}] WS→HTTP fallback connect failed "
- f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "
- f"retrying in {delay_with_jitter:.0f}ms"
- )
- await asyncio.sleep(delay_with_jitter / 1000)
-
- except Exception as http_err:
- logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")
- error_event = {
- "type": "error",
- "error": {
- "type": "server_error",
- "message": f"HTTP fallback failed: {http_err!s}"[:200],
- },
- }
- with contextlib.suppress(Exception):
- await websocket.send_text(json.dumps(error_event))
- finally:
- with contextlib.suppress(Exception):
- await websocket.close()
-
- async def handle_compress(self, request: Request) -> JSONResponse:
- """Compress messages without calling an LLM.
-
- POST /v1/compress
- Body: {"messages": [...], "model": "...", "config": {}}
- Returns compressed messages + metrics.
- """
- from fastapi.responses import JSONResponse
-
- from headroom.proxy.helpers import _read_request_json
-
- # Check bypass header
- if request.headers.get("x-headroom-bypass", "").lower() == "true":
- try:
- body = await _read_request_json(request)
- except (json.JSONDecodeError, ValueError) as e:
- return JSONResponse(
- status_code=400,
- content={"error": f"Invalid request body: {e!s}"},
- )
- messages = body.get("messages", [])
- return JSONResponse(
- {
- "messages": messages,
- "tokens_before": 0,
- "tokens_after": 0,
- "tokens_saved": 0,
- "compression_ratio": 1.0,
- "transforms_applied": [],
- "ccr_hashes": [],
- }
- )
-
- try:
- body = await _read_request_json(request)
- except Exception:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Invalid JSON in request body.",
- }
- },
- )
-
- messages = body.get("messages")
- model = body.get("model")
-
- if messages is None:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Missing required field: messages",
- }
- },
- )
-
- if model is None:
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "type": "invalid_request",
- "message": "Missing required field: model",
- }
- },
- )
-
- if not messages:
- return JSONResponse(
- {
- "messages": [],
- "tokens_before": 0,
- "tokens_after": 0,
- "tokens_saved": 0,
- "compression_ratio": 1.0,
- "transforms_applied": [],
- "ccr_hashes": [],
- }
- )
-
- try:
- # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)
- # Allow optional token_budget to override model's context limit
- # (used by OpenClaw compact() and other callers that need tighter budgets)
- token_budget = body.get("token_budget")
- context_limit = (
- token_budget
- if token_budget and isinstance(token_budget, int)
- else self.openai_provider.get_context_limit(model)
- )
- # Extract CompressConfig options from request body
- compress_config = body.get("config", {})
- compress_user_messages = compress_config.get("compress_user_messages", False)
- target_ratio = compress_config.get("target_ratio")
- protect_recent = compress_config.get("protect_recent")
- protect_analysis_context = compress_config.get("protect_analysis_context")
-
- pipeline_kwargs: dict = {"model_limit": context_limit}
- if compress_user_messages:
- pipeline_kwargs["compress_user_messages"] = True
- if target_ratio is not None:
- pipeline_kwargs["target_ratio"] = float(target_ratio)
- if protect_recent is not None:
- pipeline_kwargs["protect_recent"] = int(protect_recent)
- if protect_analysis_context is not None:
- pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)
-
- result = self.openai_pipeline.apply(
- messages=messages,
- model=model,
- **pipeline_kwargs,
- )
-
- return JSONResponse(
- {
- "messages": result.messages,
- "tokens_before": result.tokens_before,
- "tokens_after": result.tokens_after,
- "tokens_saved": result.tokens_before - result.tokens_after,
- "compression_ratio": (
- result.tokens_after / result.tokens_before
- if result.tokens_before > 0
- else 1.0
- ),
- "transforms_applied": result.transforms_applied,
- "transforms_summary": result.transforms_summary,
- "ccr_hashes": result.markers_inserted,
- }
- )
- except Exception as e:
- logger.exception("Compression failed: %s", e)
- return JSONResponse(
- status_code=503,
- content={
- "error": {
- "type": "compression_error",
- "message": str(e),
- }
- },
- )
-
- async def handle_passthrough(
- self,
- request: Request,
- base_url: str,
- endpoint_name: str | None = None,
- provider: str | None = None,
- ) -> Response:
- """Pass through request unchanged.
-
- Args:
- request: The incoming request
- base_url: The upstream API base URL
- endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")
- provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")
- """
- from fastapi.responses import Response
-
- start_time = time.time()
- path = request.url.path
- url = f"{base_url}{path}"
-
- # Preserve query string parameters
- if request.url.query:
- url = f"{url}?{request.url.query}"
-
- headers = dict(request.headers.items())
- headers.pop("host", None)
- headers.pop("accept-encoding", None)
-
- body = await request.body()
-
- headers = await apply_copilot_api_auth(headers, url=url)
- response = await self.http_client.request( # type: ignore[union-attr]
- method=request.method,
- url=url,
- headers=headers,
- content=body,
- )
-
- # Remove compression headers since httpx already decompressed the response
- response_headers = dict(response.headers)
- response_headers.pop("content-encoding", None)
- response_headers.pop("content-length", None) # Length changed after decompression
-
- # Track stats for passthrough requests
- if endpoint_name and provider:
- latency_ms = (time.time() - start_time) * 1000
- await self.metrics.record_request(
- provider=provider,
- model=f"passthrough:{endpoint_name}",
- input_tokens=0,
- output_tokens=0,
- tokens_saved=0,
- latency_ms=latency_ms,
- cached=False,
- )
-
- return Response(
- content=response.content,
- status_code=response.status_code,
- headers=response_headers,
- )
-
- async def handle_databricks_invocations(
- self,
- request: Request,
- model: str,
- ) -> Response | StreamingResponse:
- """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.
-
- This enables using the Databricks CLI directly with Headroom:
- databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'
-
- The request/response format is identical to OpenAI chat completions,
- so we inject the model from the path and delegate to handle_openai_chat.
- """
- from fastapi.responses import JSONResponse
-
- from headroom.proxy.helpers import _read_request_json
-
- request_id = await self._next_request_id()
-
- try:
- body = await _read_request_json(request)
- except Exception as e:
- logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")
- return JSONResponse(
- status_code=400,
- content={
- "error": {
- "message": f"Invalid JSON: {e}",
- "type": "invalid_request_error",
- }
- },
- )
-
- # Inject model from path into body (Databricks CLI passes model in URL, not body)
- body["model"] = model
-
- logger.info(f"[{request_id}] Databricks invocation: model={model}")
-
- # Create a new request with the modified body
- # We reuse the OpenAI chat handler since the format is identical
- from starlette.requests import Request as StarletteRequest
-
- # Build new scope with the body already parsed
- scope = dict(request.scope)
-
- # Create a simple receive function that returns our modified body
- body_bytes = json.dumps(body).encode()
-
- async def receive():
- return {"type": "http.request", "body": body_bytes}
-
- modified_request = StarletteRequest(scope, receive)
-
- # Delegate to the OpenAI chat handler (same format)
- return await self.handle_openai_chat(modified_request)
+"""OpenAI handler mixin for HeadroomProxy.
+
+Contains all OpenAI Chat Completions, Responses API, and passthrough handlers.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import contextlib
+import copy
+import json
+import logging
+import os
+import time
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+from headroom.proxy.helpers import jitter_delay_ms
+from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
+from headroom.proxy.ws_session_registry import (
+ TerminationCause,
+ WebSocketSessionRegistry,
+ WSSessionHandle,
+)
+
+if TYPE_CHECKING:
+ from fastapi import Request, WebSocket
+ from fastapi.responses import JSONResponse, Response, StreamingResponse
+
+import httpx
+
+from headroom.copilot_auth import apply_copilot_api_auth
+from headroom.pipeline import PipelineStage, summarize_routing_markers
+
+logger = logging.getLogger("headroom.proxy")
+
+
+# Interactive Responses turns are latency-sensitive. Fail open quickly rather
+# than holding the session hostage on memory lookup.
+RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0
+
+# Cap the wait for the first client frame after the WS handshake completes.
+# A zombie or malicious client that accepts the upgrade but never sends the
+# first response.create frame would otherwise hold a slot indefinitely and
+# starve the session registry. 60 s is generous for real clients (Codex
+# typically sends the first frame within a few hundred milliseconds of the
+# accept) but short enough to bound the damage from a hung peer.
+WS_FIRST_FRAME_TIMEOUT_SECONDS = 60.0
+
+
+def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:
+ """Best-effort decode of an OpenAI OAuth bearer token payload.
+
+ OpenClaw's Codex OAuth flow may forward only the bearer token after the
+ provider base URL is overridden. In that case the explicit
+ ``ChatGPT-Account-ID`` header can be missing even though the JWT still
+ carries the account id we need to route to the ChatGPT Codex backend.
+ """
+ auth = headers.get("authorization") or headers.get("Authorization")
+ if not auth:
+ return None
+
+ scheme, _, token = auth.partition(" ")
+ if scheme.lower() != "bearer" or token.count(".") < 2:
+ return None
+
+ payload = token.split(".", 2)[1]
+ payload += "=" * (-len(payload) % 4)
+ # Intentionally no signature verification here: this is only a best-effort
+ # routing hint extractor. Upstream still performs the actual auth/authz checks.
+ try:
+ decoded = base64.urlsafe_b64decode(payload.encode("ascii"))
+ data = json.loads(decoded.decode("utf-8"))
+ except (ValueError, UnicodeDecodeError):
+ return None
+
+ return data if isinstance(data, dict) else None
+
+
+def _resolve_codex_routing_headers(headers: dict[str, str]) -> tuple[dict[str, str], bool]:
+ """Resolve ChatGPT Codex routing hints from explicit headers or OAuth JWT."""
+ resolved = dict(headers)
+ lower_lookup = {k.lower(): k for k in resolved}
+
+ if "chatgpt-account-id" in lower_lookup:
+ return resolved, True
+
+ payload = _decode_openai_bearer_payload(resolved)
+ auth_claims = payload.get("https://api.openai.com/auth") if isinstance(payload, dict) else None
+ account_id = auth_claims.get("chatgpt_account_id") if isinstance(auth_claims, dict) else None
+ if isinstance(account_id, str) and account_id.strip():
+ resolved["ChatGPT-Account-ID"] = account_id.strip()
+ return resolved, True
+
+ return resolved, False
+
+
+class OpenAIHandlerMixin:
+ """Mixin providing OpenAI API handler methods for HeadroomProxy."""
+
+ @staticmethod
+ def _strict_previous_turn_frozen_count(
+ messages: list[dict[str, Any]],
+ base_frozen_count: int,
+ ) -> int:
+ """Freeze all prior turns in cache mode; only final user turn is mutable."""
+ if not messages:
+ return base_frozen_count
+ final_idx = len(messages) - 1
+ if messages[final_idx].get("role") == "user":
+ return max(base_frozen_count, final_idx)
+ return len(messages)
+
+ @staticmethod
+ def _restore_frozen_prefix(
+ original_messages: list[dict[str, Any]],
+ candidate_messages: list[dict[str, Any]],
+ *,
+ frozen_message_count: int,
+ ) -> tuple[list[dict[str, Any]], int]:
+ """Force frozen prefix bytes to match original request exactly."""
+ if frozen_message_count <= 0 or not original_messages:
+ return candidate_messages, 0
+
+ frozen = min(frozen_message_count, len(original_messages))
+ restored = list(candidate_messages)
+
+ if len(restored) < frozen:
+ return list(original_messages[:frozen]) + restored, frozen
+
+ changed = 0
+ for idx in range(frozen):
+ if restored[idx] != original_messages[idx]:
+ restored[idx] = original_messages[idx]
+ changed += 1
+ return restored, changed
+
+ async def handle_openai_chat(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle OpenAI /v1/chat/completions endpoint."""
+ if not hasattr(self, "pipeline_extensions"):
+ from headroom.pipeline import PipelineExtensionManager
+
+ self.pipeline_extensions = PipelineExtensionManager(discover=False)
+
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.ccr import CCRToolInjector
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS,
+ MAX_MESSAGE_ARRAY_LENGTH,
+ MAX_REQUEST_BODY_SIZE,
+ _read_request_json,
+ )
+ from headroom.proxy.modes import is_cache_mode, is_token_mode
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+ model = body.get("model", "unknown")
+ messages = body.get("messages", [])
+ original_client_messages = copy.deepcopy(messages)
+ input_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=messages,
+ tools=body.get("tools"),
+ metadata={"path": "/v1/chat/completions", "stream": body.get("stream", False)},
+ )
+ if input_event.messages is not None:
+ messages = input_event.messages
+ original_client_messages = copy.deepcopy(messages)
+ if input_event.tools is not None:
+ body["tools"] = input_event.tools
+
+ # Validate message array size
+ if len(messages) > MAX_MESSAGE_ARRAY_LENGTH:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Message array too large ({len(messages)} messages). "
+ f"Maximum is {MAX_MESSAGE_ARRAY_LENGTH}.",
+ "type": "invalid_request_error",
+ "code": "invalid_request",
+ }
+ },
+ )
+
+ stream = body.get("stream", False)
+
+ # Bypass: skip ALL compression for explicit opt-out
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ if _bypass:
+ logger.info(f"[{request_id}] Bypass: skipping compression (header)")
+
+ # Image compression: tile alignment + ML-based technique routing
+ if self.config.image_optimize and messages and not _bypass:
+ from headroom.proxy.helpers import _get_image_compressor
+
+ compressor = _get_image_compressor()
+ if compressor and compressor.has_images(messages):
+ messages = compressor.compress(messages, provider="openai")
+ if compressor.last_result:
+ logger.info(
+ f"[{request_id}] Image: {compressor.last_result.technique.value} "
+ f"({compressor.last_result.savings_percent:.0f}% saved, "
+ f"{compressor.last_result.original_tokens} → "
+ f"{compressor.last_result.compressed_tokens} tokens)"
+ )
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ # Strip accept-encoding so httpx negotiates its own encoding.
+ # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
+ # if httpx lacks brotli support the response body is undecipherable → 502.
+ headers.pop("accept-encoding", None)
+ tags = self._extract_tags(headers)
+
+ # Memory: Get user ID when memory is enabled
+ memory_user_id: str | None = None
+ if self.memory_handler:
+ memory_user_id = headers.get(
+ "x-headroom-user-id",
+ os.environ.get("USER", os.environ.get("USERNAME", "default")),
+ )
+
+ # Rate limiting
+ if self.rate_limiter:
+ rate_key = headers.get("authorization", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider="openai")
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Check cache
+ if self.cache and not stream:
+ cached = await self.cache.get(messages, model)
+ if cached:
+ self.pipeline_extensions.emit(
+ PipelineStage.INPUT_CACHED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=messages,
+ metadata={"cache_hit": True, "path": "/v1/chat/completions"},
+ )
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0, # Savings already counted when response was cached
+ latency_ms=(time.time() - start_time) * 1000,
+ cached=True,
+ )
+
+ # Remove compression headers from cached response
+ response_headers = dict(cached.response_headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(content=cached.response_body, headers=response_headers)
+
+ # Token counting
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Hook: pre_compress
+ _hook_biases = None
+ if self.config.hooks:
+ from headroom.hooks import CompressContext
+
+ _hook_ctx = CompressContext(model=model, provider="openai")
+ try:
+ messages = self.config.hooks.pre_compress(messages, _hook_ctx)
+ _hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
+ except Exception as e:
+ logger.debug(f"[{request_id}] Hook error: {e}")
+
+ # Optimization
+ transforms_applied = []
+ pipeline_timing: dict[str, float] = {}
+ waste_signals_dict: dict[str, int] | None = None
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ # Get prefix cache tracker for this session
+ openai_session_id = self.session_tracker_store.compute_session_id(request, model, messages)
+ openai_prefix_tracker = self.session_tracker_store.get_or_create(
+ openai_session_id, "openai"
+ )
+ openai_frozen_count = openai_prefix_tracker.get_frozen_message_count()
+ if is_cache_mode(self.config.mode):
+ openai_frozen_count = self._strict_previous_turn_frozen_count(
+ original_client_messages,
+ openai_frozen_count,
+ )
+
+ _compression_failed = False
+ original_messages = messages # Preserve for 400-retry fallback
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+ if self.config.optimize and messages and not _bypass and _license_ok:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+
+ if is_token_mode(self.config.mode):
+ comp_cache = self._get_compression_cache(openai_session_id)
+
+ # Zone 1: Swap cached compressed versions
+ working_messages = comp_cache.apply_cached(messages)
+
+ # Re-freeze boundary
+ openai_frozen_count = comp_cache.compute_frozen_count(messages)
+
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=working_messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(working_messages),
+ frozen_message_count=openai_frozen_count,
+ biases=_hook_biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != working_messages:
+ comp_cache.update_from_result(messages, result.messages)
+
+ # Always use pipeline result in token mode
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ # Keep original_tokens as the REAL original (pre-Zone-1-swap)
+ # so tokens_saved captures both Zone 1 + Zone 2 savings.
+ optimized_tokens = result.tokens_after
+ else:
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ frozen_message_count=openai_frozen_count,
+ biases=_hook_biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+
+ if result.waste_signals:
+ waste_signals_dict = result.waste_signals.to_dict()
+ except Exception as e:
+ logger.warning(f"Optimization failed: {e}")
+ # Flag compression failure for observability
+ _compression_failed = True
+
+ # Guard: if "optimization" inflated tokens, revert to originals
+ if optimized_tokens > original_tokens:
+ logger.warning(
+ f"[{request_id}] Optimization inflated tokens "
+ f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
+ )
+ optimized_messages = original_messages
+ optimized_tokens = original_tokens
+ transforms_applied = []
+
+ tokens_saved = original_tokens - optimized_tokens
+ optimization_latency = (time.time() - start_time) * 1000
+
+ routing_markers = summarize_routing_markers(transforms_applied)
+ if routing_markers:
+ routed_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_ROUTED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "routing_markers": routing_markers,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if routed_event.messages is not None:
+ optimized_messages = routed_event.messages
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ tokens_saved = original_tokens - optimized_tokens
+
+ compressed_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_COMPRESSED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ metadata={
+ "tokens_before": original_tokens,
+ "tokens_after": optimized_tokens,
+ "transforms_applied": transforms_applied,
+ },
+ )
+ if compressed_event.messages is not None:
+ optimized_messages = compressed_event.messages
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ tokens_saved = original_tokens - optimized_tokens
+
+ # Hook: post_compress
+ if self.config.hooks and tokens_saved > 0:
+ from headroom.hooks import CompressEvent
+
+ try:
+ self.config.hooks.post_compress(
+ CompressEvent(
+ tokens_before=original_tokens,
+ tokens_after=optimized_tokens,
+ tokens_saved=tokens_saved,
+ compression_ratio=tokens_saved / original_tokens
+ if original_tokens > 0
+ else 0,
+ transforms_applied=transforms_applied,
+ model=model,
+ provider="openai",
+ )
+ )
+ except Exception as e:
+ logger.debug(f"[{request_id}] post_compress hook error: {e}")
+
+ # CCR Tool Injection: Inject retrieval tool if compression occurred
+ tools = body.get("tools")
+ _original_tools = tools # Preserve for diagnostic / future retry
+ if (
+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
+ ) and not _bypass:
+ injector = CCRToolInjector(
+ provider="openai",
+ inject_tool=self.config.ccr_inject_tool,
+ inject_system_instructions=self.config.ccr_inject_system_instructions,
+ )
+ optimized_messages, tools, was_injected = injector.process_request(
+ optimized_messages, tools
+ )
+
+ if injector.has_compressed_content:
+ if was_injected:
+ logger.debug(
+ f"[{request_id}] CCR: Injected retrieval tool for hashes: {injector.detected_hashes}"
+ )
+ else:
+ logger.debug(
+ f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}"
+ )
+
+ # Query Echo: disabled — hurts prefix caching in long conversations.
+ if is_cache_mode(self.config.mode):
+ optimized_messages, restored_count = self._restore_frozen_prefix(
+ original_client_messages,
+ optimized_messages,
+ frozen_message_count=openai_frozen_count,
+ )
+ if restored_count > 0:
+ logger.warning(
+ f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
+ "to preserve cache stability (openai)"
+ )
+
+ # Memory: inject context and tools for OpenAI requests
+ memory_context_injected = False
+ memory_tools_injected = False
+ if self.memory_handler and memory_user_id:
+ try:
+ # Inject memory context (search similar memories, add as system message)
+ if self.memory_handler.config.inject_context:
+ memory_context = await self.memory_handler.search_and_format_context(
+ memory_user_id, optimized_messages
+ )
+ if memory_context:
+ # Prepend as system message for OpenAI format
+ optimized_messages = [
+ {"role": "system", "content": memory_context},
+ *optimized_messages,
+ ]
+ memory_context_injected = True
+ logger.info(
+ f"[{request_id}] Memory: Injected {len(memory_context)} chars "
+ f"of context for user {memory_user_id}"
+ )
+
+ # Inject memory tools
+ if self.memory_handler.config.inject_tools:
+ tools, mem_tools_injected = self.memory_handler.inject_tools(tools, "openai")
+ if mem_tools_injected:
+ memory_tools_injected = True
+ logger.info(f"[{request_id}] Memory: Injected memory tools (openai)")
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory injection failed: {e}")
+
+ if memory_context_injected or memory_tools_injected:
+ remembered_event = self.pipeline_extensions.emit(
+ PipelineStage.INPUT_REMEMBERED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ tools=tools,
+ metadata={
+ "memory_context_injected": memory_context_injected,
+ "memory_tools_injected": memory_tools_injected,
+ },
+ )
+ if remembered_event.messages is not None:
+ optimized_messages = remembered_event.messages
+ if remembered_event.tools is not None:
+ tools = remembered_event.tools
+
+ body["messages"] = optimized_messages
+ if tools is not None:
+ body["tools"] = tools
+
+ presend_event = self.pipeline_extensions.emit(
+ PipelineStage.PRE_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=optimized_messages,
+ tools=tools,
+ headers=headers,
+ metadata={"path": "/v1/chat/completions", "stream": stream},
+ )
+ if presend_event.messages is not None:
+ optimized_messages = presend_event.messages
+ body["messages"] = optimized_messages
+ if presend_event.tools is not None:
+ tools = presend_event.tools
+ body["tools"] = tools
+ if presend_event.headers is not None:
+ headers = presend_event.headers
+ optimized_tokens = tokenizer.count_messages(body["messages"])
+ tokens_saved = original_tokens - optimized_tokens
+
+ # Route through LiteLLM/any-llm backend if configured
+ if self.anthropic_backend is not None:
+ try:
+ if stream:
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ metadata={"path": "/v1/chat/completions", "stream": True},
+ )
+ # Streaming: use stream_openai_message() → SSE events
+ return await self._stream_openai_via_backend(
+ body,
+ headers,
+ model,
+ request_id,
+ start_time,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+ else:
+ # Non-streaming: use send_openai_message() → JSON
+ backend_response = await self.anthropic_backend.send_openai_message(
+ body, headers
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ response=backend_response.body,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": backend_response.status_code,
+ },
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ response=backend_response.body,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": backend_response.status_code,
+ },
+ )
+
+ if backend_response.error:
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+
+ # Track metrics
+ total_latency = (time.time() - start_time) * 1000
+ usage = backend_response.body.get("usage", {})
+ output_tokens = usage.get("completion_tokens", 0)
+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+
+ await self.metrics.record_request(
+ provider=self.anthropic_backend.name,
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ cached=False,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens) via {self.anthropic_backend.name}"
+ )
+
+ return JSONResponse(
+ status_code=backend_response.status_code,
+ content=backend_response.body,
+ )
+ except Exception as e:
+ logger.error(f"[{request_id}] Backend error: {e}")
+ return JSONResponse(
+ status_code=500,
+ content={
+ "error": {
+ "message": str(e),
+ "type": "api_error",
+ "code": "backend_error",
+ }
+ },
+ )
+
+ # Direct OpenAI API (no backend configured)
+ url = f"{self.OPENAI_API_URL}/v1/chat/completions"
+
+ try:
+ if stream:
+ # Inject stream_options to get usage stats in streaming response
+ # This allows accurate token counting instead of byte-based estimation
+ if "stream_options" not in body:
+ body["stream_options"] = {"include_usage": True}
+ elif isinstance(body.get("stream_options"), dict):
+ body["stream_options"]["include_usage"] = True
+
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ metadata={"path": "/v1/chat/completions", "stream": True},
+ )
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "openai",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ pipeline_timing=pipeline_timing,
+ prefix_tracker=openai_prefix_tracker,
+ )
+ else:
+ headers = await apply_copilot_api_auth(headers, url=url)
+ response = await self._retry_request("POST", url, headers, body)
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_SEND,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ messages=body["messages"],
+ tools=tools,
+ response=response,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": response.status_code,
+ },
+ )
+ self.pipeline_extensions.emit(
+ PipelineStage.RESPONSE_RECEIVED,
+ operation="proxy.request",
+ request_id=request_id,
+ provider="openai",
+ model=model,
+ response=response,
+ metadata={
+ "path": "/v1/chat/completions",
+ "stream": False,
+ "status_code": response.status_code,
+ },
+ )
+
+ # Full diagnostic dump on upstream errors (OpenAI handler)
+ if response.status_code >= 400:
+ try:
+ err_body = response.json()
+ err_msg = err_body.get("error", {}).get("message", "")
+ err_type = err_body.get("error", {}).get("type", "")
+ except Exception:
+ err_body = {"raw": response.text[:2000]}
+ err_msg = str(response.text[:500])
+ err_type = "parse_error"
+
+ logger.warning(
+ f"[{request_id}] UPSTREAM_ERROR "
+ f"status={response.status_code} "
+ f"error_type={err_type} "
+ f"error_msg={err_msg!r} "
+ f"model={model} "
+ f"compressed={'yes' if transforms_applied else 'no'} "
+ f"transforms={transforms_applied} "
+ f"original_tokens={original_tokens} "
+ f"optimized_tokens={optimized_tokens} "
+ f"message_count={len(body.get('messages', []))} "
+ f"stream={stream}"
+ )
+
+ try:
+ from headroom import paths as _hr_paths
+
+ debug_dir = _hr_paths.debug_400_dir()
+ debug_dir.mkdir(parents=True, exist_ok=True)
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+ debug_file = debug_dir / f"{ts}_{request_id}.json"
+
+ safe_headers = {}
+ for k, v in headers.items():
+ if k.lower() in ("x-api-key", "authorization"):
+ safe_headers[k] = v[:12] + "..." if v else ""
+ else:
+ safe_headers[k] = v
+
+ debug_payload = {
+ "request_id": request_id,
+ "timestamp": datetime.now().isoformat(),
+ "status_code": response.status_code,
+ "error_response": err_body,
+ "model": model,
+ "stream": stream,
+ "headers": safe_headers,
+ "compression": {
+ "was_compressed": bool(transforms_applied),
+ "transforms": transforms_applied,
+ "original_tokens": original_tokens,
+ "optimized_tokens": optimized_tokens,
+ "tokens_saved": tokens_saved,
+ "compression_failed": _compression_failed,
+ },
+ "tools_sent": body.get("tools"),
+ "tool_count": len(body.get("tools") or []),
+ "original_tool_count": len(_original_tools or []),
+ "messages_sent": body.get("messages"),
+ "message_count": len(body.get("messages", [])),
+ "original_messages": (
+ original_messages
+ if original_messages is not body.get("messages")
+ else "__same_as_sent__"
+ ),
+ "original_message_count": len(original_messages),
+ "system_prompt": body.get("system"),
+ }
+
+ with open(debug_file, "w") as f:
+ json.dump(debug_payload, f, indent=2, default=str)
+
+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
+ except Exception as dump_err:
+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
+
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = optimized_tokens # fallback
+ output_tokens = 0
+ cache_read_tokens = 0
+ resp_json = None
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usage", {})
+ total_input_tokens = usage.get("prompt_tokens", optimized_tokens)
+ output_tokens = usage.get("completion_tokens", 0)
+ # OpenAI returns cached_tokens in prompt_tokens_details
+ # These are charged at 50% of the input price
+ prompt_details = usage.get("prompt_tokens_details") or {}
+ cache_read_tokens = prompt_details.get("cached_tokens", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from OpenAI response: {e}"
+ )
+
+ # Update prefix cache tracker for next turn
+ openai_prefix_tracker.update_from_response(
+ cache_read_tokens=cache_read_tokens,
+ cache_write_tokens=0, # OpenAI doesn't report write tokens
+ messages=optimized_messages,
+ )
+
+ # OpenAI has no write penalty — uncached = total - cached
+ uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens)
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(
+ model,
+ tokens_saved,
+ optimized_tokens,
+ cache_read_tokens=cache_read_tokens,
+ uncached_tokens=uncached_input_tokens,
+ )
+
+ # Memory: handle memory tool calls in OpenAI Chat Completions response.
+ # After executing tools, send a continuation request so the model
+ # can produce a final user-facing response (not just tool_calls).
+ if (
+ self.memory_handler
+ and memory_user_id
+ and resp_json
+ and response.status_code == 200
+ and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
+ ):
+ try:
+ tool_results = await self.memory_handler.handle_memory_tool_calls(
+ resp_json, memory_user_id, "openai"
+ )
+ if tool_results:
+ # Build continuation: original messages + assistant tool_calls + tool results
+ assistant_msg = resp_json.get("choices", [{}])[0].get("message", {})
+ continuation_messages = list(optimized_messages)
+ continuation_messages.append(assistant_msg)
+ continuation_messages.extend(tool_results)
+
+ continuation_body = {
+ **body,
+ "messages": continuation_messages,
+ }
+
+ cont_response = await self._retry_request(
+ "POST", url, headers, continuation_body
+ )
+ if cont_response.status_code == 200:
+ resp_json = cont_response.json()
+ response = cont_response
+
+ logger.info(
+ f"[{request_id}] Memory: Handled {len(tool_results)} "
+ f"tool call(s) with continuation for user {memory_user_id}"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory tool handling failed: {e}")
+
+ # Cache
+ if self.cache and response.status_code == 200:
+ await self.cache.set(
+ messages,
+ model,
+ response.content,
+ dict(response.headers),
+ tokens_saved,
+ )
+
+ # Capture Codex rate-limit window data from response headers
+ from headroom.subscription.codex_rate_limits import (
+ get_codex_rate_limit_state,
+ )
+
+ get_codex_rate_limit_state().update_from_headers(dict(response.headers))
+
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ pipeline_timing=pipeline_timing,
+ waste_signals=waste_signals_dict,
+ cache_read_tokens=cache_read_tokens,
+ uncached_input_tokens=uncached_input_tokens,
+ )
+
+ if tokens_saved > 0:
+ logger.info(
+ f"[{request_id}] {model}: {original_tokens:,} → {optimized_tokens:,} "
+ f"(saved {tokens_saved:,} tokens)"
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Inject Headroom compression metrics (for SaaS metering)
+ response_headers["x-headroom-tokens-before"] = str(original_tokens)
+ response_headers["x-headroom-tokens-after"] = str(optimized_tokens)
+ response_headers["x-headroom-tokens-saved"] = str(tokens_saved)
+ response_headers["x-headroom-model"] = model
+ if transforms_applied:
+ response_headers["x-headroom-transforms"] = ",".join(transforms_applied)
+ if cache_read_tokens > 0:
+ response_headers["x-headroom-cached"] = "true"
+ if _compression_failed:
+ response_headers["x-headroom-compression-failed"] = "true"
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed(provider="openai")
+ # Log full error details internally for debugging
+ logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
+ # Return sanitized error message to client (don't expose internal details)
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "type": "server_error",
+ "code": "proxy_error",
+ }
+ },
+ )
+
+ async def handle_openai_responses(
+ self,
+ request: Request,
+ ) -> Response | StreamingResponse:
+ """Handle OpenAI /v1/responses endpoint (new Responses API).
+
+ The Responses API differs from /v1/chat/completions:
+ - Input: `input` (string or array) instead of `messages`
+ - System: `instructions` instead of system message
+ - Output: `output[]` array instead of `choices[].message`
+ - State: `previous_response_id` for multi-turn
+ - Built-in tools: web_search, file_search, code_interpreter
+ """
+ from fastapi import HTTPException
+ from fastapi.responses import JSONResponse, Response
+
+ from headroom.proxy.helpers import (
+ COMPRESSION_TIMEOUT_SECONDS,
+ MAX_REQUEST_BODY_SIZE,
+ _read_request_json,
+ )
+ from headroom.proxy.responses_converter import (
+ messages_to_responses_items,
+ responses_items_to_messages,
+ )
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ start_time = time.time()
+ request_id = await self._next_request_id()
+
+ # Check request body size
+ content_length = request.headers.get("content-length")
+ if content_length and int(content_length) > MAX_REQUEST_BODY_SIZE:
+ return JSONResponse(
+ status_code=413,
+ content={
+ "error": {
+ "message": f"Request body too large. Maximum size is {MAX_REQUEST_BODY_SIZE // (1024 * 1024)}MB",
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ },
+ )
+
+ # Parse request
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid request body: {e!s}",
+ "type": "invalid_request_error",
+ "code": "invalid_json",
+ }
+ },
+ )
+
+ model = body.get("model", "unknown")
+ stream = body.get("stream", False)
+
+ # Convert Responses API input to messages format for optimization.
+ # The Responses API uses a different item model (function_call,
+ # function_call_output, reasoning as top-level items) — we convert to
+ # Chat Completions messages for the pipeline, then convert back.
+
+ input_data = body.get("input", "")
+ instructions = body.get("instructions")
+ previous_response_id = body.get("previous_response_id")
+
+ messages: list[dict[str, Any]] = []
+ original_items: list[dict[str, Any]] | None = None
+ preserved_indices: list[int] = []
+
+ if instructions:
+ messages.append({"role": "system", "content": instructions})
+
+ if isinstance(input_data, str):
+ messages.append({"role": "user", "content": input_data})
+ elif isinstance(input_data, list):
+ original_items = input_data
+ converted, preserved_indices = responses_items_to_messages(input_data)
+ messages.extend(converted)
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("content-length", None)
+ # Strip accept-encoding so httpx negotiates its own encoding.
+ # Cloudflare Workers forward "br, zstd" which OpenAI may honor;
+ # if httpx lacks brotli support the response body is undecipherable → 502.
+ headers.pop("accept-encoding", None)
+ tags = self._extract_tags(headers)
+
+ # Memory: Get user ID when memory is enabled
+ memory_user_id: str | None = None
+ if self.memory_handler:
+ memory_user_id = headers.get(
+ "x-headroom-user-id",
+ os.environ.get("USER", os.environ.get("USERNAME", "default")),
+ )
+
+ # Rate limiting
+ if self.rate_limiter:
+ rate_key = headers.get("authorization", "default")[:20]
+ allowed, wait_seconds = await self.rate_limiter.check_request(rate_key)
+ if not allowed:
+ await self.metrics.record_rate_limited(provider="openai")
+ raise HTTPException(
+ status_code=429,
+ detail=f"Rate limited. Retry after {wait_seconds:.1f}s",
+ )
+
+ # Token counting on converted messages
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ # Optimize: convert items → compress → convert back
+ tokens_saved = 0
+ transforms_applied: list[str] = []
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+
+ _bypass = (
+ request.headers.get("x-headroom-bypass", "").lower() == "true"
+ or request.headers.get("x-headroom-mode", "").lower() == "passthrough"
+ )
+ _should_compress = (
+ self.config.optimize
+ and original_items is not None
+ and not previous_response_id
+ and not _bypass
+ and len(messages) > 1
+ )
+ _license_ok = self.usage_reporter.should_compress if self.usage_reporter else True
+
+ if _should_compress and _license_ok:
+ try:
+ context_limit = self.openai_provider.get_context_limit(model)
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+ if result.messages != messages:
+ optimized_messages = result.messages
+ transforms_applied = result.transforms_applied
+ original_tokens = result.tokens_before
+ optimized_tokens = result.tokens_after
+ except Exception as e:
+ logger.warning(f"[{request_id}] Responses API optimization failed: {e}")
+
+ # Guard: if "optimization" inflated tokens, revert to originals
+ if optimized_tokens > original_tokens:
+ logger.warning(
+ f"[{request_id}] Optimization inflated tokens "
+ f"({original_tokens} -> {optimized_tokens}), reverting to original messages"
+ )
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+ transforms_applied = []
+
+ tokens_saved = original_tokens - optimized_tokens
+ optimization_latency = (time.time() - start_time) * 1000
+
+ # Convert compressed messages back to Responses API items
+ if optimized_messages is not messages and original_items is not None:
+ opt_msgs = optimized_messages
+ # Strip system message (instructions) — it's separate in Responses API
+ if instructions and opt_msgs and opt_msgs[0].get("role") == "system":
+ body["instructions"] = opt_msgs[0]["content"]
+ opt_msgs = opt_msgs[1:]
+
+ body["input"] = messages_to_responses_items(opt_msgs, original_items, preserved_indices)
+
+ # Memory: inject context and tools for Responses API requests
+ if self.memory_handler and memory_user_id:
+ try:
+ # Inject memory context into instructions (Responses API system prompt)
+ if self.memory_handler.config.inject_context:
+ try:
+ memory_context = await asyncio.wait_for(
+ self.memory_handler.search_and_format_context(
+ memory_user_id, optimized_messages
+ ),
+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ memory_context = None
+ logger.info(
+ f"[{request_id}] Memory context lookup exceeded "
+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"
+ )
+ if memory_context:
+ existing_instructions = body.get("instructions") or ""
+ if existing_instructions:
+ body["instructions"] = f"{existing_instructions}\n\n{memory_context}"
+ else:
+ body["instructions"] = memory_context
+ logger.info(
+ f"[{request_id}] Memory: Injected {len(memory_context)} chars "
+ f"of context into instructions for user {memory_user_id}"
+ )
+
+ # Inject memory tools (Responses API format)
+ if self.memory_handler.config.inject_tools:
+ resp_tools = body.get("tools") or []
+ resp_tools, mem_tools_injected = self.memory_handler.inject_tools(
+ resp_tools, "openai"
+ )
+ if mem_tools_injected:
+ # Convert Chat Completions format to Responses API format
+ converted_tools = []
+ for t in resp_tools:
+ if t.get("type") == "function" and "function" in t:
+ fn = t["function"]
+ converted_tools.append(
+ {
+ "type": "function",
+ "name": fn.get("name"),
+ "description": fn.get("description", ""),
+ "parameters": fn.get("parameters", {}),
+ }
+ )
+ else:
+ converted_tools.append(t)
+ body["tools"] = converted_tools
+ logger.info(
+ f"[{request_id}] Memory: Injected memory tools (openai/responses)"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] Memory injection failed (responses): {e}")
+
+ # /v1/responses is OpenAI-specific (Codex) — always routes direct.
+ # LiteLLM/AnyLLM backends use /v1/chat/completions or /v1/messages.
+ if self.anthropic_backend is not None:
+ logger.debug(
+ f"[{request_id}] /v1/responses always routes to OpenAI direct "
+ f"(backend '{self.anthropic_backend.name}' not used for Responses API)"
+ )
+
+ headers, is_chatgpt_auth = _resolve_codex_routing_headers(headers)
+
+ # Route to correct endpoint based on auth mode.
+ # ChatGPT session auth (codex login) uses chatgpt.com, not api.openai.com.
+ if is_chatgpt_auth:
+ url = "https://chatgpt.com/backend-api/codex/responses"
+ else:
+ url = f"{self.OPENAI_API_URL}/v1/responses"
+
+ try:
+ if stream:
+ # Streaming for Responses API uses semantic events
+ return await self._stream_response(
+ url,
+ headers,
+ body,
+ "openai",
+ model,
+ request_id,
+ original_tokens,
+ optimized_tokens,
+ tokens_saved,
+ transforms_applied,
+ tags,
+ optimization_latency,
+ memory_user_id=memory_user_id,
+ )
+ else:
+ headers = await apply_copilot_api_auth(headers, url=url)
+ response = await self._retry_request("POST", url, headers, body)
+ total_latency = (time.time() - start_time) * 1000
+
+ total_input_tokens = original_tokens # fallback
+ output_tokens = 0
+ try:
+ resp_json = response.json()
+ usage = resp_json.get("usage", {})
+ total_input_tokens = usage.get("input_tokens", original_tokens)
+ output_tokens = usage.get("output_tokens", 0)
+ except (KeyError, TypeError, AttributeError) as e:
+ logger.debug(
+ f"[{request_id}] Failed to extract cached tokens from OpenAI passthrough response: {e}"
+ )
+
+ # Memory: handle memory tool calls in Responses API response
+ if (
+ self.memory_handler
+ and memory_user_id
+ and resp_json
+ and response.status_code == 200
+ and self.memory_handler.has_memory_tool_calls(resp_json, "openai")
+ ):
+ try:
+ # Extract function_call items from output
+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
+
+ output_items = resp_json.get("output", [])
+ memory_fc_items = [
+ item
+ for item in output_items
+ if isinstance(item, dict)
+ and item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ]
+
+ # Execute memory tool calls
+ tool_outputs: list[dict[str, Any]] = []
+ for fc in memory_fc_items:
+ call_id = fc.get("call_id", fc.get("id", ""))
+ name = fc.get("name", "")
+ args_str = fc.get("arguments", "{}")
+ try:
+ args = json.loads(args_str)
+ except json.JSONDecodeError:
+ args = {}
+
+ await self.memory_handler._ensure_initialized()
+ if self.memory_handler._backend:
+ result = await self.memory_handler._execute_memory_tool(
+ name, args, memory_user_id, "openai"
+ )
+ else:
+ result = json.dumps({"error": "Memory backend not initialized"})
+
+ tool_outputs.append(
+ {
+ "type": "function_call_output",
+ "call_id": call_id,
+ "output": result,
+ }
+ )
+
+ if tool_outputs:
+ # Make continuation request with tool results
+ response_id = resp_json.get("id")
+ continuation_body = {
+ "model": model,
+ "input": tool_outputs,
+ }
+ if response_id:
+ continuation_body["previous_response_id"] = response_id
+ existing_tools = body.get("tools")
+ if existing_tools:
+ continuation_body["tools"] = existing_tools
+
+ cont_response = await self._retry_request(
+ "POST", url, headers, continuation_body
+ )
+ resp_json = cont_response.json()
+ response = cont_response
+ logger.info(
+ f"[{request_id}] Memory: Handled {len(tool_outputs)} "
+ f"tool call(s) with continuation for user {memory_user_id} (responses)"
+ )
+ except Exception as e:
+ logger.warning(
+ f"[{request_id}] Memory tool handling failed (responses): {e}"
+ )
+
+ if self.cost_tracker:
+ self.cost_tracker.record_tokens(model, tokens_saved, total_input_tokens)
+
+ await self.metrics.record_request(
+ provider="openai",
+ model=model,
+ input_tokens=total_input_tokens,
+ output_tokens=output_tokens,
+ tokens_saved=tokens_saved,
+ latency_ms=total_latency,
+ overhead_ms=optimization_latency,
+ )
+
+ logger.info(f"[{request_id}] /v1/responses {model}: {total_input_tokens:,} tokens")
+
+ # Capture Codex rate-limit window data from response headers
+ from headroom.subscription.codex_rate_limits import (
+ get_codex_rate_limit_state,
+ )
+
+ get_codex_rate_limit_state().update_from_headers(dict(response.headers))
+
+ # Remove compression headers
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None)
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+ except Exception as e:
+ await self.metrics.record_failed(provider="openai")
+ logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "message": "An error occurred while processing your request. Please try again.",
+ "type": "server_error",
+ "code": "proxy_error",
+ }
+ },
+ )
+
+ async def handle_openai_responses_ws(self, websocket: WebSocket) -> None:
+ """WebSocket proxy for /v1/responses (Codex gpt-5.4+).
+
+ Newer Codex versions use WebSocket instead of HTTP POST for the
+ Responses API. This handler:
+ 1. Accepts the client WebSocket
+ 2. Receives the first message (``response.create`` request)
+ 3. Compresses the ``input`` array using the existing pipeline
+ 4. Opens an upstream WebSocket to OpenAI
+ 5. Sends the compressed request upstream
+ 6. Relays all subsequent messages bidirectionally
+ """
+ from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
+ from headroom.tokenizers import get_tokenizer
+ from headroom.utils import extract_user_query
+
+ try:
+ import websockets
+ except ImportError:
+ await websocket.accept()
+ await websocket.close(
+ code=1011,
+ reason="websockets package not installed. pip install websockets",
+ )
+ return
+
+ request_id = await self._next_request_id()
+ session_id = uuid.uuid4().hex
+
+ # Stage-timer — captures per-stage durations for the structured
+ # log emitted on session close. Unit 2 instrumentation.
+ stage_timer = StageTimer()
+ session_started_at = time.perf_counter()
+
+ # Unit 3: initialize registry variables *before* accept so the
+ # outermost ``finally`` can rely on them existing even if
+ # registration itself fails for some reason.
+ ws_sessions: WebSocketSessionRegistry | None = getattr(self, "ws_sessions", None)
+ session_handle: WSSessionHandle | None = None
+ termination_cause: TerminationCause = "unknown"
+
+ # Forward client headers to upstream, adding required OpenAI-Beta header
+ ws_headers = dict(websocket.headers)
+
+ # Extract subprotocol from client — this is an application-level negotiation
+ # that MUST be forwarded end-to-end (unlike sec-websocket-key which is per-connection).
+ # Codex and OpenAI negotiate a subprotocol; stripping it causes OpenAI to return 500.
+ client_subprotocols: list[str] = []
+ raw_protocol = ws_headers.get("sec-websocket-protocol", "")
+ if raw_protocol:
+ client_subprotocols = [p.strip() for p in raw_protocol.split(",") if p.strip()]
+
+ # Accept client connection with the requested subprotocol
+ async with stage_timer.measure("accept"):
+ if client_subprotocols:
+ await websocket.accept(subprotocol=client_subprotocols[0])
+ else:
+ await websocket.accept()
+
+ # --- Unit 3: register the session as soon as accept succeeds ---
+ client_addr: str | None = None
+ client_info = getattr(websocket, "client", None)
+ if client_info is not None:
+ host = getattr(client_info, "host", None)
+ port = getattr(client_info, "port", None)
+ if host is not None and port is not None:
+ client_addr = f"{host}:{port}"
+ elif host is not None:
+ client_addr = str(host)
+ if ws_sessions is not None:
+ session_handle = WSSessionHandle(
+ session_id=session_id,
+ request_id=request_id,
+ client_addr=client_addr,
+ upstream_url=None, # set below once upstream_url is computed
+ )
+ ws_sessions.register(session_handle)
+ metrics = getattr(self, "metrics", None)
+ if metrics is not None and hasattr(metrics, "inc_active_ws_sessions"):
+ try:
+ metrics.inc_active_ws_sessions()
+ except Exception: # pragma: no cover - defensive
+ pass
+
+ # Forward all client headers except hop-by-hop / per-connection headers.
+ # These are WebSocket handshake mechanics that the `websockets` library
+ # generates fresh for the upstream connection — forwarding them would conflict.
+ # Everything else (auth, org, beta, user-agent, custom headers) is forwarded as-is.
+ _skip_headers = frozenset(
+ {
+ "host", # must match upstream, not local proxy
+ "connection", # hop-by-hop
+ "upgrade", # hop-by-hop
+ "sec-websocket-key", # per-connection cryptographic nonce
+ "sec-websocket-version", # protocol version (websockets lib sets this)
+ "sec-websocket-extensions", # per-connection negotiation
+ "sec-websocket-accept", # server-side only
+ "sec-websocket-protocol", # handled via subprotocols param below
+ "content-length", # hop-by-hop
+ "transfer-encoding", # hop-by-hop
+ }
+ )
+ upstream_headers: dict[str, str] = {}
+ for k, v in ws_headers.items():
+ if k.lower() not in _skip_headers:
+ upstream_headers[k] = v
+
+ upstream_headers, is_chatgpt_auth = _resolve_codex_routing_headers(upstream_headers)
+ _lower_headers = {k.lower(): v for k, v in upstream_headers.items()}
+
+ # Build upstream WebSocket URL based on auth mode
+ if is_chatgpt_auth:
+ # ChatGPT session auth → route to chatgpt.com backend
+ upstream_url = "wss://chatgpt.com/backend-api/codex/responses"
+ logger.debug(
+ f"[{request_id}] WS: ChatGPT session auth detected, routing to chatgpt.com"
+ )
+ else:
+ # API key auth → route to configured OpenAI API URL
+ base = self.OPENAI_API_URL
+ ws_base = base.replace("https://", "wss://").replace("http://", "ws://")
+ upstream_url = f"{ws_base}/v1/responses"
+
+ # Unit 3: attach the resolved upstream URL to the session handle.
+ if session_handle is not None:
+ session_handle.upstream_url = upstream_url
+
+ # Ensure Authorization header is present — fall back to OPENAI_API_KEY env var.
+ # Safety net for clients that don't forward auth headers via WebSocket upgrade.
+ if "authorization" not in _lower_headers:
+ api_key = os.environ.get("OPENAI_API_KEY")
+ if api_key:
+ upstream_headers["Authorization"] = f"Bearer {api_key}"
+ logger.debug(f"[{request_id}] WS: injected Authorization from OPENAI_API_KEY env")
+ else:
+ logger.warning(
+ f"[{request_id}] WS: no Authorization header from client and "
+ f"OPENAI_API_KEY not set — upstream will likely reject"
+ )
+
+ upstream_headers = await apply_copilot_api_auth(upstream_headers, url=upstream_url)
+
+ # Ensure the required beta header is present — OpenAI returns 500 without it.
+ # Codex sends `responses_websockets=2026-02-06`; only inject if missing entirely.
+ if "openai-beta" not in _lower_headers:
+ upstream_headers["OpenAI-Beta"] = "responses_websockets=2026-02-06"
+
+ logger.debug(
+ f"[{request_id}] WS upstream headers: "
+ f"{[k for k in upstream_headers if k.lower() != 'authorization']}, "
+ f"subprotocols={client_subprotocols}"
+ )
+
+ try:
+ # Receive the first message from client (the response.create request).
+ # Bound the wait with WS_FIRST_FRAME_TIMEOUT_SECONDS so a zombie
+ # client that opens the WS but never sends a frame cannot hold a
+ # session slot indefinitely. The StageTimer measurement still
+ # captures the elapsed time up to the timeout so operators can
+ # see the slow-client pattern in the stage-timings log.
+ try:
+ async with stage_timer.measure("first_client_frame"):
+ first_msg_raw = await asyncio.wait_for(
+ websocket.receive_text(),
+ timeout=WS_FIRST_FRAME_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ logger.info(
+ f"[{request_id}] WS first-frame timeout after "
+ f"{WS_FIRST_FRAME_TIMEOUT_SECONDS:.0f}s; closing session "
+ f"{session_id} (no client data)"
+ )
+ termination_cause = "client_timeout"
+ with contextlib.suppress(Exception):
+ # 1001 (going away): server is cleanly terminating a slow
+ # client, not an internal error.
+ await websocket.close(code=1001, reason="first-frame timeout")
+ # Exit the outer try so the session-lifecycle ``finally`` runs
+ # deregister / metrics / stage-timings emission as usual.
+ return
+
+ # --- Optional: compress the input in the first message ---
+ body: dict[str, Any] = {}
+ try:
+ body = json.loads(first_msg_raw)
+ tokens_saved = 0
+ ws_request_body = body.get("response", body)
+ input_data = (
+ ws_request_body.get("input") if isinstance(ws_request_body, dict) else None
+ )
+
+ should_compress = (
+ self.config.optimize
+ and isinstance(input_data, list)
+ and len(input_data) > 1
+ and not (
+ ws_request_body.get("previous_response_id")
+ if isinstance(ws_request_body, dict)
+ else None
+ )
+ )
+ if should_compress:
+ try:
+ from headroom.proxy.responses_converter import (
+ messages_to_responses_items,
+ responses_items_to_messages,
+ )
+
+ model = ws_request_body.get("model", "gpt-4o")
+ converted, preserved = responses_items_to_messages(input_data)
+
+ messages: list[dict[str, Any]] = []
+ instructions = ws_request_body.get("instructions")
+ if instructions:
+ messages.append({"role": "system", "content": instructions})
+ messages.extend(converted)
+
+ tokenizer = get_tokenizer(model)
+ original_tokens = tokenizer.count_messages(messages)
+
+ context_limit = self.openai_provider.get_context_limit(model)
+ async with stage_timer.measure("compression"):
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(messages),
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+
+ if result.messages != messages:
+ opt = result.messages
+ if instructions and opt and opt[0].get("role") == "system":
+ ws_request_body["instructions"] = opt[0]["content"]
+ opt = opt[1:]
+ if result.tokens_after <= original_tokens:
+ ws_request_body["input"] = messages_to_responses_items(
+ opt, input_data, preserved
+ )
+ else:
+ logger.warning(
+ f"[{request_id}] WS optimization inflated tokens "
+ f"({original_tokens} -> {result.tokens_after}), reverting"
+ )
+ tokens_saved = max(0, original_tokens - result.tokens_after)
+ if "response" in body and isinstance(body["response"], dict):
+ body["response"] = ws_request_body
+ else:
+ body = ws_request_body
+ first_msg_raw = json.dumps(body)
+ logger.info(
+ f"[{request_id}] WS /v1/responses compressed: "
+ f"saved {tokens_saved} tokens"
+ )
+ except Exception as e:
+ logger.warning(f"[{request_id}] WS compression failed: {e}")
+
+ except json.JSONDecodeError:
+ # Not JSON — pass through as-is
+ tokens_saved = 0
+
+ # --- Memory: inject context, tools, and instructions ---
+ memory_user_id: str | None = None
+ if self.memory_handler and body:
+ memory_user_id = ws_headers.get(
+ "x-headroom-user-id",
+ os.environ.get("USER", os.environ.get("USERNAME", "default")),
+ )
+ try:
+ # Unwrap response.create envelope to access the response body
+ ws_response_body = body.get("response", body)
+
+ # Debug: log what Codex sends so we can see the full tool list
+ existing_tool_names = [
+ t.get("name") or t.get("function", {}).get("name", "?")
+ for t in (ws_response_body.get("tools") or [])
+ ]
+ instr_preview = (ws_response_body.get("instructions") or "")[:200]
+ logger.info(
+ f"[{request_id}] WS Memory: Codex tools={existing_tool_names}, "
+ f"instructions_len={len(ws_response_body.get('instructions') or '')}, "
+ f"instructions_preview={instr_preview!r}"
+ )
+
+ # Inject memory context into instructions
+ if self.memory_handler.config.inject_context:
+ ws_input = ws_response_body.get("input", "")
+ ws_instructions = ws_response_body.get("instructions")
+ ws_msgs: list[dict[str, Any]] = []
+ if ws_instructions:
+ ws_msgs.append({"role": "system", "content": ws_instructions})
+ if isinstance(ws_input, str) and ws_input:
+ ws_msgs.append({"role": "user", "content": ws_input})
+ elif isinstance(ws_input, list):
+ from headroom.proxy.responses_converter import (
+ responses_items_to_messages,
+ )
+
+ converted_msgs, _ = responses_items_to_messages(ws_input)
+ ws_msgs.extend(converted_msgs)
+
+ try:
+ async with stage_timer.measure("memory_context"):
+ memory_context = await asyncio.wait_for(
+ self.memory_handler.search_and_format_context(
+ memory_user_id, ws_msgs
+ ),
+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ memory_context = None
+ logger.info(
+ f"[{request_id}] WS Memory: Context lookup exceeded "
+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "
+ f"continuing without it"
+ )
+ if memory_context:
+ existing = ws_response_body.get("instructions") or ""
+ if existing:
+ ws_response_body["instructions"] = f"{existing}\n\n{memory_context}"
+ else:
+ ws_response_body["instructions"] = memory_context
+ logger.info(
+ f"[{request_id}] WS Memory: Injected {len(memory_context)} chars "
+ f"of context into instructions"
+ )
+
+ # Inject memory tools (Responses API format)
+ if self.memory_handler.config.inject_tools:
+ ws_tools = ws_response_body.get("tools") or []
+ ws_tools, mem_injected = self.memory_handler.inject_tools(
+ ws_tools, "openai"
+ )
+ if mem_injected:
+ converted_tools = []
+ for t in ws_tools:
+ if t.get("type") == "function" and "function" in t:
+ fn = t["function"]
+ converted_tools.append(
+ {
+ "type": "function",
+ "name": fn.get("name"),
+ "description": fn.get("description", ""),
+ "parameters": fn.get("parameters", {}),
+ }
+ )
+ else:
+ converted_tools.append(t)
+ ws_response_body["tools"] = converted_tools
+
+ # Add memory instruction so the model uses
+ # memory tools as persistent cross-session knowledge.
+ mem_instruction = (
+ "\n\n## Memory\n"
+ "You have persistent memory via memory_search and "
+ "memory_save tools. Memory stores knowledge across "
+ "sessions — user info, project details, org context, "
+ "decisions, architecture, conventions, anything worth "
+ "remembering.\n\n"
+ "- ALWAYS call memory_search BEFORE searching files "
+ "when the user asks a question that could be answered "
+ "from prior knowledge.\n"
+ "- Call memory_save to store important facts, decisions, "
+ "or context that would be useful in future sessions.\n"
+ "- Memory is your first source of truth for anything "
+ "not visible in the current conversation."
+ )
+ existing_instr = ws_response_body.get("instructions") or ""
+ ws_response_body["instructions"] = existing_instr + mem_instruction
+ logger.info(
+ f"[{request_id}] WS Memory: Injected memory tools + instruction"
+ )
+
+ # Write back into envelope if it was wrapped
+ if "response" in body and isinstance(body["response"], dict):
+ body["response"] = ws_response_body
+ else:
+ body = ws_response_body
+
+ first_msg_raw = json.dumps(body)
+ except Exception as e:
+ logger.warning(f"[{request_id}] WS Memory injection failed: {e}")
+
+ # --- Connect to upstream OpenAI WebSocket ---
+ logger.info(f"[{request_id}] WS /v1/responses connecting to {upstream_url}")
+
+ # Use ssl=True to let the websockets library handle SSL natively.
+ # Manual ssl.create_default_context() + certifi doesn't load the
+ # Windows system cert store, causing HTTP 500 on wss:// connections.
+ use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
+
+ ws_connected = False
+ ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ ws_last_err: Exception | None = None
+ _upstream_connect_started = time.perf_counter()
+ _upstream_connect_recorded = False
+ _upstream_first_event_started: float | None = None
+
+ for ws_attempt in range(ws_connect_attempts):
+ try:
+ async with websockets.connect(
+ upstream_url,
+ additional_headers=upstream_headers,
+ subprotocols=(
+ [websockets.Subprotocol(p) for p in client_subprotocols]
+ if client_subprotocols and hasattr(websockets, "Subprotocol")
+ else client_subprotocols or None
+ ),
+ ssl=use_ssl,
+ open_timeout=max(30, self.config.connect_timeout_seconds * 3),
+ close_timeout=10,
+ ping_interval=20,
+ ping_timeout=20,
+ ) as upstream:
+ ws_connected = True
+ if not _upstream_connect_recorded:
+ stage_timer.record(
+ "upstream_connect",
+ (time.perf_counter() - _upstream_connect_started) * 1000.0,
+ )
+ _upstream_connect_recorded = True
+ _upstream_first_event_started = time.perf_counter()
+ await upstream.send(first_msg_raw)
+
+ # Unit 3: flag the upstream side flips on seeing
+ # ``response.completed`` so the outer cause
+ # classifier can prefer it over the raw
+ # "upstream iterator ended" default.
+ response_completed_seen = False
+ # Captures the first exception surfaced by the
+ # inner relay ``except`` blocks so the outer
+ # classifier can still tell ``upstream_error``
+ # from ``upstream_disconnect`` / ``response_completed``
+ # even though the halves swallow and log.
+ upstream_relay_error: BaseException | None = None
+ client_relay_error: BaseException | None = None
+
+ async def _client_to_upstream() -> None:
+ nonlocal client_relay_error
+ try:
+ while True:
+ msg = await websocket.receive_text()
+ await upstream.send(msg)
+ except asyncio.CancelledError:
+ # Explicit cancel from the outer
+ # orchestrator — re-raise so
+ # ``t.cancelled()`` and ``t.exception()``
+ # behave correctly in the caller.
+ raise
+ except Exception as relay_err:
+ # Surface real errors to the classifier
+ # without re-raising (existing fork
+ # behavior: log and return so the
+ # partner task can be cancelled
+ # deterministically).
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ client_relay_error = relay_err
+ logger.debug(
+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"
+ )
+ with contextlib.suppress(Exception):
+ await upstream.close()
+
+ async def _upstream_to_client() -> None:
+ """Relay upstream→client with transparent memory tool handling.
+
+ Uses a buffer-then-decide approach:
+ 1. Buffer events until first output item arrives
+ 2. If first output is a memory tool → suppress entire response,
+ execute tools silently, send continuation upstream
+ 3. If first output is non-memory → flush buffer, stream normally
+ 4. Continuation response events are relayed to Codex seamlessly
+
+ This prevents orphaned response.created events from confusing Codex.
+ """
+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
+
+ # Unit 3: surface response.completed observation
+ # to the outer scope so the termination-cause
+ # classifier can prefer ``response_completed``
+ # over ``upstream_disconnect``.
+ nonlocal response_completed_seen
+ nonlocal upstream_relay_error
+
+ memory_enabled = bool(self.memory_handler and memory_user_id)
+
+ # Per-response state (reset after each response.completed)
+ event_buffer: list[str] = []
+ decided = False
+ suppress_response = False
+ pending_fcs: list[dict[str, Any]] = []
+ resp_id: str | None = None
+
+ def _reset() -> None:
+ nonlocal decided, suppress_response, resp_id
+ event_buffer.clear()
+ decided = False
+ suppress_response = False
+ pending_fcs.clear()
+ resp_id = None
+
+ # The retry-loop variable is safe to close over here:
+ # ``_upstream_to_client`` is defined and awaited within
+ # a single iteration and never escapes.
+ _first_event_started_at = _upstream_first_event_started # noqa: B023
+
+ try:
+ async for msg in upstream:
+ if (
+ _first_event_started_at is not None
+ and "upstream_first_event" not in stage_timer
+ ):
+ stage_timer.record(
+ "upstream_first_event",
+ (time.perf_counter() - _first_event_started_at)
+ * 1000.0,
+ )
+ if isinstance(msg, bytes):
+ await websocket.send_bytes(msg)
+ continue
+ msg_str = msg if isinstance(msg, str) else str(msg)
+
+ if not memory_enabled:
+ await websocket.send_text(msg_str)
+ continue
+
+ # Parse event
+ try:
+ event = json.loads(msg_str)
+ except (json.JSONDecodeError, TypeError):
+ await websocket.send_text(msg_str)
+ continue
+
+ event_type = event.get("type", "")
+
+ # --- Phase 1: Buffer until first output item ---
+ if not decided:
+ event_buffer.append(msg_str)
+
+ if event_type == "response.output_item.added":
+ item = event.get("item", {})
+ if (
+ item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ):
+ # Memory tool first → suppress entire response
+ suppress_response = True
+ decided = True
+ event_buffer.clear()
+ logger.info(
+ f"[{request_id}] WS Memory: Detected "
+ f"{item.get('name')} — suppressing response"
+ )
+ else:
+ # Non-memory first → flush buffer, pass through
+ decided = True
+ for buf in event_buffer:
+ await websocket.send_text(buf)
+ event_buffer.clear()
+
+ elif event_type == "response.completed":
+ # No output items at all — flush
+ decided = True
+ for buf in event_buffer:
+ await websocket.send_text(buf)
+ event_buffer.clear()
+ _reset()
+ response_completed_seen = True
+
+ continue
+
+ # --- Phase 2a: Suppress mode (memory response) ---
+ if suppress_response:
+ if event_type == "response.output_item.done":
+ item = event.get("item", {})
+ if (
+ item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ):
+ pending_fcs.append(item)
+
+ elif event_type == "response.completed":
+ response_completed_seen = True
+ resp = event.get("response", {})
+ resp_id = resp.get("id")
+
+ if pending_fcs:
+ logger.info(
+ f"[{request_id}] WS Memory: Executing "
+ f"{len(pending_fcs)} tool(s) transparently"
+ )
+
+ # Execute memory tool calls
+ tool_outputs: list[dict[str, Any]] = []
+ for fc in pending_fcs:
+ call_id = fc.get("call_id", fc.get("id", ""))
+ fc_name = fc.get("name", "")
+ args_str = fc.get("arguments", "{}")
+ try:
+ fc_args = json.loads(args_str)
+ except json.JSONDecodeError:
+ fc_args = {}
+
+ await self.memory_handler._ensure_initialized()
+ if self.memory_handler._backend:
+ result = await self.memory_handler._execute_memory_tool(
+ fc_name,
+ fc_args,
+ memory_user_id,
+ "openai",
+ )
+ else:
+ result = json.dumps(
+ {"error": "backend not ready"}
+ )
+
+ tool_outputs.append(
+ {
+ "type": "function_call_output",
+ "call_id": call_id,
+ "output": result,
+ }
+ )
+ logger.info(
+ f"[{request_id}] WS Memory: Executed "
+ f"{fc_name} for user {memory_user_id}"
+ )
+
+ # Send continuation upstream
+ cont: dict[str, Any] = {
+ "type": "response.create",
+ "response": {"input": tool_outputs},
+ }
+ if resp_id:
+ cont["response"]["previous_response_id"] = (
+ resp_id
+ )
+ await upstream.send(json.dumps(cont))
+ logger.info(
+ f"[{request_id}] WS Memory: Sent continuation "
+ f"with {len(tool_outputs)} result(s)"
+ )
+
+ _reset()
+ # All events suppressed in this mode
+ continue
+
+ # --- Phase 2b: Pass-through mode ---
+ await websocket.send_text(msg_str)
+
+ except asyncio.CancelledError:
+ raise
+ except Exception as relay_err:
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ # Capture for the outer classifier
+ # so ``upstream_error`` can be
+ # distinguished from a clean
+ # upstream disconnect.
+ upstream_relay_error = relay_err
+ logger.debug(
+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"
+ )
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close()
+
+ # --- Unit 3: deterministic relay-task cancellation ---
+ # Spawn each half as a named task so we can:
+ # (a) attach them to the session registry for
+ # ``/debug/ws-sessions``,
+ # (b) cancel the survivor explicitly when the
+ # first one exits, and
+ # (c) classify the termination cause for the
+ # duration histogram.
+ client_task = asyncio.create_task(
+ _client_to_upstream(),
+ name=f"codex-ws-c2u-{session_id}",
+ )
+ upstream_task = asyncio.create_task(
+ _upstream_to_client(),
+ name=f"codex-ws-u2c-{session_id}",
+ )
+ relay_tasks = [client_task, upstream_task]
+ if ws_sessions is not None:
+ ws_sessions.attach_tasks(session_id, relay_tasks)
+ metrics_for_tasks = getattr(self, "metrics", None)
+ if metrics_for_tasks is not None and hasattr(
+ metrics_for_tasks, "inc_active_relay_tasks"
+ ):
+ try:
+ metrics_for_tasks.inc_active_relay_tasks(len(relay_tasks))
+ except Exception: # pragma: no cover - defensive
+ pass
+
+ try:
+ done, pending = await asyncio.wait(
+ {client_task, upstream_task},
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ # Cancel the survivor so we don't leak the
+ # partner task. Suppress the CancelledError
+ # we just raised ourselves — any *other*
+ # exception from the cancelled task is
+ # already logged inside its own try/except.
+ for t in pending:
+ t.cancel()
+ if pending:
+ with contextlib.suppress(asyncio.CancelledError):
+ await asyncio.gather(*pending, return_exceptions=True)
+
+ # Classify termination cause from whichever
+ # task completed first. ``CancelledError``
+ # can show up on the "done" side if the
+ # handler itself was cancelled from outside
+ # (e.g. server shutdown).
+ for t in done:
+ exc = None
+ # Cancelled tasks raise CancelledError from
+ # .exception(); surface it explicitly so the
+ # downstream ``isinstance(exc, CancelledError)``
+ # branches actually run. For any other
+ # unexpected state (``InvalidStateError`` if
+ # the task somehow isn't done — shouldn't
+ # happen post-gather but defensive), we
+ # suppress and leave ``exc`` as ``None``.
+ if t.cancelled():
+ exc = asyncio.CancelledError()
+ else:
+ with contextlib.suppress(asyncio.InvalidStateError):
+ exc = t.exception()
+ task_name = t.get_name() or ""
+ if t is client_task:
+ if client_relay_error is not None:
+ termination_cause = "client_error"
+ elif exc is None:
+ termination_cause = "client_disconnect"
+ elif isinstance(exc, asyncio.CancelledError):
+ termination_cause = "client_disconnect"
+ else:
+ # Distinguish legitimate client
+ # disconnect exceptions from
+ # real errors: WebSocketDisconnect
+ # is a normal client exit.
+ if "WebSocketDisconnect" in type(exc).__name__:
+ termination_cause = "client_disconnect"
+ else:
+ termination_cause = "client_error"
+ elif t is upstream_task:
+ if upstream_relay_error is not None:
+ termination_cause = "upstream_error"
+ logger.debug(
+ f"[{request_id}] WS relay {task_name} "
+ f"raised: {upstream_relay_error!r}"
+ )
+ elif exc is None:
+ termination_cause = (
+ "response_completed"
+ if response_completed_seen
+ else "upstream_disconnect"
+ )
+ elif isinstance(exc, asyncio.CancelledError):
+ termination_cause = "upstream_disconnect"
+ else:
+ termination_cause = "upstream_error"
+ logger.debug(
+ f"[{request_id}] WS relay {task_name} raised: {exc!r}"
+ )
+ finally:
+ # In case anything above raised before the
+ # cancel-and-await loop ran.
+ for t in relay_tasks:
+ if not t.done():
+ t.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await asyncio.gather(*relay_tasks, return_exceptions=True)
+
+ logger.info(
+ f"[{request_id}] WS /v1/responses completed "
+ f"(tokens_saved={tokens_saved}, cause={termination_cause})"
+ )
+ break
+ except Exception as ws_err:
+ if ws_connected:
+ # WS was established but broke mid-stream — re-raise
+ raise
+
+ ws_last_err = ws_err
+ if ws_attempt >= ws_connect_attempts - 1:
+ break
+
+ delay_with_jitter = jitter_delay_ms(
+ self.config.retry_base_delay_ms,
+ self.config.retry_max_delay_ms,
+ ws_attempt,
+ )
+ logger.warning(
+ f"[{request_id}] WS upstream connect failed "
+ f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ if not ws_connected:
+ # WS upgrade failed (HTTP 500 from OpenAI is common).
+ # Fall back to HTTP POST streaming and relay SSE events
+ # back over the client WebSocket transparently.
+ ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")
+ _ws_detail = str(ws_err)
+ if hasattr(ws_err, "response"):
+ resp_body = getattr(getattr(ws_err, "response", None), "body", b"")
+ if resp_body:
+ _ws_detail += f" | {resp_body[:300].decode('utf-8', errors='replace')}"
+ logger.warning(
+ f"[{request_id}] WS upstream failed ({_ws_detail}), "
+ f"falling back to HTTP POST streaming"
+ )
+ await self._ws_http_fallback(
+ websocket, body, first_msg_raw, upstream_headers, request_id
+ )
+
+ # Record metrics
+ if tokens_saved > 0:
+ model_name = body.get("model", "unknown") if isinstance(body, dict) else "unknown"
+ await self.metrics.record_request(
+ provider="openai",
+ model=model_name,
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=tokens_saved,
+ latency_ms=0,
+ )
+
+ except Exception as e:
+ if "WebSocketDisconnect" in type(e).__name__:
+ # Unit 3: client dropped the socket before or during
+ # relay. The registry classifier may already have set
+ # ``client_disconnect`` via the relay task exit path;
+ # preserve that, otherwise set it here.
+ if termination_cause == "unknown":
+ termination_cause = "client_disconnect"
+ else:
+ # Extract response body from websockets InvalidStatus for better debugging
+ error_detail = str(e)
+ if hasattr(e, "response"):
+ try:
+ resp = e.response
+ body_bytes = getattr(resp, "body", None) or b""
+ if body_bytes:
+ error_detail += (
+ f" | body: {body_bytes[:500].decode('utf-8', errors='replace')}"
+ )
+ except Exception:
+ pass
+ logger.error(f"[{request_id}] WS proxy error: {error_detail}")
+ if termination_cause == "unknown":
+ termination_cause = "client_error"
+ with contextlib.suppress(Exception):
+ await websocket.close(code=1011, reason=str(e)[:120])
+ finally:
+ # Unit 2: emit structured per-session stage timings.
+ stage_timer.record(
+ "total_session",
+ (time.perf_counter() - session_started_at) * 1000.0,
+ )
+ # Unit 3: deregister the session before (or independently
+ # of) the stage-timings log so a failure there cannot leak
+ # the registry entry. ``deregister`` is idempotent, so a
+ # session that never registered is a no-op.
+ if ws_sessions is not None and session_handle is not None:
+ # Use deregister_and_count so the handle pop and the
+ # relay-task count are read atomically inside the
+ # registry. Capturing ``len(session_handle.relay_tasks)``
+ # separately before ``deregister`` would risk drift if
+ # the registry's bookkeeping ever changes.
+ _deregistered, released_tasks = ws_sessions.deregister_and_count(
+ session_id, cause=termination_cause
+ )
+ session_duration_ms = (time.perf_counter() - session_started_at) * 1000.0
+ metrics_for_close = getattr(self, "metrics", None)
+ if metrics_for_close is not None:
+ with contextlib.suppress(Exception):
+ if hasattr(metrics_for_close, "dec_active_ws_sessions"):
+ metrics_for_close.dec_active_ws_sessions()
+ if released_tasks and hasattr(metrics_for_close, "dec_active_relay_tasks"):
+ metrics_for_close.dec_active_relay_tasks(released_tasks)
+ if hasattr(metrics_for_close, "record_ws_session_duration"):
+ metrics_for_close.record_ws_session_duration(
+ session_duration_ms, termination_cause
+ )
+ await emit_stage_timings_log(
+ path="openai_responses_ws",
+ request_id=request_id,
+ session_id=session_id,
+ stage_timer=stage_timer,
+ expected_stages=(
+ "accept",
+ "first_client_frame",
+ "upstream_connect",
+ "upstream_first_event",
+ "memory_context",
+ "compression",
+ "total_session",
+ ),
+ metrics=getattr(self, "metrics", None),
+ )
+
+ async def _ws_http_fallback(
+ self,
+ websocket: WebSocket,
+ body: dict[str, Any],
+ first_msg_raw: str,
+ upstream_headers: dict[str, str],
+ request_id: str,
+ ) -> None:
+ """Fall back to HTTP POST streaming when upstream WS fails.
+
+ Converts the WS ``response.create`` message to an HTTP POST to
+ ``/v1/responses?stream=true``, reads SSE events, and relays each
+ ``data:`` line as a WS text message to the client. This makes
+ Codex work immediately instead of exhausting its WS retry budget.
+ """
+ # Route to correct endpoint based on auth mode
+ _lower = {k.lower() for k in upstream_headers}
+ if "chatgpt-account-id" in _lower:
+ http_url = "https://chatgpt.com/backend-api/codex/responses"
+ else:
+ http_url = f"{self.OPENAI_API_URL}/v1/responses"
+
+ # Build HTTP body from the WS response.create payload.
+ # WS messages use {"type": "response.create", "response": {...}} wrapper.
+ # The HTTP POST endpoint expects the inner response object directly.
+ http_body: dict[str, Any]
+ try:
+ parsed = json.loads(first_msg_raw) if isinstance(first_msg_raw, str) else body
+ except (json.JSONDecodeError, TypeError):
+ parsed = body
+
+ # Normalize WebSocket response.create payload into the HTTP request body.
+ # Codex may send either:
+ # 1. {"type":"response.create","response":{...}}
+ # 2. {"type":"response.create", ...response fields...}
+ if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):
+ http_body = dict(parsed["response"])
+ elif isinstance(parsed, dict):
+ http_body = dict(parsed)
+ if http_body.get("type") == "response.create":
+ http_body.pop("type", None)
+ else:
+ http_body = body if isinstance(body, dict) else {}
+
+ # Some clients include response-ish metadata that the HTTP endpoint rejects.
+ if http_body.get("type") in {"response.create", "response"}:
+ http_body.pop("type", None)
+
+ # Ensure streaming is enabled so we get SSE events
+ http_body["stream"] = True
+
+ # Build HTTP headers from the upstream headers (already stripped of WS
+ # hop-by-hop headers by the caller).
+ http_headers = dict(upstream_headers)
+ http_headers["content-type"] = "application/json"
+
+ logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")
+
+ try:
+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ for http_attempt in range(retry_attempts):
+ try:
+ async with self.http_client.stream(
+ "POST",
+ http_url,
+ headers=http_headers,
+ json=http_body,
+ timeout=120.0,
+ ) as response:
+ if response.status_code != 200:
+ error_body = b""
+ async for chunk in response.aiter_bytes():
+ error_body += chunk
+ if len(error_body) > 2000:
+ break
+ error_text = error_body.decode("utf-8", errors="replace")
+ logger.error(
+ f"[{request_id}] WS→HTTP fallback got {response.status_code}: "
+ f"{error_text[:500]}"
+ )
+ # Send error as WS message so client sees it
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "server_error",
+ "message": f"Upstream returned {response.status_code}",
+ },
+ }
+ await websocket.send_text(json.dumps(error_event))
+ return
+
+ # Relay SSE events as WS text messages
+ buffer = ""
+ async for chunk in response.aiter_text():
+ buffer += chunk
+ while "\n" in buffer:
+ line, buffer = buffer.split("\n", 1)
+ line = line.strip()
+ if not line:
+ continue
+ if line.startswith("data: "):
+ data = line[6:]
+ if data == "[DONE]":
+ continue
+ try:
+ await websocket.send_text(data)
+ except Exception:
+ return
+ elif line.startswith("event: "):
+ # SSE event type — skip, the data line contains the type
+ continue
+
+ # Flush any remaining data in buffer
+ for line in buffer.strip().splitlines():
+ line = line.strip()
+ if line.startswith("data: ") and line[6:] != "[DONE]":
+ with contextlib.suppress(Exception):
+ await websocket.send_text(line[6:])
+ return
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:
+ if http_attempt >= retry_attempts - 1:
+ raise
+
+ delay_with_jitter = jitter_delay_ms(
+ self.config.retry_base_delay_ms,
+ self.config.retry_max_delay_ms,
+ http_attempt,
+ )
+ logger.warning(
+ f"[{request_id}] WS→HTTP fallback connect failed "
+ f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ except Exception as http_err:
+ logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "server_error",
+ "message": f"HTTP fallback failed: {http_err!s}"[:200],
+ },
+ }
+ with contextlib.suppress(Exception):
+ await websocket.send_text(json.dumps(error_event))
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close()
+
+ async def handle_compress(self, request: Request) -> JSONResponse:
+ """Compress messages without calling an LLM.
+
+ POST /v1/compress
+ Body: {"messages": [...], "model": "...", "config": {}}
+ Returns compressed messages + metrics.
+ """
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+
+ # Check bypass header
+ if request.headers.get("x-headroom-bypass", "").lower() == "true":
+ try:
+ body = await _read_request_json(request)
+ except (json.JSONDecodeError, ValueError) as e:
+ return JSONResponse(
+ status_code=400,
+ content={"error": f"Invalid request body: {e!s}"},
+ )
+ messages = body.get("messages", [])
+ return JSONResponse(
+ {
+ "messages": messages,
+ "tokens_before": 0,
+ "tokens_after": 0,
+ "tokens_saved": 0,
+ "compression_ratio": 1.0,
+ "transforms_applied": [],
+ "ccr_hashes": [],
+ }
+ )
+
+ try:
+ body = await _read_request_json(request)
+ except Exception:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Invalid JSON in request body.",
+ }
+ },
+ )
+
+ messages = body.get("messages")
+ model = body.get("model")
+
+ if messages is None:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Missing required field: messages",
+ }
+ },
+ )
+
+ if model is None:
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "type": "invalid_request",
+ "message": "Missing required field: model",
+ }
+ },
+ )
+
+ if not messages:
+ return JSONResponse(
+ {
+ "messages": [],
+ "tokens_before": 0,
+ "tokens_after": 0,
+ "tokens_saved": 0,
+ "compression_ratio": 1.0,
+ "transforms_applied": [],
+ "ccr_hashes": [],
+ }
+ )
+
+ try:
+ # Use OpenAI pipeline (messages are in OpenAI format from TS SDK)
+ # Allow optional token_budget to override model's context limit
+ # (used by OpenClaw compact() and other callers that need tighter budgets)
+ token_budget = body.get("token_budget")
+ context_limit = (
+ token_budget
+ if token_budget and isinstance(token_budget, int)
+ else self.openai_provider.get_context_limit(model)
+ )
+ # Extract CompressConfig options from request body
+ compress_config = body.get("config", {})
+ compress_user_messages = compress_config.get("compress_user_messages", False)
+ target_ratio = compress_config.get("target_ratio")
+ protect_recent = compress_config.get("protect_recent")
+ protect_analysis_context = compress_config.get("protect_analysis_context")
+
+ pipeline_kwargs: dict = {"model_limit": context_limit}
+ if compress_user_messages:
+ pipeline_kwargs["compress_user_messages"] = True
+ if target_ratio is not None:
+ pipeline_kwargs["target_ratio"] = float(target_ratio)
+ if protect_recent is not None:
+ pipeline_kwargs["protect_recent"] = int(protect_recent)
+ if protect_analysis_context is not None:
+ pipeline_kwargs["protect_analysis_context"] = bool(protect_analysis_context)
+
+ result = self.openai_pipeline.apply(
+ messages=messages,
+ model=model,
+ **pipeline_kwargs,
+ )
+
+ return JSONResponse(
+ {
+ "messages": result.messages,
+ "tokens_before": result.tokens_before,
+ "tokens_after": result.tokens_after,
+ "tokens_saved": result.tokens_before - result.tokens_after,
+ "compression_ratio": (
+ result.tokens_after / result.tokens_before
+ if result.tokens_before > 0
+ else 1.0
+ ),
+ "transforms_applied": result.transforms_applied,
+ "transforms_summary": result.transforms_summary,
+ "ccr_hashes": result.markers_inserted,
+ }
+ )
+ except Exception as e:
+ logger.exception("Compression failed: %s", e)
+ return JSONResponse(
+ status_code=503,
+ content={
+ "error": {
+ "type": "compression_error",
+ "message": str(e),
+ }
+ },
+ )
+
+ async def handle_passthrough(
+ self,
+ request: Request,
+ base_url: str,
+ endpoint_name: str | None = None,
+ provider: str | None = None,
+ ) -> Response:
+ """Pass through request unchanged.
+
+ Args:
+ request: The incoming request
+ base_url: The upstream API base URL
+ endpoint_name: Optional name for stats tracking (e.g., "models", "embeddings")
+ provider: Optional provider name for stats (e.g., "openai", "anthropic", "gemini")
+ """
+ from fastapi.responses import Response
+
+ start_time = time.time()
+ path = request.url.path
+ url = f"{base_url}{path}"
+
+ # Preserve query string parameters
+ if request.url.query:
+ url = f"{url}?{request.url.query}"
+
+ headers = dict(request.headers.items())
+ headers.pop("host", None)
+ headers.pop("accept-encoding", None)
+
+ body = await request.body()
+
+ headers = await apply_copilot_api_auth(headers, url=url)
+ response = await self.http_client.request( # type: ignore[union-attr]
+ method=request.method,
+ url=url,
+ headers=headers,
+ content=body,
+ )
+
+ # Remove compression headers since httpx already decompressed the response
+ response_headers = dict(response.headers)
+ response_headers.pop("content-encoding", None)
+ response_headers.pop("content-length", None) # Length changed after decompression
+
+ # Track stats for passthrough requests
+ if endpoint_name and provider:
+ latency_ms = (time.time() - start_time) * 1000
+ await self.metrics.record_request(
+ provider=provider,
+ model=f"passthrough:{endpoint_name}",
+ input_tokens=0,
+ output_tokens=0,
+ tokens_saved=0,
+ latency_ms=latency_ms,
+ cached=False,
+ )
+
+ return Response(
+ content=response.content,
+ status_code=response.status_code,
+ headers=response_headers,
+ )
+
+ async def handle_databricks_invocations(
+ self,
+ request: Request,
+ model: str,
+ ) -> Response | StreamingResponse:
+ """Handle Databricks native /serving-endpoints/{model}/invocations endpoint.
+
+ This enables using the Databricks CLI directly with Headroom:
+ databricks serving-endpoints query <model> --profile HEADROOM --json '{"messages": [...]}'
+
+ The request/response format is identical to OpenAI chat completions,
+ so we inject the model from the path and delegate to handle_openai_chat.
+ """
+ from fastapi.responses import JSONResponse
+
+ from headroom.proxy.helpers import _read_request_json
+
+ request_id = await self._next_request_id()
+
+ try:
+ body = await _read_request_json(request)
+ except Exception as e:
+ logger.error(f"[{request_id}] Failed to parse Databricks request body: {e}")
+ return JSONResponse(
+ status_code=400,
+ content={
+ "error": {
+ "message": f"Invalid JSON: {e}",
+ "type": "invalid_request_error",
+ }
+ },
+ )
+
+ # Inject model from path into body (Databricks CLI passes model in URL, not body)
+ body["model"] = model
+
+ logger.info(f"[{request_id}] Databricks invocation: model={model}")
+
+ # Create a new request with the modified body
+ # We reuse the OpenAI chat handler since the format is identical
+ from starlette.requests import Request as StarletteRequest
+
+ # Build new scope with the body already parsed
+ scope = dict(request.scope)
+
+ # Create a simple receive function that returns our modified body
+ body_bytes = json.dumps(body).encode()
+
+ async def receive():
+ return {"type": "http.request", "body": body_bytes}
+
+ modified_request = StarletteRequest(scope, receive)
+
+ # Delegate to the OpenAI chat handler (same format)
+ return await self.handle_openai_chat(modified_request)
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 5708ee96a..559784d2e 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -211,6 +211,8 @@ class ProxyConfig:
# Compression Hooks
hooks: Any = None
+ pipeline_extensions: list[Any] = field(default_factory=list)
+ discover_pipeline_extensions: bool = True
# Subscription Window Tracking (Anthropic OAuth accounts)
subscription_tracking_enabled: bool = True
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 596893e66..7e6229119 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -87,6 +87,7 @@ from headroom.observability import (
shutdown_headroom_tracing,
shutdown_otel_metrics,
)
+from headroom.pipeline import PipelineExtensionManager, PipelineStage
from headroom.providers.anthropic import AnthropicProvider
from headroom.providers.openai import OpenAIProvider
@@ -211,6 +212,11 @@ class HeadroomProxy(
def __init__(self, config: ProxyConfig):
self.config = config
self.config.mode = normalize_proxy_mode(self.config.mode)
+ self.pipeline_extensions = PipelineExtensionManager(
+ hooks=config.hooks,
+ extensions=config.pipeline_extensions,
+ discover=config.discover_pipeline_extensions,
+ )
# Reset per-instance API targets first so test runs and multiple app instances
# do not leak class-level overrides across each other.
@@ -590,6 +596,17 @@ class HeadroomProxy(
else:
self.code_graph_watcher = None
+ self.pipeline_extensions.emit(
+ PipelineStage.SETUP,
+ operation="proxy.setup",
+ metadata={
+ "mode": self.config.mode,
+ "optimize": self.config.optimize,
+ "backend": self.config.backend,
+ "memory_enabled": self.config.memory_enabled,
+ },
+ )
+
def _get_compression_cache(self, session_id: str) -> CompressionCache:
"""Get or create a CompressionCache for a session."""
if session_id not in self._compression_caches:
@@ -645,6 +662,11 @@ class HeadroomProxy(
async def startup(self):
"""Initialize async resources."""
+ self.pipeline_extensions.emit(
+ PipelineStage.PRE_START,
+ operation="proxy.startup",
+ metadata={"port": self.config.port, "host": self.config.host},
+ )
self.http_client = httpx.AsyncClient(
timeout=httpx.Timeout(
connect=self.config.connect_timeout_seconds,
@@ -863,6 +885,16 @@ class HeadroomProxy(
else:
logger.info("Anonymous telemetry: DISABLED")
+ self.pipeline_extensions.emit(
+ PipelineStage.POST_START,
+ operation="proxy.startup",
+ metadata={
+ "port": self.config.port,
+ "host": self.config.host,
+ "warmup": self.warmup.to_dict(),
+ },
+ )
+
async def shutdown(self):
"""Cleanup async resources."""
if self.http_client:
diff --git a/tests/test_canonical_pipeline.py b/tests/test_canonical_pipeline.py
new file mode 100644
index 000000000..0f59cd762
--- /dev/null
+++ b/tests/test_canonical_pipeline.py
@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import importlib
+from types import SimpleNamespace
+from typing import Any
+
+from headroom.client import HeadroomClient
+from headroom.compress import compress
+from headroom.config import HeadroomConfig, HeadroomMode, TransformResult
+from headroom.hooks import CompressionHooks
+from headroom.pipeline import (
+ CANONICAL_PIPELINE_STAGES,
+ PipelineExtensionManager,
+ PipelineStage,
+ summarize_routing_markers,
+)
+from headroom.providers.base import Provider, TokenCounter
+
+
+class RecordingExtension:
+ def __init__(self) -> None:
+ self.stages: list[PipelineStage] = []
+
+ def on_pipeline_event(self, event):
+ self.stages.append(event.stage)
+ return None
+
+
+class MutatingExtension:
+ def on_pipeline_event(self, event):
+ if event.stage == PipelineStage.INPUT_RECEIVED:
+ event.messages = [{"role": "user", "content": "mutated"}]
+ return event
+
+
+class RecordingHooks(CompressionHooks):
+ def __init__(self) -> None:
+ self.stages: list[PipelineStage] = []
+ self.post_event = None
+
+ def pre_compress(self, messages, ctx):
+ return messages
+
+ def compute_biases(self, messages, ctx):
+ return {}
+
+ def post_compress(self, event):
+ self.post_event = event
+
+ def on_pipeline_event(self, event):
+ self.stages.append(event.stage)
+ return None
+
+
+class StubPipeline:
+ def apply(self, messages, model, **kwargs):
+ return TransformResult(
+ messages=messages,
+ tokens_before=20,
+ tokens_after=8,
+ transforms_applied=["router:text:kompress", "kompress:user:0.40"],
+ )
+
+ def _get_tokenizer(self, model):
+ return StubTokenCounter()
+
+
+class StubTokenCounter(TokenCounter):
+ def count_text(self, text: str) -> int:
+ return len(text.split())
+
+ def count_message(self, message: dict[str, Any]) -> int:
+ content = message.get("content", "")
+ if isinstance(content, str):
+ return len(content.split())
+ return 1
+
+ def count_messages(self, messages: list[dict[str, Any]]) -> int:
+ return sum(self.count_message(message) for message in messages)
+
+
+class StubProvider(Provider):
+ @property
+ def name(self) -> str:
+ return "openai"
+
+ def get_token_counter(self, model: str) -> TokenCounter:
+ return StubTokenCounter()
+
+ def get_context_limit(self, model: str) -> int:
+ return 128000
+
+ def supports_model(self, model: str) -> bool:
+ return True
+
+
+class DummyCompletions:
+ def __init__(self) -> None:
+ self.calls: list[dict[str, Any]] = []
+
+ def create(self, **kwargs: Any) -> dict[str, Any]:
+ self.calls.append(kwargs)
+ return {"id": "resp_123", "messages": kwargs["messages"]}
+
+
+class DummyOriginalClient:
+ def __init__(self) -> None:
+ self.chat = SimpleNamespace(completions=DummyCompletions())
+
+
+def test_pipeline_extension_manager_uses_canonical_stage_contract():
+ recorder = RecordingExtension()
+ manager = PipelineExtensionManager(
+ extensions=[recorder, MutatingExtension()],
+ discover=False,
+ )
+
+ event = manager.emit(
+ PipelineStage.INPUT_RECEIVED,
+ operation="test",
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "hello"}],
+ )
+
+ assert list(CANONICAL_PIPELINE_STAGES)[0] is PipelineStage.SETUP
+ assert summarize_routing_markers(["router:text:kompress", "smart:kept=3"]) == [
+ "router:text:kompress"
+ ]
+ assert recorder.stages == [PipelineStage.INPUT_RECEIVED]
+ assert event.messages == [{"role": "user", "content": "mutated"}]
+
+
+def test_compress_emits_canonical_pipeline_events(monkeypatch):
+ hooks = RecordingHooks()
+ compress_module = importlib.import_module("headroom.compress")
+ monkeypatch.setattr(compress_module, "_get_pipeline", lambda: StubPipeline())
+
+ result = compress(
+ [{"role": "user", "content": "hello world"}],
+ model="gpt-4o",
+ hooks=hooks,
+ )
+
+ assert result.tokens_before == 20
+ assert result.tokens_after == 2
+ assert hooks.post_event is not None
+ assert hooks.post_event.tokens_saved == 18
+ assert hooks.stages == [
+ PipelineStage.INPUT_RECEIVED,
+ PipelineStage.INPUT_ROUTED,
+ PipelineStage.INPUT_COMPRESSED,
+ ]
+
+
+def test_headroom_client_emits_canonical_pipeline_events(tmp_path):
+ recorder = RecordingExtension()
+ original = DummyOriginalClient()
+ config = HeadroomConfig(
+ store_url=f"jsonl://{tmp_path / 'headroom.jsonl'}",
+ default_mode=HeadroomMode.OPTIMIZE,
+ pipeline_extensions=[recorder],
+ discover_pipeline_extensions=False,
+ )
+ client = HeadroomClient(
+ original_client=original,
+ provider=StubProvider(),
+ store_url=f"jsonl://{tmp_path / 'headroom-client.jsonl'}",
+ enable_cache_optimizer=False,
+ config=config,
+ )
+ client._pipeline = StubPipeline()
+
+ response = client.chat.completions.create(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "hello world"}],
+ )
+
+ assert response["id"] == "resp_123"
+ assert recorder.stages == [
+ PipelineStage.SETUP,
+ PipelineStage.INPUT_RECEIVED,
+ PipelineStage.INPUT_ROUTED,
+ PipelineStage.INPUT_COMPRESSED,
+ PipelineStage.PRE_SEND,
+ PipelineStage.POST_SEND,
+ PipelineStage.RESPONSE_RECEIVED,
+ ]