Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): retry responses transport without bypassing compression

Adryan Eka Vandrab0422cd27ae5
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index b631c7b26..593fd4d6c 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -12,6 +12,7 @@ import copy
import json
import logging
import os
+import random
import time
from datetime import datetime
from typing import TYPE_CHECKING, Any
@@ -20,10 +21,17 @@ if TYPE_CHECKING:
from fastapi import Request, WebSocket
from fastapi.responses import JSONResponse, Response, StreamingResponse
+import httpx
+
logger = logging.getLogger("headroom.proxy")
+# Interactive Responses turns are latency-sensitive. Fail open quickly rather
+# than holding the session hostage on memory lookup.
+RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0
+
+
def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:
"""Best-effort decode of an OpenAI OAuth bearer token payload.
@@ -979,9 +987,19 @@ class OpenAIHandlerMixin:
try:
# Inject memory context into instructions (Responses API system prompt)
if self.memory_handler.config.inject_context:
- memory_context = await self.memory_handler.search_and_format_context(
- memory_user_id, optimized_messages
- )
+ try:
+ memory_context = await asyncio.wait_for(
+ self.memory_handler.search_and_format_context(
+ memory_user_id, optimized_messages
+ ),
+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
+ )
+ except TimeoutError:
+ memory_context = None
+ logger.info(
+ f"[{request_id}] Memory context lookup exceeded "
+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"
+ )
if memory_context:
existing_instructions = body.get("instructions") or ""
if existing_instructions:
@@ -1310,14 +1328,19 @@ class OpenAIHandlerMixin:
body: dict[str, Any] = {}
try:
body = json.loads(first_msg_raw)
- input_data = body.get("input")
tokens_saved = 0
+ ws_request_body = body.get("response", body)
+ input_data = ws_request_body.get("input") if isinstance(ws_request_body, dict) else None
should_compress = (
self.config.optimize
and isinstance(input_data, list)
and len(input_data) > 1
- and not body.get("previous_response_id")
+ and not (
+ ws_request_body.get("previous_response_id")
+ if isinstance(ws_request_body, dict)
+ else None
+ )
)
if should_compress:
try:
@@ -1326,11 +1349,11 @@ class OpenAIHandlerMixin:
responses_items_to_messages,
)
- model = body.get("model", "gpt-4o")
+ model = ws_request_body.get("model", "gpt-4o")
converted, preserved = responses_items_to_messages(input_data)
messages: list[dict[str, Any]] = []
- instructions = body.get("instructions")
+ instructions = ws_request_body.get("instructions")
if instructions:
messages.append({"role": "system", "content": instructions})
messages.extend(converted)
@@ -1354,10 +1377,10 @@ class OpenAIHandlerMixin:
if result.messages != messages:
opt = result.messages
if instructions and opt and opt[0].get("role") == "system":
- body["instructions"] = opt[0]["content"]
+ ws_request_body["instructions"] = opt[0]["content"]
opt = opt[1:]
if result.tokens_after <= original_tokens:
- body["input"] = messages_to_responses_items(
+ ws_request_body["input"] = messages_to_responses_items(
opt, input_data, preserved
)
else:
@@ -1366,6 +1389,10 @@ class OpenAIHandlerMixin:
f"({original_tokens} -> {result.tokens_after}), reverting"
)
tokens_saved = max(0, original_tokens - result.tokens_after)
+ if "response" in body and isinstance(body["response"], dict):
+ body["response"] = ws_request_body
+ else:
+ body = ws_request_body
first_msg_raw = json.dumps(body)
logger.info(
f"[{request_id}] WS /v1/responses compressed: "
@@ -1418,9 +1445,20 @@ class OpenAIHandlerMixin:
converted_msgs, _ = responses_items_to_messages(ws_input)
ws_msgs.extend(converted_msgs)
- memory_context = await self.memory_handler.search_and_format_context(
- memory_user_id, ws_msgs
- )
+ try:
+ memory_context = await asyncio.wait_for(
+ self.memory_handler.search_and_format_context(
+ memory_user_id, ws_msgs
+ ),
+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,
+ )
+ except TimeoutError:
+ memory_context = None
+ logger.info(
+ f"[{request_id}] WS Memory: Context lookup exceeded "
+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "
+ f"continuing without it"
+ )
if memory_context:
existing = ws_response_body.get("instructions") or ""
if existing:
@@ -1497,220 +1535,249 @@ class OpenAIHandlerMixin:
use_ssl: bool | None = True if upstream_url.startswith("wss://") else None
ws_connected = False
- try:
- async with websockets.connect(
- upstream_url,
- additional_headers=upstream_headers,
- subprotocols=(
- [websockets.Subprotocol(p) for p in client_subprotocols]
- if client_subprotocols and hasattr(websockets, "Subprotocol")
- else client_subprotocols or None
- ),
- ssl=use_ssl,
- ) as upstream:
- ws_connected = True
- # Send (potentially compressed) first message
- await upstream.send(first_msg_raw)
-
- # Bidirectional relay with memory tool interception
- async def _client_to_upstream() -> None:
- try:
- while True:
- msg = await websocket.receive_text()
- await upstream.send(msg)
- except Exception as relay_err:
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- logger.debug(
- f"[{request_id}] WS client→upstream relay ended: {relay_err}"
- )
- with contextlib.suppress(Exception):
- await upstream.close()
+ ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ ws_last_err: Exception | None = None
- async def _upstream_to_client() -> None:
- """Relay upstream→client with transparent memory tool handling.
+ for ws_attempt in range(ws_connect_attempts):
+ try:
+ async with websockets.connect(
+ upstream_url,
+ additional_headers=upstream_headers,
+ subprotocols=(
+ [websockets.Subprotocol(p) for p in client_subprotocols]
+ if client_subprotocols and hasattr(websockets, "Subprotocol")
+ else client_subprotocols or None
+ ),
+ ssl=use_ssl,
+ open_timeout=max(30, self.config.connect_timeout_seconds * 3),
+ close_timeout=10,
+ ping_interval=20,
+ ping_timeout=20,
+ ) as upstream:
+ ws_connected = True
+ await upstream.send(first_msg_raw)
+
+ async def _client_to_upstream() -> None:
+ try:
+ while True:
+ msg = await websocket.receive_text()
+ await upstream.send(msg)
+ except Exception as relay_err:
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ logger.debug(
+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"
+ )
+ with contextlib.suppress(Exception):
+ await upstream.close()
- Uses a buffer-then-decide approach:
- 1. Buffer events until first output item arrives
- 2. If first output is a memory tool → suppress entire response,
- execute tools silently, send continuation upstream
- 3. If first output is non-memory → flush buffer, stream normally
- 4. Continuation response events are relayed to Codex seamlessly
+ async def _upstream_to_client() -> None:
+ """Relay upstream→client with transparent memory tool handling.
- This prevents orphaned response.created events from confusing Codex.
- """
- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
+ Uses a buffer-then-decide approach:
+ 1. Buffer events until first output item arrives
+ 2. If first output is a memory tool → suppress entire response,
+ execute tools silently, send continuation upstream
+ 3. If first output is non-memory → flush buffer, stream normally
+ 4. Continuation response events are relayed to Codex seamlessly
- memory_enabled = bool(self.memory_handler and memory_user_id)
+ This prevents orphaned response.created events from confusing Codex.
+ """
+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES
- # Per-response state (reset after each response.completed)
- event_buffer: list[str] = []
- decided = False
- suppress_response = False
- pending_fcs: list[dict[str, Any]] = []
- resp_id: str | None = None
+ memory_enabled = bool(self.memory_handler and memory_user_id)
- def _reset() -> None:
- nonlocal decided, suppress_response, resp_id
- event_buffer.clear()
+ # Per-response state (reset after each response.completed)
+ event_buffer: list[str] = []
decided = False
suppress_response = False
- pending_fcs.clear()
- resp_id = None
-
- try:
- async for msg in upstream:
- if isinstance(msg, bytes):
- await websocket.send_bytes(msg)
- continue
- msg_str = msg if isinstance(msg, str) else str(msg)
-
- if not memory_enabled:
- await websocket.send_text(msg_str)
- continue
-
- # Parse event
- try:
- event = json.loads(msg_str)
- except (json.JSONDecodeError, TypeError):
- await websocket.send_text(msg_str)
- continue
-
- event_type = event.get("type", "")
+ pending_fcs: list[dict[str, Any]] = []
+ resp_id: str | None = None
- # --- Phase 1: Buffer until first output item ---
- if not decided:
- event_buffer.append(msg_str)
+ def _reset() -> None:
+ nonlocal decided, suppress_response, resp_id
+ event_buffer.clear()
+ decided = False
+ suppress_response = False
+ pending_fcs.clear()
+ resp_id = None
- if event_type == "response.output_item.added":
- item = event.get("item", {})
- if (
- item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ):
- # Memory tool first → suppress entire response
- suppress_response = True
- decided = True
- event_buffer.clear()
- logger.info(
- f"[{request_id}] WS Memory: Detected "
- f"{item.get('name')} — suppressing response"
- )
- else:
- # Non-memory first → flush buffer, pass through
+ try:
+ async for msg in upstream:
+ if isinstance(msg, bytes):
+ await websocket.send_bytes(msg)
+ continue
+ msg_str = msg if isinstance(msg, str) else str(msg)
+
+ if not memory_enabled:
+ await websocket.send_text(msg_str)
+ continue
+
+ # Parse event
+ try:
+ event = json.loads(msg_str)
+ except (json.JSONDecodeError, TypeError):
+ await websocket.send_text(msg_str)
+ continue
+
+ event_type = event.get("type", "")
+
+ # --- Phase 1: Buffer until first output item ---
+ if not decided:
+ event_buffer.append(msg_str)
+
+ if event_type == "response.output_item.added":
+ item = event.get("item", {})
+ if (
+ item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ):
+ # Memory tool first → suppress entire response
+ suppress_response = True
+ decided = True
+ event_buffer.clear()
+ logger.info(
+ f"[{request_id}] WS Memory: Detected "
+ f"{item.get('name')} — suppressing response"
+ )
+ else:
+ # Non-memory first → flush buffer, pass through
+ decided = True
+ for buf in event_buffer:
+ await websocket.send_text(buf)
+ event_buffer.clear()
+
+ elif event_type == "response.completed":
+ # No output items at all — flush
decided = True
for buf in event_buffer:
await websocket.send_text(buf)
event_buffer.clear()
+ _reset()
- elif event_type == "response.completed":
- # No output items at all — flush
- decided = True
- for buf in event_buffer:
- await websocket.send_text(buf)
- event_buffer.clear()
- _reset()
+ continue
- continue
+ # --- Phase 2a: Suppress mode (memory response) ---
+ if suppress_response:
+ if event_type == "response.output_item.done":
+ item = event.get("item", {})
+ if (
+ item.get("type") == "function_call"
+ and item.get("name") in MEMORY_TOOL_NAMES
+ ):
+ pending_fcs.append(item)
+
+ elif event_type == "response.completed":
+ resp = event.get("response", {})
+ resp_id = resp.get("id")
- # --- Phase 2a: Suppress mode (memory response) ---
- if suppress_response:
- if event_type == "response.output_item.done":
- item = event.get("item", {})
- if (
- item.get("type") == "function_call"
- and item.get("name") in MEMORY_TOOL_NAMES
- ):
- pending_fcs.append(item)
-
- elif event_type == "response.completed":
- resp = event.get("response", {})
- resp_id = resp.get("id")
-
- if pending_fcs:
- logger.info(
- f"[{request_id}] WS Memory: Executing "
- f"{len(pending_fcs)} tool(s) transparently"
- )
-
- # Execute memory tool calls
- tool_outputs: list[dict[str, Any]] = []
- for fc in pending_fcs:
- call_id = fc.get("call_id", fc.get("id", ""))
- fc_name = fc.get("name", "")
- args_str = fc.get("arguments", "{}")
- try:
- fc_args = json.loads(args_str)
- except json.JSONDecodeError:
- fc_args = {}
-
- await self.memory_handler._ensure_initialized()
- if self.memory_handler._backend:
- result = await self.memory_handler._execute_memory_tool(
- fc_name, fc_args, memory_user_id, "openai"
+ if pending_fcs:
+ logger.info(
+ f"[{request_id}] WS Memory: Executing "
+ f"{len(pending_fcs)} tool(s) transparently"
+ )
+
+ # Execute memory tool calls
+ tool_outputs: list[dict[str, Any]] = []
+ for fc in pending_fcs:
+ call_id = fc.get("call_id", fc.get("id", ""))
+ fc_name = fc.get("name", "")
+ args_str = fc.get("arguments", "{}")
+ try:
+ fc_args = json.loads(args_str)
+ except json.JSONDecodeError:
+ fc_args = {}
+
+ await self.memory_handler._ensure_initialized()
+ if self.memory_handler._backend:
+ result = await self.memory_handler._execute_memory_tool(
+ fc_name,
+ fc_args,
+ memory_user_id,
+ "openai",
+ )
+ else:
+ result = json.dumps(
+ {"error": "backend not ready"}
+ )
+
+ tool_outputs.append(
+ {
+ "type": "function_call_output",
+ "call_id": call_id,
+ "output": result,
+ }
)
- else:
- result = json.dumps(
- {"error": "backend not ready"}
+ logger.info(
+ f"[{request_id}] WS Memory: Executed "
+ f"{fc_name} for user {memory_user_id}"
)
- tool_outputs.append(
- {
- "type": "function_call_output",
- "call_id": call_id,
- "output": result,
- }
- )
+ # Send continuation upstream
+ cont: dict[str, Any] = {
+ "type": "response.create",
+ "response": {"input": tool_outputs},
+ }
+ if resp_id:
+ cont["response"]["previous_response_id"] = (
+ resp_id
+ )
+ await upstream.send(json.dumps(cont))
logger.info(
- f"[{request_id}] WS Memory: Executed "
- f"{fc_name} for user {memory_user_id}"
+ f"[{request_id}] WS Memory: Sent continuation "
+ f"with {len(tool_outputs)} result(s)"
)
- # Send continuation upstream
- cont: dict[str, Any] = {
- "type": "response.create",
- "response": {"input": tool_outputs},
- }
- if resp_id:
- cont["response"]["previous_response_id"] = resp_id
- await upstream.send(json.dumps(cont))
- logger.info(
- f"[{request_id}] WS Memory: Sent continuation "
- f"with {len(tool_outputs)} result(s)"
- )
-
- _reset()
- # All events suppressed in this mode
- continue
+ _reset()
+ # All events suppressed in this mode
+ continue
- # --- Phase 2b: Pass-through mode ---
- await websocket.send_text(msg_str)
+ # --- Phase 2b: Pass-through mode ---
+ await websocket.send_text(msg_str)
- except Exception as relay_err:
- if "WebSocketDisconnect" not in type(relay_err).__name__:
- logger.debug(
- f"[{request_id}] WS upstream→client relay ended: {relay_err}"
- )
- finally:
- with contextlib.suppress(Exception):
- await websocket.close()
-
- await asyncio.gather(
- _client_to_upstream(),
- _upstream_to_client(),
- return_exceptions=True,
- )
+ except Exception as relay_err:
+ if "WebSocketDisconnect" not in type(relay_err).__name__:
+ logger.debug(
+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"
+ )
+ finally:
+ with contextlib.suppress(Exception):
+ await websocket.close()
+
+ await asyncio.gather(
+ _client_to_upstream(),
+ _upstream_to_client(),
+ return_exceptions=True,
+ )
- logger.info(
- f"[{request_id}] WS /v1/responses completed (tokens_saved={tokens_saved})"
+ logger.info(
+ f"[{request_id}] WS /v1/responses completed (tokens_saved={tokens_saved})"
+ )
+ break
+ except Exception as ws_err:
+ if ws_connected:
+ # WS was established but broke mid-stream — re-raise
+ raise
+
+ ws_last_err = ws_err
+ if ws_attempt >= ws_connect_attempts - 1:
+ break
+
+ delay_with_jitter = min(
+ self.config.retry_base_delay_ms * (2**ws_attempt),
+ self.config.retry_max_delay_ms,
+ ) * (0.5 + random.random())
+ logger.warning(
+ f"[{request_id}] WS upstream connect failed "
+ f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
)
+ await asyncio.sleep(delay_with_jitter / 1000)
- except Exception as ws_err:
- if ws_connected:
- # WS was established but broke mid-stream — re-raise
- raise
+ if not ws_connected:
# WS upgrade failed (HTTP 500 from OpenAI is common).
# Fall back to HTTP POST streaming and relay SSE events
# back over the client WebSocket transparently.
+ ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")
_ws_detail = str(ws_err)
if hasattr(ws_err, "response"):
resp_body = getattr(getattr(ws_err, "response", None), "body", b"")
@@ -1785,11 +1852,22 @@ class OpenAIHandlerMixin:
except (json.JSONDecodeError, TypeError):
parsed = body
- # Unwrap the response.create envelope if present
- if isinstance(parsed.get("response"), dict):
- http_body = parsed["response"]
+ # Normalize WebSocket response.create payload into the HTTP request body.
+ # Codex may send either:
+ # 1. {"type":"response.create","response":{...}}
+ # 2. {"type":"response.create", ...response fields...}
+ if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):
+ http_body = dict(parsed["response"])
+ elif isinstance(parsed, dict):
+ http_body = dict(parsed)
+ if http_body.get("type") == "response.create":
+ http_body.pop("type", None)
else:
- http_body = parsed
+ http_body = body if isinstance(body, dict) else {}
+
+ # Some clients include response-ish metadata that the HTTP endpoint rejects.
+ if http_body.get("type") in {"response.create", "response"}:
+ http_body.pop("type", None)
# Ensure streaming is enabled so we get SSE events
http_body["stream"] = True
@@ -1802,62 +1880,80 @@ class OpenAIHandlerMixin:
logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")
try:
- async with self.http_client.stream(
- "POST",
- http_url,
- headers=http_headers,
- json=http_body,
- timeout=120.0,
- ) as response:
- if response.status_code != 200:
- error_body = b""
- async for chunk in response.aiter_bytes():
- error_body += chunk
- if len(error_body) > 2000:
- break
- error_text = error_body.decode("utf-8", errors="replace")
- logger.error(
- f"[{request_id}] WS→HTTP fallback got {response.status_code}: "
- f"{error_text[:500]}"
+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ for http_attempt in range(retry_attempts):
+ try:
+ async with self.http_client.stream(
+ "POST",
+ http_url,
+ headers=http_headers,
+ json=http_body,
+ timeout=120.0,
+ ) as response:
+ if response.status_code != 200:
+ error_body = b""
+ async for chunk in response.aiter_bytes():
+ error_body += chunk
+ if len(error_body) > 2000:
+ break
+ error_text = error_body.decode("utf-8", errors="replace")
+ logger.error(
+ f"[{request_id}] WS→HTTP fallback got {response.status_code}: "
+ f"{error_text[:500]}"
+ )
+ # Send error as WS message so client sees it
+ error_event = {
+ "type": "error",
+ "error": {
+ "type": "server_error",
+ "message": f"Upstream returned {response.status_code}",
+ },
+ }
+ await websocket.send_text(json.dumps(error_event))
+ return
+
+ # Relay SSE events as WS text messages
+ buffer = ""
+ async for chunk in response.aiter_text():
+ buffer += chunk
+ while "\n" in buffer:
+ line, buffer = buffer.split("\n", 1)
+ line = line.strip()
+ if not line:
+ continue
+ if line.startswith("data: "):
+ data = line[6:]
+ if data == "[DONE]":
+ continue
+ try:
+ await websocket.send_text(data)
+ except Exception:
+ return
+ elif line.startswith("event: "):
+ # SSE event type — skip, the data line contains the type
+ continue
+
+ # Flush any remaining data in buffer
+ for line in buffer.strip().splitlines():
+ line = line.strip()
+ if line.startswith("data: ") and line[6:] != "[DONE]":
+ with contextlib.suppress(Exception):
+ await websocket.send_text(line[6:])
+ return
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:
+ if http_attempt >= retry_attempts - 1:
+ raise
+
+ delay_with_jitter = min(
+ self.config.retry_base_delay_ms * (2**http_attempt),
+ self.config.retry_max_delay_ms,
+ ) * (0.5 + random.random())
+ logger.warning(
+ f"[{request_id}] WS→HTTP fallback connect failed "
+ f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
)
- # Send error as WS message so client sees it
- error_event = {
- "type": "error",
- "error": {
- "type": "server_error",
- "message": f"Upstream returned {response.status_code}",
- },
- }
- await websocket.send_text(json.dumps(error_event))
- return
-
- # Relay SSE events as WS text messages
- buffer = ""
- async for chunk in response.aiter_text():
- buffer += chunk
- while "\n" in buffer:
- line, buffer = buffer.split("\n", 1)
- line = line.strip()
- if not line:
- continue
- if line.startswith("data: "):
- data = line[6:]
- if data == "[DONE]":
- continue
- try:
- await websocket.send_text(data)
- except Exception:
- return
- elif line.startswith("event: "):
- # SSE event type — skip, the data line contains the type
- continue
-
- # Flush any remaining data in buffer
- for line in buffer.strip().splitlines():
- line = line.strip()
- if line.startswith("data: ") and line[6:] != "[DONE]":
- with contextlib.suppress(Exception):
- await websocket.send_text(line[6:])
+ await asyncio.sleep(delay_with_jitter / 1000)
except Exception as http_err:
logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 3bb3f0b45..046035d68 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -5,9 +5,11 @@ Contains SSE parsing, streaming response generation, and related utilities.
from __future__ import annotations
+import asyncio
import contextlib
import json
import logging
+import random
import time
from datetime import datetime
from typing import TYPE_CHECKING, Any
@@ -493,10 +495,37 @@ class StreamingMixin:
# (needed to forward ratelimit headers to the client via StreamingResponse)
assert self.http_client is not None, "http_client must be initialized before streaming"
try:
- _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)
- upstream_response = await self.http_client.send(_upstream_req, stream=True)
+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))
+ upstream_response = None
+ last_connect_error = None
+
+ for attempt in range(retry_attempts):
+ try:
+ _upstream_req = self.http_client.build_request(
+ "POST", url, json=body, headers=headers
+ )
+ upstream_response = await self.http_client.send(_upstream_req, stream=True)
+ break
+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
+ last_connect_error = e
+ if attempt >= retry_attempts - 1:
+ raise
+
+ delay_with_jitter = min(
+ self.config.retry_base_delay_ms * (2**attempt),
+ self.config.retry_max_delay_ms,
+ ) * (0.5 + random.random())
+ logger.warning(
+ f"[{request_id}] Connection error to upstream API "
+ f"(attempt {attempt + 1}/{retry_attempts}): {e!r}; "
+ f"retrying in {delay_with_jitter:.0f}ms"
+ )
+ await asyncio.sleep(delay_with_jitter / 1000)
+
+ if upstream_response is None:
+ raise last_connect_error or RuntimeError("upstream connection did not start")
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
- error_msg = str(e)
+ error_msg = str(e) or repr(e)
logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")
async def _error_gen():
diff --git a/tests/test_openai_codex_routing.py b/tests/test_openai_codex_routing.py
index 46934b8df..62eb5e327 100644
--- a/tests/test_openai_codex_routing.py
+++ b/tests/test_openai_codex_routing.py
@@ -104,7 +104,7 @@ class _DummyMetrics:
async def record_request(self, **kwargs): # noqa: ANN003
return None
- async def record_failed(self):
+ async def record_failed(self, **kwargs): # noqa: ANN003
return None
@@ -128,24 +128,58 @@ class _DummyOpenAIHandler(OpenAIHandlerMixin):
def __init__(self) -> None:
self.rate_limiter = None
self.metrics = _DummyMetrics()
- self.config = SimpleNamespace(optimize=False)
+ self.config = SimpleNamespace(
+ optimize=False,
+ retry_max_attempts=3,
+ retry_base_delay_ms=10,
+ retry_max_delay_ms=50,
+ connect_timeout_seconds=10,
+ )
self.usage_reporter = None
- self.openai_provider = SimpleNamespace()
+ self.openai_provider = SimpleNamespace(get_context_limit=lambda model: 128_000)
+ self.openai_pipeline = SimpleNamespace(apply=MagicMock())
self.anthropic_backend = None
self.cost_tracker = None
self.memory_handler = None
self.captured_request: tuple[str, str, dict, dict] | None = None
+ self.captured_stream_request: tuple[str, dict, dict] | None = None
async def _next_request_id(self) -> str:
return "req-1"
- def _extract_tags(self, headers: dict[str, str]) -> list[str]:
- return []
+ def _extract_tags(self, headers: dict[str, str]) -> dict[str, str]:
+ return {}
async def _retry_request(self, method: str, url: str, headers: dict, body: dict):
self.captured_request = (method, url, headers, body)
return _ResponseStub()
+ async def _stream_response(
+ self,
+ url: str,
+ headers: dict,
+ body: dict,
+ provider: str,
+ model: str,
+ request_id: str,
+ original_tokens: int,
+ optimized_tokens: int,
+ tokens_saved: int,
+ transforms_applied: list[str],
+ tags: dict[str, str],
+ optimization_latency: float,
+ memory_user_id: str | None = None,
+ **kwargs,
+ ):
+ self.captured_stream_request = (url, headers, body)
+ return SimpleNamespace(
+ status_code=200,
+ url=url,
+ headers=headers,
+ body=body,
+ memory_user_id=memory_user_id,
+ )
+
def _build_request(body: dict, headers: dict[str, str]) -> Request:
payload = json.dumps(body).encode("utf-8")
@@ -198,6 +232,79 @@ def test_handle_openai_responses_routes_chatgpt_auth_to_backend_api(monkeypatch)
assert response.status_code == 200
+def test_handle_openai_responses_stream_keeps_compression(monkeypatch):
+ request = _build_request(
+ {
+ "model": "gpt-5.4",
+ "stream": True,
+ "instructions": "Keep it short",
+ "input": [
+ {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": "hello"}],
+ }
+ ],
+ },
+ {"Authorization": "Bearer sk-test"},
+ )
+ handler = _DummyOpenAIHandler()
+ handler.config.optimize = True
+ handler.openai_pipeline.apply.return_value = SimpleNamespace(
+ messages=[
+ {"role": "system", "content": "Keep it short"},
+ {"role": "user", "content": "hello"},
+ ],
+ transforms_applied=[],
+ tokens_before=2,
+ tokens_after=2,
+ )
+
+ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())
+
+ response = anyio.run(handler.handle_openai_responses, request)
+
+ assert response.status_code == 200
+ assert handler.captured_stream_request is not None
+ assert handler.openai_pipeline.apply.call_count == 1
+ assert handler.captured_stream_request[2]["stream"] is True
+
+
+def test_handle_openai_responses_memory_timeout_fails_open(monkeypatch):
+ class _SlowMemoryHandler:
+ def __init__(self):
+ self.config = SimpleNamespace(inject_context=True, inject_tools=False)
+
+ async def search_and_format_context(self, memory_user_id, messages):
+ return "should not be used"
+
+ def has_memory_tool_calls(self, response, provider):
+ return False
+
+ async def _timeout_wait_for(awaitable, timeout):
+ close = getattr(awaitable, "close", None)
+ if callable(close):
+ close()
+ raise TimeoutError
+
+ request = _build_request(
+ {"model": "gpt-5.4", "input": "hello"},
+ {"Authorization": "Bearer sk-test", "x-headroom-user-id": "user-1"},
+ )
+ handler = _DummyOpenAIHandler()
+ handler.memory_handler = _SlowMemoryHandler()
+
+ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())
+ monkeypatch.setattr("headroom.proxy.handlers.openai.asyncio.wait_for", _timeout_wait_for)
+
+ response = anyio.run(handler.handle_openai_responses, request)
+
+ assert response.status_code == 200
+ assert handler.captured_request is not None
+ _, _, _, body = handler.captured_request
+ assert body.get("instructions") is None
+
+
class _DummyWebSocket:
def __init__(self, headers: dict[str, str]):
self.headers = headers
diff --git a/tests/test_proxy_streaming_ratelimit_headers.py b/tests/test_proxy_streaming_ratelimit_headers.py
index 6c1d306b3..3fbcc377e 100644
--- a/tests/test_proxy_streaming_ratelimit_headers.py
+++ b/tests/test_proxy_streaming_ratelimit_headers.py
@@ -24,6 +24,9 @@ class TestStreamingRatelimitHeaderForwarding:
"""Create a HeadroomProxy with mocked internals for unit testing."""
proxy = object.__new__(HeadroomProxy)
proxy.http_client = MagicMock(spec=httpx.AsyncClient)
+ proxy.metrics = MagicMock()
+ proxy.metrics.record_request = AsyncMock(return_value=None)
+ proxy.metrics.record_failed = AsyncMock(return_value=None)
proxy.cost_tracker = MagicMock()
proxy.cost_tracker.estimate_cost.return_value = 0.001
proxy.cost_tracker.record_request.return_value = None
@@ -40,6 +43,10 @@ class TestStreamingRatelimitHeaderForwarding:
proxy._config = MagicMock()
proxy._config.memory_enabled = False
proxy._config.ccr_inject_tool = False
+ proxy._config.retry_max_attempts = 3
+ proxy._config.retry_base_delay_ms = 0
+ proxy._config.retry_max_delay_ms = 0
+ proxy.config = proxy._config
proxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)
proxy.memory_handler = None
return proxy
@@ -241,3 +248,48 @@ class TestStreamingRatelimitHeaderForwarding:
error_data = json.loads(raw.split("data: ")[1].strip())
assert error_data["error"]["type"] == "connection_error"
assert "Connection refused" in error_data["error"]["message"]
+
+ @pytest.mark.asyncio
+ async def test_connect_timeout_retries_before_returning_stream(self):
+ """Transient connect timeouts should retry before failing the stream."""
+ proxy = self._create_mock_proxy()
+ mock_response = self._create_mock_upstream_response()
+
+ mock_request = MagicMock()
+ proxy.http_client.build_request = MagicMock(return_value=mock_request)
+ attempts = {"count": 0}
+
+ async def flaky_send(*args, **kwargs):
+ attempts["count"] += 1
+ if attempts["count"] == 1:
+ raise httpx.ConnectTimeout("timed out")
+ return mock_response
+
+ proxy.http_client.send = AsyncMock(side_effect=flaky_send)
+
+ result = await proxy._stream_response(
+ url="https://api.anthropic.com/v1/messages",
+ headers={"x-api-key": "sk-test"},
+ body={
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 100,
+ "stream": True,
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ provider="anthropic",
+ model="claude-sonnet-4-20250514",
+ request_id="test-retry",
+ original_tokens=10,
+ optimized_tokens=10,
+ tokens_saved=0,
+ transforms_applied=[],
+ tags={},
+ optimization_latency=0.0,
+ )
+
+ chunks = []
+ async for chunk in result.body_iterator:
+ chunks.append(chunk)
+
+ assert attempts["count"] == 2
+ assert chunks
diff --git a/tests/test_ws_http_fallback.py b/tests/test_ws_http_fallback.py
index 47e40b800..ac2ffe23e 100644
--- a/tests/test_ws_http_fallback.py
+++ b/tests/test_ws_http_fallback.py
@@ -9,6 +9,9 @@ from __future__ import annotations
import asyncio
import json
+from types import SimpleNamespace
+
+import httpx
class FakeWebSocket:
@@ -63,6 +66,11 @@ def _make_handler():
obj = object.__new__(OpenAIHandlerMixin)
obj.OPENAI_API_URL = "https://api.openai.com"
obj.http_client = None
+ obj.config = SimpleNamespace(
+ retry_max_attempts=3,
+ retry_base_delay_ms=0,
+ retry_max_delay_ms=0,
+ )
return obj
@@ -160,6 +168,27 @@ class TestWsHttpFallback:
assert posted["stream"] is True
assert "input" in posted
+ def test_fallback_strips_top_level_response_create_type(self):
+ """HTTP fallback should strip top-level response.create metadata."""
+ handler = _make_handler()
+ ws = FakeWebSocket()
+ captured_kwargs: dict = {}
+
+ class CapturingClient:
+ def stream(self, method, url, **kwargs):
+ captured_kwargs.update(kwargs)
+ return FakeStreamResponse(200, ["data: [DONE]\n\n"])
+
+ handler.http_client = CapturingClient()
+
+ body = {"type": "response.create", "model": "gpt-5.4", "input": "hi"}
+ asyncio.run(handler._ws_http_fallback(ws, body, json.dumps(body), {}, "req_type_strip"))
+
+ posted = captured_kwargs["json"]
+ assert posted["model"] == "gpt-5.4"
+ assert posted["stream"] is True
+ assert "type" not in posted
+
def test_fallback_handles_http_exception(self):
"""HTTP fallback should send error event when HTTP request fails."""
handler = _make_handler()
@@ -179,6 +208,28 @@ class TestWsHttpFallback:
assert event["type"] == "error"
assert "unreachable" in event["error"]["message"]
+ def test_fallback_retries_connect_timeout(self):
+ """HTTP fallback should retry transient connect timeouts."""
+ handler = _make_handler()
+ ws = FakeWebSocket()
+ attempts = {"count": 0}
+
+ class FlakyClient:
+ def stream(self, method, url, **kwargs):
+ attempts["count"] += 1
+ if attempts["count"] == 1:
+ raise httpx.ConnectTimeout("timed out")
+ return FakeStreamResponse(200, ['data: {"type":"response.completed"}\n\n'])
+
+ handler.http_client = FlakyClient()
+
+ body = {"model": "gpt-5.4", "input": "test"}
+ asyncio.run(handler._ws_http_fallback(ws, body, json.dumps(body), {}, "req_retry"))
+
+ assert attempts["count"] == 2
+ assert len(ws.sent_texts) == 1
+ assert json.loads(ws.sent_texts[0])["type"] == "response.completed"
+
def test_fallback_routes_chatgpt_auth_to_chatgpt_domain(self):
"""ChatGPT session auth should route to chatgpt.com, not api.openai.com."""
handler = _make_handler()