Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): retry responses transport without bypassing compression
Adryan Eka Vandrab0422cd27ae5
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex b631c7b26..593fd4d6c 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -12,6 +12,7 @@ import copyimport jsonimport loggingimport os+import randomimport timefrom datetime import datetimefrom typing import TYPE_CHECKING, Any@@ -20,10 +21,17 @@ if TYPE_CHECKING:from fastapi import Request, WebSocketfrom fastapi.responses import JSONResponse, Response, StreamingResponse+import httpx+logger = logging.getLogger("headroom.proxy")+# Interactive Responses turns are latency-sensitive. Fail open quickly rather+# than holding the session hostage on memory lookup.+RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS = 2.0++def _decode_openai_bearer_payload(headers: dict[str, str]) -> dict[str, Any] | None:"""Best-effort decode of an OpenAI OAuth bearer token payload.@@ -979,9 +987,19 @@ class OpenAIHandlerMixin:try:# Inject memory context into instructions (Responses API system prompt)if self.memory_handler.config.inject_context:- memory_context = await self.memory_handler.search_and_format_context(- memory_user_id, optimized_messages- )+ try:+ memory_context = await asyncio.wait_for(+ self.memory_handler.search_and_format_context(+ memory_user_id, optimized_messages+ ),+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,+ )+ except TimeoutError:+ memory_context = None+ logger.info(+ f"[{request_id}] Memory context lookup exceeded "+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; continuing without it"+ )if memory_context:existing_instructions = body.get("instructions") or ""if existing_instructions:@@ -1310,14 +1328,19 @@ class OpenAIHandlerMixin:body: dict[str, Any] = {}try:body = json.loads(first_msg_raw)- input_data = body.get("input")tokens_saved = 0+ ws_request_body = body.get("response", body)+ input_data = ws_request_body.get("input") if isinstance(ws_request_body, dict) else Noneshould_compress = (self.config.optimizeand isinstance(input_data, list)and len(input_data) > 1- and not body.get("previous_response_id")+ and not (+ ws_request_body.get("previous_response_id")+ if isinstance(ws_request_body, dict)+ else None+ ))if should_compress:try:@@ -1326,11 +1349,11 @@ class OpenAIHandlerMixin:responses_items_to_messages,)- model = body.get("model", "gpt-4o")+ model = ws_request_body.get("model", "gpt-4o")converted, preserved = responses_items_to_messages(input_data)messages: list[dict[str, Any]] = []- instructions = body.get("instructions")+ instructions = ws_request_body.get("instructions")if instructions:messages.append({"role": "system", "content": instructions})messages.extend(converted)@@ -1354,10 +1377,10 @@ class OpenAIHandlerMixin:if result.messages != messages:opt = result.messagesif instructions and opt and opt[0].get("role") == "system":- body["instructions"] = opt[0]["content"]+ ws_request_body["instructions"] = opt[0]["content"]opt = opt[1:]if result.tokens_after <= original_tokens:- body["input"] = messages_to_responses_items(+ ws_request_body["input"] = messages_to_responses_items(opt, input_data, preserved)else:@@ -1366,6 +1389,10 @@ class OpenAIHandlerMixin:f"({original_tokens} -> {result.tokens_after}), reverting")tokens_saved = max(0, original_tokens - result.tokens_after)+ if "response" in body and isinstance(body["response"], dict):+ body["response"] = ws_request_body+ else:+ body = ws_request_bodyfirst_msg_raw = json.dumps(body)logger.info(f"[{request_id}] WS /v1/responses compressed: "@@ -1418,9 +1445,20 @@ class OpenAIHandlerMixin:converted_msgs, _ = responses_items_to_messages(ws_input)ws_msgs.extend(converted_msgs)- memory_context = await self.memory_handler.search_and_format_context(- memory_user_id, ws_msgs- )+ try:+ memory_context = await asyncio.wait_for(+ self.memory_handler.search_and_format_context(+ memory_user_id, ws_msgs+ ),+ timeout=RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS,+ )+ except TimeoutError:+ memory_context = None+ logger.info(+ f"[{request_id}] WS Memory: Context lookup exceeded "+ f"{RESPONSES_CONTEXT_SEARCH_TIMEOUT_SECONDS:.1f}s; "+ f"continuing without it"+ )if memory_context:existing = ws_response_body.get("instructions") or ""if existing:@@ -1497,220 +1535,249 @@ class OpenAIHandlerMixin:use_ssl: bool | None = True if upstream_url.startswith("wss://") else Nonews_connected = False- try:- async with websockets.connect(- upstream_url,- additional_headers=upstream_headers,- subprotocols=(- [websockets.Subprotocol(p) for p in client_subprotocols]- if client_subprotocols and hasattr(websockets, "Subprotocol")- else client_subprotocols or None- ),- ssl=use_ssl,- ) as upstream:- ws_connected = True- # Send (potentially compressed) first message- await upstream.send(first_msg_raw)-- # Bidirectional relay with memory tool interception- async def _client_to_upstream() -> None:- try:- while True:- msg = await websocket.receive_text()- await upstream.send(msg)- except Exception as relay_err:- if "WebSocketDisconnect" not in type(relay_err).__name__:- logger.debug(- f"[{request_id}] WS client→upstream relay ended: {relay_err}"- )- with contextlib.suppress(Exception):- await upstream.close()+ ws_connect_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))+ ws_last_err: Exception | None = None- async def _upstream_to_client() -> None:- """Relay upstream→client with transparent memory tool handling.+ for ws_attempt in range(ws_connect_attempts):+ try:+ async with websockets.connect(+ upstream_url,+ additional_headers=upstream_headers,+ subprotocols=(+ [websockets.Subprotocol(p) for p in client_subprotocols]+ if client_subprotocols and hasattr(websockets, "Subprotocol")+ else client_subprotocols or None+ ),+ ssl=use_ssl,+ open_timeout=max(30, self.config.connect_timeout_seconds * 3),+ close_timeout=10,+ ping_interval=20,+ ping_timeout=20,+ ) as upstream:+ ws_connected = True+ await upstream.send(first_msg_raw)++ async def _client_to_upstream() -> None:+ try:+ while True:+ msg = await websocket.receive_text()+ await upstream.send(msg)+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ logger.debug(+ f"[{request_id}] WS client→upstream relay ended: {relay_err}"+ )+ with contextlib.suppress(Exception):+ await upstream.close()- Uses a buffer-then-decide approach:- 1. Buffer events until first output item arrives- 2. If first output is a memory tool → suppress entire response,- execute tools silently, send continuation upstream- 3. If first output is non-memory → flush buffer, stream normally- 4. Continuation response events are relayed to Codex seamlessly+ async def _upstream_to_client() -> None:+ """Relay upstream→client with transparent memory tool handling.- This prevents orphaned response.created events from confusing Codex.- """- from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES+ Uses a buffer-then-decide approach:+ 1. Buffer events until first output item arrives+ 2. If first output is a memory tool → suppress entire response,+ execute tools silently, send continuation upstream+ 3. If first output is non-memory → flush buffer, stream normally+ 4. Continuation response events are relayed to Codex seamlessly- memory_enabled = bool(self.memory_handler and memory_user_id)+ This prevents orphaned response.created events from confusing Codex.+ """+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES- # Per-response state (reset after each response.completed)- event_buffer: list[str] = []- decided = False- suppress_response = False- pending_fcs: list[dict[str, Any]] = []- resp_id: str | None = None+ memory_enabled = bool(self.memory_handler and memory_user_id)- def _reset() -> None:- nonlocal decided, suppress_response, resp_id- event_buffer.clear()+ # Per-response state (reset after each response.completed)+ event_buffer: list[str] = []decided = Falsesuppress_response = False- pending_fcs.clear()- resp_id = None-- try:- async for msg in upstream:- if isinstance(msg, bytes):- await websocket.send_bytes(msg)- continue- msg_str = msg if isinstance(msg, str) else str(msg)-- if not memory_enabled:- await websocket.send_text(msg_str)- continue-- # Parse event- try:- event = json.loads(msg_str)- except (json.JSONDecodeError, TypeError):- await websocket.send_text(msg_str)- continue-- event_type = event.get("type", "")+ pending_fcs: list[dict[str, Any]] = []+ resp_id: str | None = None- # --- Phase 1: Buffer until first output item ---- if not decided:- event_buffer.append(msg_str)+ def _reset() -> None:+ nonlocal decided, suppress_response, resp_id+ event_buffer.clear()+ decided = False+ suppress_response = False+ pending_fcs.clear()+ resp_id = None- if event_type == "response.output_item.added":- item = event.get("item", {})- if (- item.get("type") == "function_call"- and item.get("name") in MEMORY_TOOL_NAMES- ):- # Memory tool first → suppress entire response- suppress_response = True- decided = True- event_buffer.clear()- logger.info(- f"[{request_id}] WS Memory: Detected "- f"{item.get('name')} — suppressing response"- )- else:- # Non-memory first → flush buffer, pass through+ try:+ async for msg in upstream:+ if isinstance(msg, bytes):+ await websocket.send_bytes(msg)+ continue+ msg_str = msg if isinstance(msg, str) else str(msg)++ if not memory_enabled:+ await websocket.send_text(msg_str)+ continue++ # Parse event+ try:+ event = json.loads(msg_str)+ except (json.JSONDecodeError, TypeError):+ await websocket.send_text(msg_str)+ continue++ event_type = event.get("type", "")++ # --- Phase 1: Buffer until first output item ---+ if not decided:+ event_buffer.append(msg_str)++ if event_type == "response.output_item.added":+ item = event.get("item", {})+ if (+ item.get("type") == "function_call"+ and item.get("name") in MEMORY_TOOL_NAMES+ ):+ # Memory tool first → suppress entire response+ suppress_response = True+ decided = True+ event_buffer.clear()+ logger.info(+ f"[{request_id}] WS Memory: Detected "+ f"{item.get('name')} — suppressing response"+ )+ else:+ # Non-memory first → flush buffer, pass through+ decided = True+ for buf in event_buffer:+ await websocket.send_text(buf)+ event_buffer.clear()++ elif event_type == "response.completed":+ # No output items at all — flushdecided = Truefor buf in event_buffer:await websocket.send_text(buf)event_buffer.clear()+ _reset()- elif event_type == "response.completed":- # No output items at all — flush- decided = True- for buf in event_buffer:- await websocket.send_text(buf)- event_buffer.clear()- _reset()+ continue- continue+ # --- Phase 2a: Suppress mode (memory response) ---+ if suppress_response:+ if event_type == "response.output_item.done":+ item = event.get("item", {})+ if (+ item.get("type") == "function_call"+ and item.get("name") in MEMORY_TOOL_NAMES+ ):+ pending_fcs.append(item)++ elif event_type == "response.completed":+ resp = event.get("response", {})+ resp_id = resp.get("id")- # --- Phase 2a: Suppress mode (memory response) ---- if suppress_response:- if event_type == "response.output_item.done":- item = event.get("item", {})- if (- item.get("type") == "function_call"- and item.get("name") in MEMORY_TOOL_NAMES- ):- pending_fcs.append(item)-- elif event_type == "response.completed":- resp = event.get("response", {})- resp_id = resp.get("id")-- if pending_fcs:- logger.info(- f"[{request_id}] WS Memory: Executing "- f"{len(pending_fcs)} tool(s) transparently"- )-- # Execute memory tool calls- tool_outputs: list[dict[str, Any]] = []- for fc in pending_fcs:- call_id = fc.get("call_id", fc.get("id", ""))- fc_name = fc.get("name", "")- args_str = fc.get("arguments", "{}")- try:- fc_args = json.loads(args_str)- except json.JSONDecodeError:- fc_args = {}-- await self.memory_handler._ensure_initialized()- if self.memory_handler._backend:- result = await self.memory_handler._execute_memory_tool(- fc_name, fc_args, memory_user_id, "openai"+ if pending_fcs:+ logger.info(+ f"[{request_id}] WS Memory: Executing "+ f"{len(pending_fcs)} tool(s) transparently"+ )++ # Execute memory tool calls+ tool_outputs: list[dict[str, Any]] = []+ for fc in pending_fcs:+ call_id = fc.get("call_id", fc.get("id", ""))+ fc_name = fc.get("name", "")+ args_str = fc.get("arguments", "{}")+ try:+ fc_args = json.loads(args_str)+ except json.JSONDecodeError:+ fc_args = {}++ await self.memory_handler._ensure_initialized()+ if self.memory_handler._backend:+ result = await self.memory_handler._execute_memory_tool(+ fc_name,+ fc_args,+ memory_user_id,+ "openai",+ )+ else:+ result = json.dumps(+ {"error": "backend not ready"}+ )++ tool_outputs.append(+ {+ "type": "function_call_output",+ "call_id": call_id,+ "output": result,+ })- else:- result = json.dumps(- {"error": "backend not ready"}+ logger.info(+ f"[{request_id}] WS Memory: Executed "+ f"{fc_name} for user {memory_user_id}")- tool_outputs.append(- {- "type": "function_call_output",- "call_id": call_id,- "output": result,- }- )+ # Send continuation upstream+ cont: dict[str, Any] = {+ "type": "response.create",+ "response": {"input": tool_outputs},+ }+ if resp_id:+ cont["response"]["previous_response_id"] = (+ resp_id+ )+ await upstream.send(json.dumps(cont))logger.info(- f"[{request_id}] WS Memory: Executed "- f"{fc_name} for user {memory_user_id}"+ f"[{request_id}] WS Memory: Sent continuation "+ f"with {len(tool_outputs)} result(s)")- # Send continuation upstream- cont: dict[str, Any] = {- "type": "response.create",- "response": {"input": tool_outputs},- }- if resp_id:- cont["response"]["previous_response_id"] = resp_id- await upstream.send(json.dumps(cont))- logger.info(- f"[{request_id}] WS Memory: Sent continuation "- f"with {len(tool_outputs)} result(s)"- )-- _reset()- # All events suppressed in this mode- continue+ _reset()+ # All events suppressed in this mode+ continue- # --- Phase 2b: Pass-through mode ---- await websocket.send_text(msg_str)+ # --- Phase 2b: Pass-through mode ---+ await websocket.send_text(msg_str)- except Exception as relay_err:- if "WebSocketDisconnect" not in type(relay_err).__name__:- logger.debug(- f"[{request_id}] WS upstream→client relay ended: {relay_err}"- )- finally:- with contextlib.suppress(Exception):- await websocket.close()-- await asyncio.gather(- _client_to_upstream(),- _upstream_to_client(),- return_exceptions=True,- )+ except Exception as relay_err:+ if "WebSocketDisconnect" not in type(relay_err).__name__:+ logger.debug(+ f"[{request_id}] WS upstream→client relay ended: {relay_err}"+ )+ finally:+ with contextlib.suppress(Exception):+ await websocket.close()++ await asyncio.gather(+ _client_to_upstream(),+ _upstream_to_client(),+ return_exceptions=True,+ )- logger.info(- f"[{request_id}] WS /v1/responses completed (tokens_saved={tokens_saved})"+ logger.info(+ f"[{request_id}] WS /v1/responses completed (tokens_saved={tokens_saved})"+ )+ break+ except Exception as ws_err:+ if ws_connected:+ # WS was established but broke mid-stream — re-raise+ raise++ ws_last_err = ws_err+ if ws_attempt >= ws_connect_attempts - 1:+ break++ delay_with_jitter = min(+ self.config.retry_base_delay_ms * (2**ws_attempt),+ self.config.retry_max_delay_ms,+ ) * (0.5 + random.random())+ logger.warning(+ f"[{request_id}] WS upstream connect failed "+ f"(attempt {ws_attempt + 1}/{ws_connect_attempts}): {ws_err}; "+ f"retrying in {delay_with_jitter:.0f}ms")+ await asyncio.sleep(delay_with_jitter / 1000)- except Exception as ws_err:- if ws_connected:- # WS was established but broke mid-stream — re-raise- raise+ if not ws_connected:# WS upgrade failed (HTTP 500 from OpenAI is common).# Fall back to HTTP POST streaming and relay SSE events# back over the client WebSocket transparently.+ ws_err = ws_last_err or RuntimeError("unknown websocket connect failure")_ws_detail = str(ws_err)if hasattr(ws_err, "response"):resp_body = getattr(getattr(ws_err, "response", None), "body", b"")@@ -1785,11 +1852,22 @@ class OpenAIHandlerMixin:except (json.JSONDecodeError, TypeError):parsed = body- # Unwrap the response.create envelope if present- if isinstance(parsed.get("response"), dict):- http_body = parsed["response"]+ # Normalize WebSocket response.create payload into the HTTP request body.+ # Codex may send either:+ # 1. {"type":"response.create","response":{...}}+ # 2. {"type":"response.create", ...response fields...}+ if isinstance(parsed, dict) and isinstance(parsed.get("response"), dict):+ http_body = dict(parsed["response"])+ elif isinstance(parsed, dict):+ http_body = dict(parsed)+ if http_body.get("type") == "response.create":+ http_body.pop("type", None)else:- http_body = parsed+ http_body = body if isinstance(body, dict) else {}++ # Some clients include response-ish metadata that the HTTP endpoint rejects.+ if http_body.get("type") in {"response.create", "response"}:+ http_body.pop("type", None)# Ensure streaming is enabled so we get SSE eventshttp_body["stream"] = True@@ -1802,62 +1880,80 @@ class OpenAIHandlerMixin:logger.debug(f"[{request_id}] WS→HTTP fallback POST to {http_url}")try:- async with self.http_client.stream(- "POST",- http_url,- headers=http_headers,- json=http_body,- timeout=120.0,- ) as response:- if response.status_code != 200:- error_body = b""- async for chunk in response.aiter_bytes():- error_body += chunk- if len(error_body) > 2000:- break- error_text = error_body.decode("utf-8", errors="replace")- logger.error(- f"[{request_id}] WS→HTTP fallback got {response.status_code}: "- f"{error_text[:500]}"+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))+ for http_attempt in range(retry_attempts):+ try:+ async with self.http_client.stream(+ "POST",+ http_url,+ headers=http_headers,+ json=http_body,+ timeout=120.0,+ ) as response:+ if response.status_code != 200:+ error_body = b""+ async for chunk in response.aiter_bytes():+ error_body += chunk+ if len(error_body) > 2000:+ break+ error_text = error_body.decode("utf-8", errors="replace")+ logger.error(+ f"[{request_id}] WS→HTTP fallback got {response.status_code}: "+ f"{error_text[:500]}"+ )+ # Send error as WS message so client sees it+ error_event = {+ "type": "error",+ "error": {+ "type": "server_error",+ "message": f"Upstream returned {response.status_code}",+ },+ }+ await websocket.send_text(json.dumps(error_event))+ return++ # Relay SSE events as WS text messages+ buffer = ""+ async for chunk in response.aiter_text():+ buffer += chunk+ while "\n" in buffer:+ line, buffer = buffer.split("\n", 1)+ line = line.strip()+ if not line:+ continue+ if line.startswith("data: "):+ data = line[6:]+ if data == "[DONE]":+ continue+ try:+ await websocket.send_text(data)+ except Exception:+ return+ elif line.startswith("event: "):+ # SSE event type — skip, the data line contains the type+ continue++ # Flush any remaining data in buffer+ for line in buffer.strip().splitlines():+ line = line.strip()+ if line.startswith("data: ") and line[6:] != "[DONE]":+ with contextlib.suppress(Exception):+ await websocket.send_text(line[6:])+ return+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err:+ if http_attempt >= retry_attempts - 1:+ raise++ delay_with_jitter = min(+ self.config.retry_base_delay_ms * (2**http_attempt),+ self.config.retry_max_delay_ms,+ ) * (0.5 + random.random())+ logger.warning(+ f"[{request_id}] WS→HTTP fallback connect failed "+ f"(attempt {http_attempt + 1}/{retry_attempts}): {http_err}; "+ f"retrying in {delay_with_jitter:.0f}ms")- # Send error as WS message so client sees it- error_event = {- "type": "error",- "error": {- "type": "server_error",- "message": f"Upstream returned {response.status_code}",- },- }- await websocket.send_text(json.dumps(error_event))- return-- # Relay SSE events as WS text messages- buffer = ""- async for chunk in response.aiter_text():- buffer += chunk- while "\n" in buffer:- line, buffer = buffer.split("\n", 1)- line = line.strip()- if not line:- continue- if line.startswith("data: "):- data = line[6:]- if data == "[DONE]":- continue- try:- await websocket.send_text(data)- except Exception:- return- elif line.startswith("event: "):- # SSE event type — skip, the data line contains the type- continue-- # Flush any remaining data in buffer- for line in buffer.strip().splitlines():- line = line.strip()- if line.startswith("data: ") and line[6:] != "[DONE]":- with contextlib.suppress(Exception):- await websocket.send_text(line[6:])+ await asyncio.sleep(delay_with_jitter / 1000)except Exception as http_err:logger.error(f"[{request_id}] WS→HTTP fallback failed: {http_err}")diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 3bb3f0b45..046035d68 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -5,9 +5,11 @@ Contains SSE parsing, streaming response generation, and related utilities.from __future__ import annotations+import asyncioimport contextlibimport jsonimport logging+import randomimport timefrom datetime import datetimefrom typing import TYPE_CHECKING, Any@@ -493,10 +495,37 @@ class StreamingMixin:# (needed to forward ratelimit headers to the client via StreamingResponse)assert self.http_client is not None, "http_client must be initialized before streaming"try:- _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)- upstream_response = await self.http_client.send(_upstream_req, stream=True)+ retry_attempts = max(1, getattr(self.config, "retry_max_attempts", 3))+ upstream_response = None+ last_connect_error = None++ for attempt in range(retry_attempts):+ try:+ _upstream_req = self.http_client.build_request(+ "POST", url, json=body, headers=headers+ )+ upstream_response = await self.http_client.send(_upstream_req, stream=True)+ break+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ last_connect_error = e+ if attempt >= retry_attempts - 1:+ raise++ delay_with_jitter = min(+ self.config.retry_base_delay_ms * (2**attempt),+ self.config.retry_max_delay_ms,+ ) * (0.5 + random.random())+ logger.warning(+ f"[{request_id}] Connection error to upstream API "+ f"(attempt {attempt + 1}/{retry_attempts}): {e!r}; "+ f"retrying in {delay_with_jitter:.0f}ms"+ )+ await asyncio.sleep(delay_with_jitter / 1000)++ if upstream_response is None:+ raise last_connect_error or RuntimeError("upstream connection did not start")except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:- error_msg = str(e)+ error_msg = str(e) or repr(e)logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")async def _error_gen():diff --git a/tests/test_openai_codex_routing.py b/tests/test_openai_codex_routing.pyindex 46934b8df..62eb5e327 100644--- a/tests/test_openai_codex_routing.py+++ b/tests/test_openai_codex_routing.py@@ -104,7 +104,7 @@ class _DummyMetrics:async def record_request(self, **kwargs): # noqa: ANN003return None- async def record_failed(self):+ async def record_failed(self, **kwargs): # noqa: ANN003return None@@ -128,24 +128,58 @@ class _DummyOpenAIHandler(OpenAIHandlerMixin):def __init__(self) -> None:self.rate_limiter = Noneself.metrics = _DummyMetrics()- self.config = SimpleNamespace(optimize=False)+ self.config = SimpleNamespace(+ optimize=False,+ retry_max_attempts=3,+ retry_base_delay_ms=10,+ retry_max_delay_ms=50,+ connect_timeout_seconds=10,+ )self.usage_reporter = None- self.openai_provider = SimpleNamespace()+ self.openai_provider = SimpleNamespace(get_context_limit=lambda model: 128_000)+ self.openai_pipeline = SimpleNamespace(apply=MagicMock())self.anthropic_backend = Noneself.cost_tracker = Noneself.memory_handler = Noneself.captured_request: tuple[str, str, dict, dict] | None = None+ self.captured_stream_request: tuple[str, dict, dict] | None = Noneasync def _next_request_id(self) -> str:return "req-1"- def _extract_tags(self, headers: dict[str, str]) -> list[str]:- return []+ def _extract_tags(self, headers: dict[str, str]) -> dict[str, str]:+ return {}async def _retry_request(self, method: str, url: str, headers: dict, body: dict):self.captured_request = (method, url, headers, body)return _ResponseStub()+ async def _stream_response(+ self,+ url: str,+ headers: dict,+ body: dict,+ provider: str,+ model: str,+ request_id: str,+ original_tokens: int,+ optimized_tokens: int,+ tokens_saved: int,+ transforms_applied: list[str],+ tags: dict[str, str],+ optimization_latency: float,+ memory_user_id: str | None = None,+ **kwargs,+ ):+ self.captured_stream_request = (url, headers, body)+ return SimpleNamespace(+ status_code=200,+ url=url,+ headers=headers,+ body=body,+ memory_user_id=memory_user_id,+ )+def _build_request(body: dict, headers: dict[str, str]) -> Request:payload = json.dumps(body).encode("utf-8")@@ -198,6 +232,79 @@ def test_handle_openai_responses_routes_chatgpt_auth_to_backend_api(monkeypatch)assert response.status_code == 200+def test_handle_openai_responses_stream_keeps_compression(monkeypatch):+ request = _build_request(+ {+ "model": "gpt-5.4",+ "stream": True,+ "instructions": "Keep it short",+ "input": [+ {+ "type": "message",+ "role": "user",+ "content": [{"type": "input_text", "text": "hello"}],+ }+ ],+ },+ {"Authorization": "Bearer sk-test"},+ )+ handler = _DummyOpenAIHandler()+ handler.config.optimize = True+ handler.openai_pipeline.apply.return_value = SimpleNamespace(+ messages=[+ {"role": "system", "content": "Keep it short"},+ {"role": "user", "content": "hello"},+ ],+ transforms_applied=[],+ tokens_before=2,+ tokens_after=2,+ )++ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())++ response = anyio.run(handler.handle_openai_responses, request)++ assert response.status_code == 200+ assert handler.captured_stream_request is not None+ assert handler.openai_pipeline.apply.call_count == 1+ assert handler.captured_stream_request[2]["stream"] is True+++def test_handle_openai_responses_memory_timeout_fails_open(monkeypatch):+ class _SlowMemoryHandler:+ def __init__(self):+ self.config = SimpleNamespace(inject_context=True, inject_tools=False)++ async def search_and_format_context(self, memory_user_id, messages):+ return "should not be used"++ def has_memory_tool_calls(self, response, provider):+ return False++ async def _timeout_wait_for(awaitable, timeout):+ close = getattr(awaitable, "close", None)+ if callable(close):+ close()+ raise TimeoutError++ request = _build_request(+ {"model": "gpt-5.4", "input": "hello"},+ {"Authorization": "Bearer sk-test", "x-headroom-user-id": "user-1"},+ )+ handler = _DummyOpenAIHandler()+ handler.memory_handler = _SlowMemoryHandler()++ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())+ monkeypatch.setattr("headroom.proxy.handlers.openai.asyncio.wait_for", _timeout_wait_for)++ response = anyio.run(handler.handle_openai_responses, request)++ assert response.status_code == 200+ assert handler.captured_request is not None+ _, _, _, body = handler.captured_request+ assert body.get("instructions") is None++class _DummyWebSocket:def __init__(self, headers: dict[str, str]):self.headers = headersdiff --git a/tests/test_proxy_streaming_ratelimit_headers.py b/tests/test_proxy_streaming_ratelimit_headers.pyindex 6c1d306b3..3fbcc377e 100644--- a/tests/test_proxy_streaming_ratelimit_headers.py+++ b/tests/test_proxy_streaming_ratelimit_headers.py@@ -24,6 +24,9 @@ class TestStreamingRatelimitHeaderForwarding:"""Create a HeadroomProxy with mocked internals for unit testing."""proxy = object.__new__(HeadroomProxy)proxy.http_client = MagicMock(spec=httpx.AsyncClient)+ proxy.metrics = MagicMock()+ proxy.metrics.record_request = AsyncMock(return_value=None)+ proxy.metrics.record_failed = AsyncMock(return_value=None)proxy.cost_tracker = MagicMock()proxy.cost_tracker.estimate_cost.return_value = 0.001proxy.cost_tracker.record_request.return_value = None@@ -40,6 +43,10 @@ class TestStreamingRatelimitHeaderForwarding:proxy._config = MagicMock()proxy._config.memory_enabled = Falseproxy._config.ccr_inject_tool = False+ proxy._config.retry_max_attempts = 3+ proxy._config.retry_base_delay_ms = 0+ proxy._config.retry_max_delay_ms = 0+ proxy.config = proxy._configproxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)proxy.memory_handler = Nonereturn proxy@@ -241,3 +248,48 @@ class TestStreamingRatelimitHeaderForwarding:error_data = json.loads(raw.split("data: ")[1].strip())assert error_data["error"]["type"] == "connection_error"assert "Connection refused" in error_data["error"]["message"]++ @pytest.mark.asyncio+ async def test_connect_timeout_retries_before_returning_stream(self):+ """Transient connect timeouts should retry before failing the stream."""+ proxy = self._create_mock_proxy()+ mock_response = self._create_mock_upstream_response()++ mock_request = MagicMock()+ proxy.http_client.build_request = MagicMock(return_value=mock_request)+ attempts = {"count": 0}++ async def flaky_send(*args, **kwargs):+ attempts["count"] += 1+ if attempts["count"] == 1:+ raise httpx.ConnectTimeout("timed out")+ return mock_response++ proxy.http_client.send = AsyncMock(side_effect=flaky_send)++ result = await proxy._stream_response(+ url="https://api.anthropic.com/v1/messages",+ headers={"x-api-key": "sk-test"},+ body={+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 100,+ "stream": True,+ "messages": [{"role": "user", "content": "hi"}],+ },+ provider="anthropic",+ model="claude-sonnet-4-20250514",+ request_id="test-retry",+ original_tokens=10,+ optimized_tokens=10,+ tokens_saved=0,+ transforms_applied=[],+ tags={},+ optimization_latency=0.0,+ )++ chunks = []+ async for chunk in result.body_iterator:+ chunks.append(chunk)++ assert attempts["count"] == 2+ assert chunksdiff --git a/tests/test_ws_http_fallback.py b/tests/test_ws_http_fallback.pyindex 47e40b800..ac2ffe23e 100644--- a/tests/test_ws_http_fallback.py+++ b/tests/test_ws_http_fallback.py@@ -9,6 +9,9 @@ from __future__ import annotationsimport asyncioimport json+from types import SimpleNamespace++import httpxclass FakeWebSocket:@@ -63,6 +66,11 @@ def _make_handler():obj = object.__new__(OpenAIHandlerMixin)obj.OPENAI_API_URL = "https://api.openai.com"obj.http_client = None+ obj.config = SimpleNamespace(+ retry_max_attempts=3,+ retry_base_delay_ms=0,+ retry_max_delay_ms=0,+ )return obj@@ -160,6 +168,27 @@ class TestWsHttpFallback:assert posted["stream"] is Trueassert "input" in posted+ def test_fallback_strips_top_level_response_create_type(self):+ """HTTP fallback should strip top-level response.create metadata."""+ handler = _make_handler()+ ws = FakeWebSocket()+ captured_kwargs: dict = {}++ class CapturingClient:+ def stream(self, method, url, **kwargs):+ captured_kwargs.update(kwargs)+ return FakeStreamResponse(200, ["data: [DONE]\n\n"])++ handler.http_client = CapturingClient()++ body = {"type": "response.create", "model": "gpt-5.4", "input": "hi"}+ asyncio.run(handler._ws_http_fallback(ws, body, json.dumps(body), {}, "req_type_strip"))++ posted = captured_kwargs["json"]+ assert posted["model"] == "gpt-5.4"+ assert posted["stream"] is True+ assert "type" not in posted+def test_fallback_handles_http_exception(self):"""HTTP fallback should send error event when HTTP request fails."""handler = _make_handler()@@ -179,6 +208,28 @@ class TestWsHttpFallback:assert event["type"] == "error"assert "unreachable" in event["error"]["message"]+ def test_fallback_retries_connect_timeout(self):+ """HTTP fallback should retry transient connect timeouts."""+ handler = _make_handler()+ ws = FakeWebSocket()+ attempts = {"count": 0}++ class FlakyClient:+ def stream(self, method, url, **kwargs):+ attempts["count"] += 1+ if attempts["count"] == 1:+ raise httpx.ConnectTimeout("timed out")+ return FakeStreamResponse(200, ['data: {"type":"response.completed"}\n\n'])++ handler.http_client = FlakyClient()++ body = {"model": "gpt-5.4", "input": "test"}+ asyncio.run(handler._ws_http_fallback(ws, body, json.dumps(body), {}, "req_retry"))++ assert attempts["count"] == 2+ assert len(ws.sent_texts) == 1+ assert json.loads(ws.sent_texts[0])["type"] == "response.completed"+def test_fallback_routes_chatgpt_auth_to_chatgpt_domain(self):"""ChatGPT session auth should route to chatgpt.com, not api.openai.com."""handler = _make_handler()