Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): adapt 200 SSE upstream replies on buffered /v1/responses instead of 502 (#2622)
Pragadeeshd76fce04a39b
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 4da3de6a6..8e38dce35 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -1182,6 +1182,47 @@ def _openai_responses_to_sse(response: dict[str, Any]) -> list[bytes]:return events+def _openai_responses_from_sse(sse_text: str) -> dict[str, Any] | None:+ """Reassemble the final Responses API JSON body from an SSE stream.++ Inverse of ``_openai_responses_to_sse``, for upstreams that answer a+ ``stream: false`` request with a valid ``200 text/event-stream`` body+ (#2613). The terminal ``response.completed`` event carries the complete+ response object, so no delta accumulation is needed. Returns ``None``+ when no terminal event is present — the caller forwards the raw body+ unchanged in that case.+ """+ completed: dict[str, Any] | None = None+ data_lines: list[str] = []++ def _consume(lines: list[str]) -> None:+ nonlocal completed+ if not lines:+ return+ data_str = "\n".join(lines)+ if data_str == "[DONE]":+ return+ try:+ data = json.loads(data_str)+ except json.JSONDecodeError:+ return+ if isinstance(data, dict) and data.get("type") == "response.completed":+ response = data.get("response")+ if isinstance(response, dict):+ completed = response++ for raw_line in sse_text.split("\n"):+ line = raw_line.rstrip("\r")+ if not line:+ _consume(data_lines)+ data_lines = []+ elif line.startswith("data:"):+ # Per the SSE spec, strip at most one leading space after the colon.+ data_lines.append(line[5:].removeprefix(" "))+ _consume(data_lines)+ return completed++def _output_shaping_holdout_fraction() -> float:from headroom.proxy import runtime_env@@ -5674,6 +5715,45 @@ class OpenAIHandlerMixin:total_latency = (time.time() - start_time) * 1000+ # Some OpenAI-compatible upstreams answer a ``stream: false``+ # request with a valid 200 SSE body carrying Responses API+ # events (#2613). Reassemble the terminal response JSON so+ # the buffered path keeps working — previously+ # ``response.json()`` raised JSONDecodeError past the+ # narrow usage-extraction catch below and the successful+ # upstream reply surfaced as a 502 proxy_error.+ upstream_content_type = response.headers.get("content-type", "")+ if (+ response.status_code == 200+ and "text/event-stream" in upstream_content_type.lower()+ ):+ completed_json = _openai_responses_from_sse(response.text)+ if completed_json is None:+ # No terminal event to adapt — forward the+ # successful upstream body unchanged rather than+ # fabricating a 502.+ return Response(+ content=response.content,+ status_code=response.status_code,+ headers=_sanitize_forwarded_response_headers(response.headers),+ )+ adapted_headers = {+ k: v+ for k, v in response.headers.items()+ if k.lower()+ not in (+ "content-type",+ "content-length",+ "content-encoding",+ "transfer-encoding",+ )+ }+ response = httpx.Response(+ status_code=response.status_code,+ content=json.dumps(completed_json).encode(),+ headers={**adapted_headers, "content-type": "application/json"},+ )+total_input_tokens = original_tokens # fallbackoutput_tokens = 0cache_read_tokens = 0diff --git a/tests/test_openai_codex_routing.py b/tests/test_openai_codex_routing.pyindex 16a44ea1b..1666184f8 100644--- a/tests/test_openai_codex_routing.py+++ b/tests/test_openai_codex_routing.py@@ -468,6 +468,74 @@ def test_handle_openai_responses_routes_api_key_auth_direct_to_openai(monkeypatcassert response.status_code == 200+def test_handle_openai_responses_non_stream_adapts_sse_upstream(monkeypatch):+ """A ``stream: false`` request whose upstream replies ``200+ text/event-stream`` must be adapted to the terminal response JSON, not+ converted into a 502 proxy_error (#2613)."""+ import httpx++ sse = (+ b"event: response.completed\n"+ b'data: {"type":"response.completed","response":{"id":"resp_sse_repro",'+ b'"output":[],"usage":{"input_tokens":2,"output_tokens":1}}}\n\n'+ )++ class _SSEUpstreamHandler(_DummyOpenAIHandler):+ async def _retry_request(self, method, url, headers, body, **kwargs):+ self.captured_request = (method, url, headers, body)+ return httpx.Response(+ 200,+ headers={"content-type": "text/event-stream"},+ content=sse,+ )++ request = _build_request(+ {"model": "gpt-5.4", "stream": False, "input": "hello"},+ {"Authorization": "Bearer sk-test"},+ )+ handler = _SSEUpstreamHandler()++ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())++ response = anyio.run(handler.handle_openai_responses, request)++ assert response.status_code == 200, response.body+ payload = json.loads(response.body)+ assert payload["id"] == "resp_sse_repro"+ assert response.headers["content-type"].startswith("application/json")+++def test_handle_openai_responses_non_stream_passes_through_unparseable_sse(monkeypatch):+ """A 200 SSE upstream body with no recognizable terminal response event+ must be forwarded as-is — never converted into a 502 (#2613)."""+ import httpx++ sse = b"event: response.weird\ndata: not-json\n\n"++ class _SSEUpstreamHandler(_DummyOpenAIHandler):+ async def _retry_request(self, method, url, headers, body, **kwargs):+ self.captured_request = (method, url, headers, body)+ return httpx.Response(+ 200,+ headers={"content-type": "text/event-stream"},+ content=sse,+ )++ request = _build_request(+ {"model": "gpt-5.4", "stream": False, "input": "hello"},+ {"Authorization": "Bearer sk-test"},+ )+ handler = _SSEUpstreamHandler()++ monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())++ response = anyio.run(handler.handle_openai_responses, request)++ assert response.status_code == 200, response.body+ assert response.body == sse+ assert response.headers["content-type"] == "text/event-stream"++def test_handle_openai_responses_stream_skips_python_compression(monkeypatch):"""PR-C5: Python no longer compresses /v1/responses (Rust handles itnatively). The streaming forward path must still fire — only the