Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): preserve chatgpt responses streaming (#2012)

Ben Younesa617455f0242
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 133a5d58f..066337172 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -648,6 +648,23 @@ def _has_headroom_retrieve_tool_responses(tools: Any) -> bool:
return False
+def _should_buffer_openai_responses_stream_ccr(
+ *,
+ stream: bool,
+ ccr_response_handler_enabled: bool,
+ tools: Any,
+ is_chatgpt_auth: bool,
+) -> bool:
+ """Return whether streaming Responses CCR should use buffered JSON mode."""
+
+ return bool(
+ stream
+ and ccr_response_handler_enabled
+ and not is_chatgpt_auth
+ and _has_headroom_retrieve_tool_responses(tools)
+ )
+
+
def _responses_input_to_items(input_data: Any) -> list[dict[str, Any]]:
"""Normalize a Responses ``input`` field into an item list for CCR continuation.
@@ -4206,10 +4223,11 @@ class OpenAIHandlerMixin:
_ccr_response_handler_enabled = bool(
_ccr_response_handler and getattr(_ccr_handler_config, "enabled", True)
)
- buffered_stream_ccr = bool(
- stream
- and _ccr_response_handler_enabled
- and _has_headroom_retrieve_tool_responses(body.get("tools"))
+ buffered_stream_ccr = _should_buffer_openai_responses_stream_ccr(
+ stream=stream,
+ ccr_response_handler_enabled=_ccr_response_handler_enabled,
+ tools=body.get("tools"),
+ is_chatgpt_auth=is_chatgpt_auth,
)
if buffered_stream_ccr:
if body.get("stream") is not False:
diff --git a/tests/test_proxy_openai_responses_stream_ccr.py b/tests/test_proxy_openai_responses_stream_ccr.py
new file mode 100644
index 000000000..6102346cd
--- /dev/null
+++ b/tests/test_proxy_openai_responses_stream_ccr.py
@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from headroom.ccr import CCR_TOOL_NAME
+from headroom.proxy.handlers.openai import _should_buffer_openai_responses_stream_ccr
+
+_TOOL_TYPE_FUNCTION = "function"
+_UNRELATED_TOOL_NAME = "unrelated_tool"
+
+
+def _ccr_tool() -> dict[str, str]:
+ return {"type": _TOOL_TYPE_FUNCTION, "name": CCR_TOOL_NAME}
+
+
+def _unrelated_tool() -> dict[str, str]:
+ return {"type": _TOOL_TYPE_FUNCTION, "name": _UNRELATED_TOOL_NAME}
+
+
+def _should_buffer(*, tools: list[dict[str, str]], is_chatgpt_auth: bool) -> bool:
+ return _should_buffer_openai_responses_stream_ccr(
+ stream=True,
+ ccr_response_handler_enabled=True,
+ tools=tools,
+ is_chatgpt_auth=is_chatgpt_auth,
+ )
+
+
+def test_responses_ccr_buffers_streaming_openai_requests() -> None:
+ assert _should_buffer(tools=[_ccr_tool()], is_chatgpt_auth=False)
+
+
+def test_responses_ccr_keeps_chatgpt_oauth_requests_streaming() -> None:
+ assert not _should_buffer(tools=[_ccr_tool()], is_chatgpt_auth=True)
+
+
+def test_responses_ccr_ignores_requests_without_retrieve_tool() -> None:
+ assert not _should_buffer(tools=[_unrelated_tool()], is_chatgpt_auth=False)