Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): preserve chatgpt responses streaming (#2012)
Ben Younesa617455f0242
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 133a5d58f..066337172 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -648,6 +648,23 @@ def _has_headroom_retrieve_tool_responses(tools: Any) -> bool:return False+def _should_buffer_openai_responses_stream_ccr(+ *,+ stream: bool,+ ccr_response_handler_enabled: bool,+ tools: Any,+ is_chatgpt_auth: bool,+) -> bool:+ """Return whether streaming Responses CCR should use buffered JSON mode."""++ return bool(+ stream+ and ccr_response_handler_enabled+ and not is_chatgpt_auth+ and _has_headroom_retrieve_tool_responses(tools)+ )++def _responses_input_to_items(input_data: Any) -> list[dict[str, Any]]:"""Normalize a Responses ``input`` field into an item list for CCR continuation.@@ -4206,10 +4223,11 @@ class OpenAIHandlerMixin:_ccr_response_handler_enabled = bool(_ccr_response_handler and getattr(_ccr_handler_config, "enabled", True))- buffered_stream_ccr = bool(- stream- and _ccr_response_handler_enabled- and _has_headroom_retrieve_tool_responses(body.get("tools"))+ buffered_stream_ccr = _should_buffer_openai_responses_stream_ccr(+ stream=stream,+ ccr_response_handler_enabled=_ccr_response_handler_enabled,+ tools=body.get("tools"),+ is_chatgpt_auth=is_chatgpt_auth,)if buffered_stream_ccr:if body.get("stream") is not False:diff --git a/tests/test_proxy_openai_responses_stream_ccr.py b/tests/test_proxy_openai_responses_stream_ccr.pynew file mode 100644index 000000000..6102346cd--- /dev/null+++ b/tests/test_proxy_openai_responses_stream_ccr.py@@ -0,0 +1,36 @@+from __future__ import annotations++from headroom.ccr import CCR_TOOL_NAME+from headroom.proxy.handlers.openai import _should_buffer_openai_responses_stream_ccr++_TOOL_TYPE_FUNCTION = "function"+_UNRELATED_TOOL_NAME = "unrelated_tool"+++def _ccr_tool() -> dict[str, str]:+ return {"type": _TOOL_TYPE_FUNCTION, "name": CCR_TOOL_NAME}+++def _unrelated_tool() -> dict[str, str]:+ return {"type": _TOOL_TYPE_FUNCTION, "name": _UNRELATED_TOOL_NAME}+++def _should_buffer(*, tools: list[dict[str, str]], is_chatgpt_auth: bool) -> bool:+ return _should_buffer_openai_responses_stream_ccr(+ stream=True,+ ccr_response_handler_enabled=True,+ tools=tools,+ is_chatgpt_auth=is_chatgpt_auth,+ )+++def test_responses_ccr_buffers_streaming_openai_requests() -> None:+ assert _should_buffer(tools=[_ccr_tool()], is_chatgpt_auth=False)+++def test_responses_ccr_keeps_chatgpt_oauth_requests_streaming() -> None:+ assert not _should_buffer(tools=[_ccr_tool()], is_chatgpt_auth=True)+++def test_responses_ccr_ignores_requests_without_retrieve_tool() -> None:+ assert not _should_buffer(tools=[_unrelated_tool()], is_chatgpt_auth=False)