Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(ccr): answer headroom_retrieve on streaming chat completions (#4631)

SK Arin31dcaf41f7fa
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index 819328dad..4f8c00bb6 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -349,6 +349,7 @@ By default Headroom stores originals so the model can recover them via `headroom
|---|---|---|
| `--no-ccr` / `HEADROOM_NO_CCR` | CCR on | Disable retrieval markers **and** the injected `headroom_retrieve` tool. |
| `--lossless` / `HEADROOM_LOSSLESS` | `false` | Format-native lossless compaction only — no CCR marker, no retrieval tool. |
+| `--ccr-buffered-chat-streaming` / `HEADROOM_CCR_BUFFERED_CHAT_STREAMING` | `false` | Answer `headroom_retrieve` on streaming `/v1/chat/completions`. A turn carrying a redeemable marker runs as `stream: false` upstream and is re-emitted as `chat.completion.chunk` frames, so its text arrives all at once; other turns keep their real stream |
| `--no-ccr-proactive-expansion` | expansion on | Stop proactively re-expanding compressed content when the model appears to need it. |
### File-read handling
diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.py
index f9bcab143..c027d2c13 100644
--- a/headroom/ccr/response_handler.py
+++ b/headroom/ccr/response_handler.py
@@ -54,6 +54,20 @@ RESIDUAL_CCR_SKIPPED_MIXED = "skipped_mixed_tools"
RESIDUAL_CCR_ERROR = "error"
+def _openai_message_extras(message: dict[str, Any]) -> dict[str, Any]:
+ """The fields a provider put on a chat message besides the standard three.
+
+ Whoever replays the turn may have to send these back (DeepSeek's
+ ``reasoning_content``, OpenRouter's ``reasoning_details``) and gets rejected
+ if they went missing. Empty ones (``refusal: null``) stay behind.
+ """
+ return {
+ key: value
+ for key, value in message.items()
+ if key not in ("role", "content", "tool_calls") and value
+ }
+
+
@dataclass
class CCRToolResult:
"""Result of handling a CCR tool call."""
@@ -397,6 +411,7 @@ class CCRResponseHandler:
first = choices[0] if isinstance(choices, list) and choices else {}
message = first.get("message", {}) if isinstance(first, dict) else {}
return {
+ **_openai_message_extras(message),
"role": "assistant",
"content": message.get("content"),
"tool_calls": message.get("tool_calls"),
@@ -951,7 +966,8 @@ class StreamingCCRHandler:
response["usage"] = usage
return response
- def _openai_response_to_chunks(self, response: dict[str, Any]) -> list[bytes]:
+ @staticmethod
+ def _openai_response_to_chunks(response: dict[str, Any]) -> list[bytes]:
"""Split a non-streaming ``chat.completion`` body into SSE chunk frames.
A streaming client reads ``choices[].delta``, not ``choices[].message``.
@@ -968,10 +984,7 @@ class StreamingCCRHandler:
message = {}
finish_reason = choice.get("finish_reason") or "stop"
- base: dict[str, Any] = {"object": "chat.completion.chunk"}
- for key in ("id", "created", "model", "system_fingerprint"):
- if response.get(key) is not None:
- base[key] = response[key]
+ base = StreamingCCRHandler._openai_chunk_envelope(response)
def frame(delta: dict[str, Any], reason: str | None) -> bytes:
payload = {
@@ -982,6 +995,10 @@ class StreamingCCRHandler:
frames = [frame({"role": message.get("role") or "assistant"}, None)]
+ extras = _openai_message_extras(message)
+ if extras:
+ frames.append(frame(extras, None))
+
content = message.get("content")
if content:
frames.append(frame({"content": content}, None))
@@ -999,10 +1016,15 @@ class StreamingCCRHandler:
{
"tool_calls": [
{
+ # Spread first: gateways hang opaque fields
+ # off a tool call that must survive the
+ # round trip (Gemini's thought signatures).
+ **tool_call,
"index": index,
"id": tool_call.get("id", ""),
"type": tool_call.get("type", "function"),
"function": {
+ **function,
"name": function.get("name", ""),
"arguments": function.get("arguments", ""),
},
@@ -1016,6 +1038,31 @@ class StreamingCCRHandler:
frames.append(frame({}, finish_reason))
return frames
+ @staticmethod
+ def _openai_chunk_envelope(response: dict[str, Any]) -> dict[str, Any]:
+ """The fields every ``chat.completion.chunk`` frame of a reply repeats."""
+ envelope: dict[str, Any] = {"object": "chat.completion.chunk"}
+ for key in ("id", "created", "model", "system_fingerprint"):
+ if response.get(key) is not None:
+ envelope[key] = response[key]
+ return envelope
+
+ @staticmethod
+ def openai_chat_sse_frames(response: dict[str, Any], *, include_usage: bool) -> list[bytes]:
+ """Render a whole ``chat.completion`` body as the SSE a stream would be.
+
+ ``include_usage`` adds the trailing usage-only chunk (``choices: []``)
+ that ``stream_options.include_usage`` asks for.
+ """
+ frames = StreamingCCRHandler._openai_response_to_chunks(response)
+ usage = response.get("usage")
+ if include_usage and usage is not None:
+ envelope = StreamingCCRHandler._openai_chunk_envelope(response)
+ payload = {**envelope, "choices": [], "usage": usage}
+ frames.append(f"data: {json.dumps(payload)}\n\n".encode())
+ frames.append(b"data: [DONE]\n\n")
+ return frames
+
async def _response_to_sse(
self,
response: dict[str, Any],
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index b49702456..a18f6101d 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -528,6 +528,17 @@ def dashboard(port: int, no_open: bool) -> None:
"Env: HEADROOM_CCR_INLINE_RESOLVE."
),
)
+ "--ccr-buffered-chat-streaming",
+ is_flag=True,
+ envvar="HEADROOM_CCR_BUFFERED_CHAT_STREAMING",
+ help=(
+ "Answer headroom_retrieve on streaming /v1/chat/completions. A turn that "
+ "carries a redeemable marker runs as stream:false upstream and is "
+ "re-emitted as chat.completion.chunk frames, so that turn's text arrives "
+ "all at once. Off by default. Env: HEADROOM_CCR_BUFFERED_CHAT_STREAMING."
+ ),
+)
@click.option(
"--no-ccr-proactive-expansion",
is_flag=True,
@@ -1178,6 +1189,7 @@ def proxy(
no_ccr: bool,
lossless: bool,
ccr_inline_resolve: bool,
+ ccr_buffered_chat_streaming: bool,
no_ccr_proactive_expansion: bool,
proxy_extension: tuple[str, ...],
compressor: tuple[str, ...],
@@ -1518,6 +1530,7 @@ def proxy(
ccr_inject_marker=not no_ccr,
ccr_handle_responses=not no_ccr,
ccr_resolve_markers_inline=ccr_inline_resolve,
+ ccr_buffered_chat_streaming=ccr_buffered_chat_streaming,
lossless=lossless,
ccr_proactive_expansion=not no_ccr_proactive_expansion,
# Flatten repeat-flag tuple AND any comma-separated values inside it.
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index cd627c461..8d7dcc533 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -1399,16 +1399,171 @@ def _should_buffer_openai_responses_stream_ccr(
)
-def _should_inject_openai_chat_ccr_tool(*, ccr_inject_tool: bool, stream: bool) -> bool:
+def _should_inject_openai_chat_ccr_tool(
+ *, ccr_inject_tool: bool, stream: bool, buffered_eligible: bool = False
+) -> bool:
"""Return whether chat-completions can redeem an injected CCR tool.
- The chat streaming path forwards SSE events immediately and deliberately
- does not run the response continuation loop. Injecting ``headroom_retrieve``
- there makes OpenAI-compatible clients attempt an unknown tool call. The
+ A plain chat streaming turn forwards SSE events immediately and does not run
+ the response continuation loop, so injecting ``headroom_retrieve`` there
+ makes OpenAI-compatible clients attempt an unknown tool call. The
non-streaming path can intercept and resolve it; Responses streaming has a
separate buffered-CCR path and is unaffected by this predicate.
+
+ ``buffered_eligible`` is the third case: if this turn carries a marker it
+ will run the continuation loop before replying, so the tool is redeemable.
+ If it carries none, the model has no hash to call it with and the tool stays
+ resident only to keep the tools array byte-stable.
+ """
+ return bool(ccr_inject_tool and (buffered_eligible or not stream))
+
+
+def _is_openai_chat_stream_ccr_eligible(
+ *,
+ body: dict[str, Any],
+ enabled: bool,
+ ccr_response_handler_enabled: bool,
+ ccr_inject_tool: bool,
+ routed_to_backend: bool,
+ upstream_base_url: str | None = None,
+) -> bool:
+ """Return whether a streaming chat turn may answer retrievals by buffering.
+
+ Unlike :func:`_should_buffer_openai_responses_stream_ccr` this cannot key on
+ ``headroom_retrieve`` already being in ``tools``: on this path that test is
+ what decides injection, so it would never pass.
+
+ Requiring a non-empty client ``tools`` array keeps the rule from #3810: a
+ request with no tools is not an agent harness and must not be handed a tools
+ array it never sent. The rest are turns buffering cannot serve:
+
+ - A backend route streams through its own path, with no continuation loop.
+ - ``n > 1`` and ``logprobs`` ask for output that resynthesis does not carry.
+ - OpenCode Zen rejects requests Headroom has reshaped (``stream:false`` plus
+ ``accept: application/json``) with ``403 FreeTierError`` (#3656).
+ """
+ n = body.get("n")
+ return bool(
+ body.get("stream")
+ and enabled
+ and ccr_response_handler_enabled
+ and ccr_inject_tool
+ and body.get("tools")
+ and not routed_to_backend
+ and not (isinstance(n, int) and n > 1)
+ and not body.get("logprobs")
+ and not is_opencode_zen_base(upstream_base_url)
+ )
+
+
+def _chat_body_has_ccr_marker(body: dict[str, Any]) -> bool:
+ """Is there anything in the outgoing messages a resident tool could be called for?
+
+ Goes by marker shape alone, without the ownership check that decides
+ injection (#2836). An expired hash can still draw a retrieval call once the
+ tool is resident, and only the continuation loop can answer that call.
+ """
+ from headroom.ccr.tool_injection import CCRToolInjector
+
+ messages = body.get("messages")
+ if not isinstance(messages, list):
+ return False
+ probe = CCRToolInjector(provider="openai", inject_tool=False, inject_system_instructions=False)
+ try:
+ return bool(probe.scan_for_markers(messages))
+ except Exception: # pragma: no cover - defensive
+ logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)
+ return True
+
+
+def _take_stream_options_for_buffered_chat(body: dict[str, Any]) -> bool:
+ """Drop ``stream_options`` from a body going out as ``stream: false``.
+
+ OpenAI answers 400 to ``stream_options`` on a non-streaming request. Returns
+ whether the client should still get the usage chunk, decided by the same
+ rule a plain streamed turn uses so the two cannot drift apart.
+ """
+ _apply_stream_usage_option(body)
+ stream_options = body.pop("stream_options", None)
+ return isinstance(stream_options, dict) and bool(stream_options.get("include_usage"))
+
+
+def _buffered_chat_reply_as_sse(
+ result: Response,
+ *,
+ ccr_response_handler: Any,
+ include_usage: bool,
+ continuation_status: int | None,
+ request_id: str,
+) -> Response:
+ """Turn the direct chat path's JSON reply into what a streaming client reads.
+
+ Anything but a 200 goes back untouched. Inside the grace window that hands
+ the client the real status, and past it :func:`buffered_ccr_asgi_call` turns
+ it into a typed stream error (#2997, #3079).
"""
- return bool(ccr_inject_tool and not stream)
+ from fastapi.responses import JSONResponse, StreamingResponse
+
+ from headroom.ccr.response_handler import RESIDUAL_CCR_ERROR, StreamingCCRHandler
+
+ if result.status_code != 200:
+ return result
+
+ try:
+ reply = json.loads(result.body)
+ except (TypeError, ValueError):
+ reply = None
+ choices = reply.get("choices") if isinstance(reply, dict) else None
+ first_choice = choices[0] if isinstance(choices, list) and choices else None
+ if not isinstance(first_choice, dict) or not isinstance(first_choice.get("message"), dict):
+ logger.warning(f"[{request_id}] CCR: rejecting malformed buffered chat 200 reply")
+ return JSONResponse(
+ status_code=502,
+ content={
+ "error": {
+ "type": "upstream_protocol_error",
+ "message": "Upstream returned an invalid buffered response.",
+ }
+ },
+ )
+
+ sse_headers = {
+ k: v
+ for k, v in result.headers.items()
+ if k.lower() not in ("content-length", "content-type")
+ }
+
+ # A retrieval call standing alone means the continuation failed, and the
+ # client cannot act on a tool it never declared. Next to a client tool it is
+ # the turn handle_response gives back when a continuation fails, because the
+ # model made no newer decision, so that goes to the client as is. A plain JSON
+ # error works on both sides of the commit: the wrapper turns it into a typed
+ # stream error if SSE is already on the wire, so a rate limit on the
+ # continuation still reads as a rate limit.
+ if ccr_response_handler.residual_ccr_status(reply, "openai") == RESIDUAL_CCR_ERROR:
+ status = continuation_status or 502
+ logger.warning(
+ f"[{request_id}] CCR: buffered chat reply still contains headroom_retrieve "
+ f"after handling; failing closed with {status}"
+ )
+ return JSONResponse(
+ status_code=status,
+ headers=sse_headers,
+ content={
+ "error": {
+ "message": "Unable to safely complete streamed CCR retrieval.",
+ "type": OPENAI_ERROR_FORMAT.error_type_for_status(status),
+ }
+ },
+ )
+
+ frames = StreamingCCRHandler.openai_chat_sse_frames(reply, include_usage=include_usage)
+
+ async def _frames():
+ for frame in frames:
+ yield frame
+
+ return StreamingResponse(_frames(), media_type="text/event-stream", headers=sse_headers)
def _responses_input_to_items(input_data: Any) -> list[dict[str, Any]]:
@@ -4237,14 +4392,34 @@ class OpenAIHandlerMixin:
# anchored on the previous turn's tool list never busts.
tools = body.get("tools")
_original_tools = tools # Preserve for diagnostic / future retry
+ # Decided before injection: a turn that can buffer can answer the tool,
+ # so it may carry it even though the client asked for a stream. Whether
+ # it actually buffers is settled at dispatch, from the final body.
+ from headroom.proxy.route_advice import advice_from
+
+ _ccr_handler = getattr(self, "ccr_response_handler", None)
+ chat_stream_ccr_eligible = _is_openai_chat_stream_ccr_eligible(
+ body=body,
+ enabled=self.config.ccr_buffered_chat_streaming,
+ ccr_response_handler_enabled=bool(
+ _ccr_handler and getattr(_ccr_handler.config, "enabled", True)
+ ),
+ ccr_inject_tool=self.config.ccr_inject_tool,
+ routed_to_backend=(
+ getattr(self, "anthropic_backend", None) is not None
+ or advice_from(request) is not None
+ ),
+ upstream_base_url=upstream_base_url,
+ )
can_inject_ccr_tool = _should_inject_openai_chat_ccr_tool(
ccr_inject_tool=self.config.ccr_inject_tool,
stream=stream,
+ buffered_eligible=chat_stream_ccr_eligible,
)
if (
self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
) and not _bypass:
- if self.config.ccr_inject_tool and stream:
+ if self.config.ccr_inject_tool and stream and not chat_stream_ccr_eligible:
logger.info(
f"[{request_id}] CCR: skipping retrieval-tool injection for "
"OpenAI chat streaming; this path cannot intercept tool calls"
@@ -4253,7 +4428,8 @@ class OpenAIHandlerMixin:
provider="openai",
inject_tool=False, # routed through sticky helper below
inject_system_instructions=(
- self.config.ccr_inject_system_instructions and not stream
+ self.config.ccr_inject_system_instructions
+ and (not stream or chat_stream_ccr_eligible)
),
)
injector.scan_for_markers(optimized_messages)
@@ -4263,7 +4439,7 @@ class OpenAIHandlerMixin:
injector.verify_ownership()
if (
self.config.ccr_inject_system_instructions
- and not stream
+ and (not stream or chat_stream_ccr_eligible)
and injector.has_compressed_content
):
optimized_messages = injector.inject_into_system_message(
@@ -5175,8 +5351,31 @@ class OpenAIHandlerMixin:
)
url = _append_request_query(url, request.url.query)
+ # The tool stays resident for the session so the tools array is
+ # byte-stable, but the model has nothing to call it with unless this body
+ # carries a marker. A turn with none keeps its real stream: buffering it
+ # would make time-to-first-byte the whole generation for nothing (#3071).
+ # Checking for the tool too keeps a bypassed turn, which got none, streaming.
+ buffered_chat_ccr = (
+ chat_stream_ccr_eligible
+ and has_ccr_retrieve_tool(body.get("tools"))
+ and _chat_body_has_ccr_marker(body)
+ )
+ buffered_chat_include_usage = False
+ buffered_chat_continuation_status: int | None = None
+ if buffered_chat_ccr:
+ buffered_chat_include_usage = _take_stream_options_for_buffered_chat(body)
+ body["stream"] = False
+ # A strict gateway rejects `stream: false` with `Accept: text/event-stream` (#3078).
+ _accept_key = next((k for k in headers if k.lower() == "accept"), "accept")
+ headers[_accept_key] = "application/json"
+ logger.info(
+ f"[{request_id}] CCR: buffering streaming chat turn "
+ "(stream:false upstream, resynthesized as chat.completion.chunk)"
+ )
+
try:
- if stream:
+ if stream and not buffered_chat_ccr:
# Request usage stats for accurate token counting instead of
# byte-based estimation — without overriding an explicit client
# choice (see _apply_stream_usage_option).
@@ -5210,7 +5409,12 @@ class OpenAIHandlerMixin:
original_messages=original_client_messages,
outcome_provider=openai_chat_outcome_provider,
)
- else:
+
+ async def _direct_chat_turn() -> Response:
+ # Usage, the prefix tracker, savings and CCR continuations all
+ # live in here, so a buffered turn is accounted for exactly like
+ # a non-streaming one instead of through a second copy.
+ nonlocal headers, buffered_chat_continuation_status
headers = await apply_copilot_api_auth(headers, url=url)
response = await self._retry_request("POST", url, headers, body)
@@ -5232,7 +5436,13 @@ class OpenAIHandlerMixin:
# hook actually calls the model again.
_hook_usage = TurnHookUsage()
- if _registered_turn_hooks() and response.status_code == 200:
+ # A buffered turn only ran stream-safe on_request hooks (that is
+ # decided before dispatch), so it skips their response half too.
+ if (
+ _registered_turn_hooks()
+ and response.status_code == 200
+ and not buffered_chat_ccr
+ ):
try:
_hook_resp_json = response.json()
except (ValueError, json.JSONDecodeError):
@@ -5512,6 +5722,7 @@ class OpenAIHandlerMixin:
# means on a path that changed nothing.
if _ccr_cont_error is not None or _ccr_final_json is _pre_ccr_json:
if _ccr_cont_error is not None:
+ buffered_chat_continuation_status = _ccr_cont_error
logger.warning(
f"[{request_id}] CCR: continuation returned HTTP "
f"{_ccr_cont_error}; forwarding the upstream reply "
@@ -5668,13 +5879,9 @@ class OpenAIHandlerMixin:
# the live (hooked) `messages` (#327).
#
# ``not stream`` mirrors the read gate at the cache lookup
- # above. It is currently redundant here — a streaming chat
- # request returns via ``_stream_response`` well before this
- # point — but the Anthropic handler had the same shape until a
- # buffered-CCR branch started falling through to its store
- # site, which let a response built for a stream:true request
- # answer a later non-streaming caller (#3019). Stating the
- # invariant keeps that from being reintroduced silently.
+ # above. A buffered turn reaches this point with ``stream`` still
+ # true, and a response built for a stream:true request must not
+ # answer a later non-streaming caller (#3019).
if (
self.cache
and not stream
@@ -5795,6 +6002,37 @@ class OpenAIHandlerMixin:
status_code=response.status_code,
headers=response_headers,
)
+
+ if not buffered_chat_ccr:
+ return await _direct_chat_turn()
+
+ async def _buffered_chat_turn() -> Response:
+ return _buffered_chat_reply_as_sse(
+ await _direct_chat_turn(),
+ ccr_response_handler=_ccr_handler,
+ include_usage=buffered_chat_include_usage,
+ continuation_status=buffered_chat_continuation_status,
+ request_id=request_id,
+ )
+
+ async def _record_buffered_failure(**_: Any) -> None:
+ # The shared wrapper only knows "openai"; keep the label this
+ # handler uses for xAI and custom upstreams.
+ await self.metrics.record_failed(provider=openai_chat_outcome_provider)
+
+ _buffered_call = buffered_ccr_asgi_call(
+ operation=asyncio.create_task(_buffered_chat_turn()),
+ fmt=OPENAI_ERROR_FORMAT,
+ grace_seconds=self.config.buffered_ccr_grace_seconds,
+ record_failed=_record_buffered_failure,
+ request_id=request_id,
+ )
+
+ class _BufferedCCRResponse(Response):
+ async def __call__(self, scope, receive, send): # noqa: ANN001
+ await _buffered_call(scope, receive, send)
+
+ return _BufferedCCRResponse(media_type="text/event-stream")
except Exception as e:
await self.metrics.record_failed(provider=openai_chat_outcome_provider)
# Error names the failure; text and traceback go to debug.
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 18bb2d364..894acfc6e 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -422,6 +422,16 @@ class ProxyConfig:
# stream-idle watchdog fires. 0 or less disables the keepalive entirely.
# See headroom/proxy/buffered_ccr_response.py (#3079).
buffered_ccr_grace_seconds: float = DEFAULT_BUFFERED_CCR_GRACE_SECONDS
+ # Buffered CCR for /v1/chat/completions streaming. Without it this is the
+ # one streaming path with no way to answer a retrieval: content carries
+ # markers but `headroom_retrieve` is withheld, and the tool toggles in and
+ # out for sessions that mix streaming with non-streaming turns, busting the
+ # tools cache segment each way. Enabling it flips turns that carry a
+ # redeemable marker to `stream: false` upstream and resynthesizes
+ # chat.completion.chunk frames, as Responses streaming already does. Off by
+ # default: this is the most incident-prone path in the proxy (#2465, #2997,
+ # #3079), so the default flips separately, with measurements (#3815).
+ ccr_buffered_chat_streaming: bool = False
# Connection pool
max_connections: int = 500
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 24c3d1a03..b3afc278c 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -6497,6 +6497,7 @@ def _proxy_config_from_env() -> ProxyConfig:
"HEADROOM_BUFFERED_CCR_GRACE_SECONDS",
DEFAULT_BUFFERED_CCR_GRACE_SECONDS,
),
+ ccr_buffered_chat_streaming=_get_env_bool("HEADROOM_CCR_BUFFERED_CHAT_STREAMING", False),
vertex_api_url=os.environ.get("VERTEX_TARGET_API_URL"),
backend=_get_env_str("HEADROOM_BACKEND", "anthropic"),
bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", "us-west-2"),
diff --git a/headroom/settings_store.py b/headroom/settings_store.py
index ffc0c73a3..014740f4b 100644
--- a/headroom/settings_store.py
+++ b/headroom/settings_store.py
@@ -165,6 +165,19 @@ SETTINGS: tuple[SettingField, ...] = (
),
tier="advanced",
),
+ SettingField(
+ "HEADROOM_CCR_BUFFERED_CHAT_STREAMING",
+ "ccr_buffered_chat_streaming",
+ "Buffered CCR for streaming chat",
+ "Compression",
+ "bool",
+ default=False,
+ help=(
+ "Answer retrieval calls on streaming Chat Completions by buffering "
+ "turns that carry CCR markers. Buffered text arrives all at once."
+ ),
+ tier="advanced",
+ ),
# --- Limits ---
SettingField(
"HEADROOM_RPM",
diff --git a/tests/test_cli_proxy_env.py b/tests/test_cli_proxy_env.py
index c47497ec0..33724d992 100644
--- a/tests/test_cli_proxy_env.py
+++ b/tests/test_cli_proxy_env.py
@@ -797,6 +797,27 @@ class TestCLIProxyEnvVars:
assert captured_config["config"].openai_api_url == "http://my-vllm:4000"
assert captured_config["config"].gemini_api_url == "http://my-gemini:5000"
+ @pytest.mark.parametrize(
+ ("args", "env", "expected"),
+ [
+ ([], {}, False),
+ (["--ccr-buffered-chat-streaming"], {}, True),
+ ([], {"HEADROOM_CCR_BUFFERED_CHAT_STREAMING": "1"}, True),
+ ],
+ )
+ def test_ccr_buffered_chat_streaming_reaches_config(self, runner, args, env, expected):
+ """`headroom proxy` builds its own ProxyConfig, so the flag has to be wired here too."""
+ captured_config = {}
+
+ def mock_run_server(config, **kwargs):
+ captured_config["config"] = config
+
+ with patch("headroom.proxy.server.run_server", mock_run_server):
+ result = runner.invoke(main, ["proxy", *args], env=env, catch_exceptions=False)
+
+ assert result.exit_code == 0, result.output
+ assert captured_config["config"].ccr_buffered_chat_streaming is expected
+
@pytest.mark.parametrize("timeout", [-1, 0, 1, 10000])
def test_request_timeout_cli_flags(self, runner, timeout):
"""Fast-fail CLI flags should map into ProxyConfig."""
diff --git a/tests/test_openai_chat_buffered_ccr.py b/tests/test_openai_chat_buffered_ccr.py
new file mode 100644
index 000000000..5cdffc0c7
--- /dev/null
+++ b/tests/test_openai_chat_buffered_ccr.py
@@ -0,0 +1,718 @@
+"""Buffered CCR on streaming /v1/chat/completions (#3815).
+
+Compression still runs on a streaming chat turn, so content carries retrieval
+markers, but this path had no way to answer ``headroom_retrieve``. With
+``ccr_buffered_chat_streaming`` on, a turn whose outgoing body carries a
+redeemable marker runs as ``stream: false`` upstream through the same code as a
+non-streaming turn, then goes back to the client as ``chat.completion.chunk``
+frames. Everything else keeps its real stream.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+httpx = pytest.importorskip("httpx")
+
+from fastapi.responses import Response # noqa: E402
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.cache.compression_store import ( # noqa: E402
+ get_compression_store,
+ reset_compression_store,
+)
+from headroom.ccr.response_handler import CCRResponseHandler, StreamingCCRHandler # noqa: E402
+from headroom.ccr.tool_injection import CCR_TOOL_NAME # noqa: E402
+from headroom.proxy.handlers.openai import ( # noqa: E402
+ _buffered_chat_reply_as_sse,
+ _is_openai_chat_stream_ccr_eligible,
+ _take_stream_options_for_buffered_chat,
+)
+from headroom.proxy.helpers import _reset_session_ccr_tracker_for_test # noqa: E402
+from headroom.proxy.server import ProxyConfig, _proxy_config_from_env, create_app # noqa: E402
+
+ORIGINAL = "row 1: the original uncompressed rows\nrow 2: more of them"
+CLIENT_TOOL = {"type": "function", "function": {"name": "Read", "description": "read a file"}}
+USAGE_FIRST = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}
+USAGE_FINAL = {"prompt_tokens": 400, "completion_tokens": 10, "total_tokens": 410}
+
+
[email protected](autouse=True)
+def reset_store():
+ reset_compression_store()
+ _reset_session_ccr_tracker_for_test()
+ yield
+ reset_compression_store()
+ _reset_session_ccr_tracker_for_test()
+
+
+def _stored_hash() -> str:
+ return get_compression_store().store(
+ original=ORIGINAL,
+ compressed="[2 items compressed to 0]",
+ original_item_count=2,
+ compressed_item_count=0,
+ )
+
+
+def _completion(message: dict[str, Any], finish_reason: str, usage: dict[str, int]) -> dict:
+ return {
+ "id": "chatcmpl-1",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "gpt-4o",
+ "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],
+ "usage": usage,
+ }
+
+
+def _tool_call(name: str, arguments: dict[str, Any], call_id: str = "call_1") -> dict:
+ return {
+ "id": call_id,
+ "type": "function",
+ "function": {"name": name, "arguments": json.dumps(arguments)},
+ }
+
+
+def _retrieve_reply(hash_key: str) -> dict:
+ message = {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [_tool_call(CCR_TOOL_NAME, {"hash": hash_key})],
+ }
+ return _completion(message, "tool_calls", USAGE_FIRST)
+
+
+def _final_reply() -> dict:
+ return _completion(
+ {"role": "assistant", "content": "there are 2 rows"},
+ "stop",
+ USAGE_FINAL,
+ )
+
+
+def _messages(hash_key: str | None) -> list[dict[str, Any]]:
+ content = "how many rows?"
+ if hash_key is not None:
+ content += f" [2 items compressed to 0. Retrieve more: hash={hash_key}]"
+ return [{"role": "user", "content": content}]
+
+
+def _request(hash_key: str | None, **overrides: Any) -> dict[str, Any]:
+ return {
+ "model": "gpt-4o",
+ "messages": _messages(hash_key),
+ "stream": True,
+ "tools": [CLIENT_TOOL],
+ **overrides,
+ }
+
+
+def _events(text: str) -> list[Any]:
+ """Parse an SSE body into its JSON payloads, keeping the [DONE] sentinel."""
+ events: list[Any] = []
+ for line in text.splitlines():
+ if line.startswith("data: "):
+ data = line[len("data: ") :]
+ events.append(data if data == "[DONE]" else json.loads(data))
+ return events
+
+
+def _chunks(text: str) -> list[dict[str, Any]]:
+ return [event for event in _events(text) if event != "[DONE]"]
+
+
+class _Proxy:
+ """A live app whose upstream calls are served from a script."""
+
+ def __init__(self, upstream: list[Any], **config: Any) -> None:
+ self._remaining = list(upstream)
+ self.sent: list[dict[str, Any]] = []
+ self.sent_headers: list[dict[str, str]] = []
+ self.outcomes: list[Any] = []
+ self.plain_streams = 0
+ settings = {
+ "optimize": False,
+ "cache_enabled": False,
+ "rate_limit_enabled": False,
+ "ccr_handle_responses": True,
+ "ccr_inject_tool": True,
+ "ccr_buffered_chat_streaming": True,
+ **config,
+ }
+ self._app = create_app(ProxyConfig(**settings))
+
+ async def _retry(self, method, url, headers, body, *args, **kwargs): # noqa: ANN001
+ self.sent.append(json.loads(json.dumps(body)))
+ self.sent_headers.append(dict(headers))
+ entry = self._remaining.pop(0)
+ if isinstance(entry, Exception):
+ raise entry
+ status, payload = entry if isinstance(entry, tuple) else (200, entry)
+ return httpx.Response(status, json=payload, headers={"content-type": "application/json"})
+
+ def __enter__(self) -> _Proxy:
+ self._client = TestClient(self._app)
+ self._client.__enter__()
+ proxy = self._client.app.state.proxy
+ proxy._retry_request = self._retry
+ self.proxy = proxy
+
+ async def _plain_stream(*args: Any, **kwargs: Any) -> Response:
+ self.plain_streams += 1
+ return Response(b"data: plain\n\n", media_type="text/event-stream")
+
+ proxy._stream_response = _plain_stream
+
+ real_record = proxy._record_request_outcome
+
+ async def _capture(outcome, *args, **kwargs): # noqa: ANN001
+ self.outcomes.append(outcome)
+ return await real_record(outcome, *args, **kwargs)
+
+ proxy._record_request_outcome = _capture
+ return self
+
+ def __exit__(self, *exc: Any) -> None:
+ self._client.__exit__(*exc)
+
+ def post(self, body: dict[str, Any], **extra_headers: str) -> httpx.Response:
+ return self._client.post(
+ "/v1/chat/completions",
+ json=body,
+ headers={
+ "Authorization": "Bearer test-key",
+ "x-api-key": "test-key",
+ "Accept": "text/event-stream",
+ **{name.replace("_", "-"): value for name, value in extra_headers.items()},
+ },
+ )
+
+
+def test_flag_off_keeps_the_plain_stream_and_withholds_the_tool():
+ hash_key = _stored_hash()
+ with _Proxy([], ccr_buffered_chat_streaming=False) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.text == "data: plain\n\n"
+ assert proxy.plain_streams == 1
+ assert proxy.sent == [], "flag off must not take the buffered path"
+
+
+def test_flag_defaults_off():
+ assert ProxyConfig().ccr_buffered_chat_streaming is False
+
+
+def test_env_var_turns_the_flag_on(monkeypatch):
+ monkeypatch.setenv("HEADROOM_CCR_BUFFERED_CHAT_STREAMING", "1")
+ assert _proxy_config_from_env().ccr_buffered_chat_streaming is True
+
+
+def test_retrieval_is_answered_and_the_client_reads_a_real_stream():
+ hash_key = _stored_hash()
+ with _Proxy([_retrieve_reply(hash_key), _final_reply()]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ assert resp.headers["content-type"].startswith("text/event-stream")
+ assert "content-length" not in resp.headers or int(resp.headers["content-length"]) == len(
+ resp.content
+ )
+ events = _events(resp.text)
+ assert events[-1] == "[DONE]"
+
+ chunks = _chunks(resp.text)
+ assert {chunk["object"] for chunk in chunks} == {"chat.completion.chunk"}
+ choices = [chunk for chunk in chunks if chunk["choices"]]
+ assert choices[0]["choices"][0]["delta"]["role"] == "assistant"
+ text = "".join(chunk["choices"][0]["delta"].get("content") or "" for chunk in choices)
+ assert text == "there are 2 rows"
+ assert choices[-1]["choices"][0]["finish_reason"] == "stop"
+ assert CCR_TOOL_NAME not in resp.text
+
+ first, continuation = proxy.sent
+ assert first["stream"] is False
+ assert "stream_options" not in first, "OpenAI answers 400 to stream_options without a stream"
+ assert CCR_TOOL_NAME in {tool["function"]["name"] for tool in first["tools"]}
+ assert ORIGINAL.splitlines()[0] in json.dumps(continuation)
+ assert {k.lower(): v for k, v in proxy.sent_headers[0].items()}["accept"] == "application/json"
+
+
+def test_a_turn_with_no_marker_keeps_its_real_stream():
+ """Buffering a turn retrieval cannot help only costs time to first byte (#3071)."""
+ with _Proxy([]) as proxy:
+ resp = proxy.post(_request(None))
+
+ assert resp.text == "data: plain\n\n"
+ assert proxy.plain_streams == 1
+ assert proxy.sent == []
+
+
+def test_a_foreign_marker_does_not_buffer():
+ """Another context tool can use the same marker shape; only owned hashes count (#2836)."""
+ with _Proxy([]) as proxy:
+ resp = proxy.post(_request("0123456789abcdef0123456789abcdef"))
+
+ assert resp.text == "data: plain\n\n"
+ assert proxy.sent == []
+
+
+def test_a_client_tool_call_still_ends_in_tool_calls():
+ hash_key = _stored_hash()
+ client_call = _tool_call("Read", {"file_path": "a.py"})
+ reply = _completion(
+ {"role": "assistant", "content": None, "tool_calls": [client_call]},
+ "tool_calls",
+ USAGE_FIRST,
+ )
+ with _Proxy([reply]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ choices = [chunk["choices"][0] for chunk in _chunks(resp.text) if chunk["choices"]]
+ assert choices[-1]["finish_reason"] == "tool_calls"
+ tool_deltas = [
+ choice["delta"]["tool_calls"][0] for choice in choices if "tool_calls" in choice["delta"]
+ ]
+ assert [delta["function"]["name"] for delta in tool_deltas] == ["Read"]
+ assert tool_deltas[0]["index"] == 0
+
+
+ ("stream_options", "expect_usage"),
+ [
+ (None, True),
+ ({"include_usage": True}, True),
+ ({"include_usage": False}, False),
+ ({}, True),
+ ],
+)
+def test_usage_chunk_follows_the_plain_stream_rule(stream_options, expect_usage):
+ hash_key = _stored_hash()
+ overrides = {} if stream_options is None else {"stream_options": stream_options}
+ with _Proxy([_final_reply()]) as proxy:
+ resp = proxy.post(_request(hash_key, **overrides))
+
+ usage_chunks = [chunk for chunk in _chunks(resp.text) if chunk.get("usage")]
+ if expect_usage:
+ assert len(usage_chunks) == 1
+ assert usage_chunks[0]["choices"] == []
+ assert usage_chunks[0]["usage"] == USAGE_FINAL
+ assert _chunks(resp.text)[-1] is not None and _chunks(resp.text)[-1]["usage"]
+ else:
+ assert usage_chunks == []
+ assert "stream_options" not in proxy.sent[0]
+
+
+def test_upstream_error_inside_the_grace_window_keeps_its_status():
+ """A 429 must reach the client as a 429, not a 200 with no frames (#2997)."""
+ hash_key = _stored_hash()
+ error = {"error": {"message": "slow down", "type": "rate_limit_error"}}
+ with _Proxy([(429, error)]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 429
+ assert resp.json()["error"]["message"] == "slow down"
+
+
+class _Slow(_Proxy):
+ async def _retry(self, method, url, headers, body, *args, **kwargs): # noqa: ANN001
+ import asyncio
+
+ await asyncio.sleep(0.05)
+ return await super()._retry(method, url, headers, body, *args, **kwargs)
+
+
+def test_a_slow_turn_gets_a_first_byte_before_the_answer():
+ """Past the grace window the stream commits and a heartbeat runs (#2465, #3079)."""
+ hash_key = _stored_hash()
+ with _Slow([_final_reply()], buffered_ccr_grace_seconds=0.01) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200
+ assert resp.text.startswith(": ping\n\n")
+ assert _events(resp.text)[-1] == "[DONE]"
+
+
+def test_an_error_after_the_commit_is_a_typed_stream_error():
+ hash_key = _stored_hash()
+ error = {"error": {"message": "slow down", "type": "rate_limit_error"}}
+ with _Slow([(429, error)], buffered_ccr_grace_seconds=0.01) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, "the status was already committed"
+ events = _events(resp.text)
+ assert events[-1] == {"error": {"message": "slow down", "type": "rate_limit_error"}}
+
+
+def test_a_transport_failure_inside_the_grace_window_is_a_clean_502():
+ hash_key = _stored_hash()
+ with _Proxy([httpx.ConnectError("boom")], ccr_buffered_chat_streaming=True) as proxy:
+ before = proxy.proxy.metrics.requests_failed
+ resp = proxy.post(_request(hash_key))
+ after = proxy.proxy.metrics.requests_failed
+
+ assert resp.status_code == 502
+ assert resp.json()["error"]["type"] == "server_error"
+ assert after == before + 1
+
+
+def test_buffered_and_non_streaming_turns_account_identically():
+ """Same turn both ways: the recorded numbers have to match (#3615, #3699)."""
+ fields = (
+ "status_code",
+ "provider_input_tokens",
+ "output_tokens",
+ "tokens_saved",
+ "cache_read_tokens",
+ "cache_write_tokens",
+ "uncached_input_tokens",
+ )
+
+ def run(stream: bool) -> dict[str, Any]:
+ reset_compression_store()
+ hash_key = _stored_hash()
+ with _Proxy([_retrieve_reply(hash_key), _final_reply()]) as proxy:
+ resp = proxy.post(_request(hash_key, stream=stream))
+ assert resp.status_code == 200
+ assert len(proxy.outcomes) == 1
+ return {field: getattr(proxy.outcomes[0], field) for field in fields}
+
+ buffered = run(stream=True)
+ assert buffered == run(stream=False)
+ assert buffered["provider_input_tokens"] == 500, "both upstream calls must be counted"
+
+
+def test_tools_stay_byte_stable_across_a_session_that_mixes_modes():
+ """The toggle from #3815: stream and non-stream turns must send the same tools."""
+ hash_key = _stored_hash()
+ upstream = [_final_reply(), _final_reply(), _final_reply()]
+ messages = _messages(hash_key)
+ with _Proxy(upstream) as proxy:
+ for stream in (False, True, False):
+ resp = proxy.post(_request(hash_key, messages=messages, stream=stream))
+ assert resp.status_code == 200, resp.text
+
+ assert len(proxy.sent) == 3
+ tools = [json.dumps(body["tools"]) for body in proxy.sent]
+ assert tools[0] == tools[1] == tools[2]
+ assert CCR_TOOL_NAME in tools[0]
+
+
+ ("status", "error_type"),
+ [(429, "rate_limit_error"), (400, "invalid_request_error"), (500, "server_error")],
+)
+def test_failed_continuation_fails_closed_with_its_status(status, error_type):
+ """The client must not get a tool it never declared, and backoff still has to fire."""
+ hash_key = _stored_hash()
+ upstream = [_retrieve_reply(hash_key), (status, {"error": {"message": "nope"}})]
+ with _Proxy(upstream) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == status
+ assert CCR_TOOL_NAME not in resp.text
+ assert resp.json()["error"]["type"] == error_type
+ assert "nope" not in resp.text, "the upstream error body is untrusted and stays in the proxy"
+
+
+def test_failed_continuation_after_the_commit_is_a_typed_stream_error():
+ hash_key = _stored_hash()
+ upstream = [_retrieve_reply(hash_key), (429, {"error": {"message": "nope"}})]
+ with _Slow(upstream, buffered_ccr_grace_seconds=0.01) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, "the status was already committed"
+ assert CCR_TOOL_NAME not in resp.text
+ assert _events(resp.text)[-1]["error"]["type"] == "rate_limit_error"
+
+
+def test_an_expired_marker_is_still_answered_while_the_tool_is_resident():
+ """The hash is gone but the model can still ask for it, and the client cannot answer."""
+ hash_key = _stored_hash()
+ messages = _messages(hash_key)
+ upstream = [_final_reply(), _retrieve_reply(hash_key), _final_reply()]
+ with _Proxy(upstream) as proxy:
+ first = proxy.post(_request(hash_key, messages=messages, stream=False))
+ assert first.status_code == 200, first.text
+
+ reset_compression_store()
+ resp = proxy.post(_request(hash_key, messages=messages))
+
+ assert resp.status_code == 200, resp.text
+ assert proxy.plain_streams == 0
+ assert CCR_TOOL_NAME not in resp.text
+ assert "there are 2 rows" in resp.text
+
+
+def test_a_bypassed_turn_keeps_its_stream():
+ """Bypass skips injection, so there is nothing to answer and no reason to buffer."""
+ hash_key = _stored_hash()
+ with _Proxy([]) as proxy:
+ resp = proxy.post(_request(hash_key), x_headroom_bypass="true")
+
+ assert resp.text == "data: plain\n\n"
+ assert proxy.sent == []
+
+
+def test_response_hooks_do_not_run_on_a_buffered_turn(monkeypatch):
+ """Their on_request half never ran; it is decided before dispatch, like a stream."""
+ import headroom.proxy.turn_hooks as turn_hooks
+
+ calls: list[str] = []
+
+ class _Hook:
+ name = "buffered-only"
+
+ def on_request(self, ctx): # noqa: ANN001
+ calls.append("on_request")
+
+ async def on_response(self, ctx, response, call_model): # noqa: ANN001
+ calls.append("on_response")
+ return None
+
+ hash_key = _stored_hash()
+ with _Proxy([_final_reply()]) as proxy:
+ monkeypatch.setattr(turn_hooks, "_hooks", [_Hook()], raising=False)
+ monkeypatch.setattr(turn_hooks, "registered_turn_hooks", lambda: [_Hook()])
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ assert calls == []
+
+
+def _mixed_reply(hash_key: str) -> dict:
+ return _completion(
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ _tool_call(CCR_TOOL_NAME, {"hash": hash_key}),
+ _tool_call("Read", {"file_path": "a.py"}, call_id="call_2"),
+ ],
+ },
+ "tool_calls",
+ USAGE_FIRST,
+ )
+
+
+def test_retrieval_next_to_a_client_tool_lets_the_model_decide_again():
+ """The unrun client call is dropped, and the continuation replaces the turn."""
+ hash_key = _stored_hash()
+ with _Proxy([_mixed_reply(hash_key), _final_reply()]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ assert CCR_TOOL_NAME not in resp.text
+ assert "there are 2 rows" in resp.text
+ replayed = proxy.sent[1]["messages"][-2]
+ assert [call["function"]["name"] for call in replayed["tool_calls"]] == [CCR_TOOL_NAME]
+
+
+def test_mixed_turn_goes_back_whole_when_its_continuation_fails():
+ """The model made no newer decision, so the client gets the turn it did make."""
+ hash_key = _stored_hash()
+ upstream = [_mixed_reply(hash_key), (500, {"error": {"message": "nope"}})]
+ with _Proxy(upstream) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ choices = [chunk["choices"][0] for chunk in _chunks(resp.text) if chunk["choices"]]
+ names = [
+ choice["delta"]["tool_calls"][0]["function"]["name"]
+ for choice in choices
+ if "tool_calls" in choice["delta"]
+ ]
+ assert names == [CCR_TOOL_NAME, "Read"]
+ assert choices[-1]["finish_reason"] == "tool_calls"
+
+
+def test_continuation_replays_what_the_provider_put_on_the_assistant_message():
+ """DeepSeek rejects a tool-call turn replayed without its reasoning_content."""
+ hash_key = _stored_hash()
+ reply = _retrieve_reply(hash_key)
+ message = reply["choices"][0]["message"]
+ message["reasoning_content"] = "thinking it over"
+ message["reasoning_details"] = [{"text": "t"}]
+ message["refusal"] = None
+ message["annotations"] = []
+ with _Proxy([reply, _final_reply()]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ replayed = proxy.sent[1]["messages"][-2]
+ assert replayed["role"] == "assistant"
+ assert replayed["reasoning_content"] == "thinking it over"
+ assert replayed["reasoning_details"] == [{"text": "t"}]
+ assert "refusal" not in replayed, "empty fields stay out of the replayed message"
+ assert "annotations" not in replayed
+
+
+class _RequiresReasoning(_Proxy):
+ """An upstream that rejects a tool-call turn replayed without its reasoning."""
+
+ async def _retry(self, method, url, headers, body, *args, **kwargs): # noqa: ANN001
+ for message in body["messages"]:
+ if message.get("tool_calls") and "reasoning_content" not in message:
+ self.sent.append(json.loads(json.dumps(body)))
+ return httpx.Response(400, json={"error": {"message": "reasoning_content missing"}})
+ return await super()._retry(method, url, headers, body, *args, **kwargs)
+
+
+def test_a_provider_that_requires_reasoning_on_replay_gets_it():
+ hash_key = _stored_hash()
+ reply = _retrieve_reply(hash_key)
+ reply["choices"][0]["message"]["reasoning_content"] = "required-replay"
+ with _RequiresReasoning([reply, _final_reply()]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ assert "there are 2 rows" in resp.text
+
+
+ "payload",
+ [
+ {"error": {"message": "gateway said no"}},
+ {"choices": []},
+ {"choices": None},
+ {"choices": "nope"},
+ {"choices": [None]},
+ {"choices": [{}]},
+ {"choices": [{"finish_reason": "stop"}]},
+ {"choices": [{"message": None, "finish_reason": "stop"}]},
+ ],
+)
+def test_a_200_with_no_usable_choice_is_not_replayed_as_an_empty_stream(payload):
+ hash_key = _stored_hash()
+ with _Proxy([payload]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 502
+ assert resp.json()["error"]["type"] == "upstream_protocol_error"
+
+
+def _as_sse(payload: Any) -> Response:
+ return _buffered_chat_reply_as_sse(
+ Response(json.dumps(payload).encode(), status_code=200),
+ ccr_response_handler=CCRResponseHandler(),
+ include_usage=False,
+ continuation_status=None,
+ request_id="test",
+ )
+
+
+ "payload",
+ [
+ None,
+ [],
+ {"choices": [None]},
+ {"choices": [{}]},
+ {"choices": [{"message": "text"}]},
+ ],
+)
+def test_the_helper_rejects_a_reply_with_no_usable_choice(payload):
+ result = _as_sse(payload)
+
+ assert result.status_code == 502
+ assert json.loads(result.body)["error"]["type"] == "upstream_protocol_error"
+
+
+def test_a_refusal_is_a_valid_reply():
+ hash_key = _stored_hash()
+ reply = _completion(
+ {"role": "assistant", "content": None, "refusal": "no"}, "stop", USAGE_FIRST
+ )
+ with _Proxy([reply]) as proxy:
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200, resp.text
+ deltas = [chunk["choices"][0]["delta"] for chunk in _chunks(resp.text) if chunk["choices"]]
+ assert {"refusal": "no"} in deltas
+
+
+def test_a_buffered_reply_never_lands_in_the_response_cache():
+ """It was built for a stream:true request; a later caller must not get it (#3019)."""
+ hash_key = _stored_hash()
+ with _Proxy([_final_reply()], cache_enabled=True) as proxy:
+ calls: list[Any] = []
+
+ async def _set(*args: Any, **kwargs: Any) -> None:
+ calls.append(args)
+
+ proxy.proxy.cache.set = _set
+ resp = proxy.post(_request(hash_key))
+
+ assert resp.status_code == 200
+ assert calls == []
+
+
+ ("overrides", "expected"),
+ [
+ ({}, True),
+ ({"body": {"stream": False, "tools": [CLIENT_TOOL]}}, False),
+ ({"body": {"stream": True, "tools": []}}, False),
+ ({"body": {"stream": True}}, False),
+ ({"body": {"stream": True, "tools": [CLIENT_TOOL], "n": 2}}, False),
+ ({"body": {"stream": True, "tools": [CLIENT_TOOL], "n": 1}}, True),
+ ({"body": {"stream": True, "tools": [CLIENT_TOOL], "logprobs": True}}, False),
+ ({"enabled": False}, False),
+ ({"ccr_response_handler_enabled": False}, False),
+ ({"ccr_inject_tool": False}, False),
+ ({"routed_to_backend": True}, False),
+ ({"upstream_base_url": "https://opencode.ai/zen/v1"}, False),
+ ],
+)
+def test_eligibility(overrides, expected):
+ args: dict[str, Any] = {
+ "body": {"stream": True, "tools": [CLIENT_TOOL]},
+ "enabled": True,
+ "ccr_response_handler_enabled": True,
+ "ccr_inject_tool": True,
+ "routed_to_backend": False,
+ "upstream_base_url": "https://api.openai.com",
+ **overrides,
+ }
+ assert _is_openai_chat_stream_ccr_eligible(**args) is expected
+
+
+def test_taking_stream_options_leaves_other_fields_alone():
+ body = {"stream": True, "stream_options": {"include_usage": True}, "temperature": 0}
+ assert _take_stream_options_for_buffered_chat(body) is True
+ assert body == {"stream": True, "temperature": 0}
+
+
+def test_frames_keep_the_fields_a_client_replays_on_its_next_turn():
+ call = {
+ **_tool_call("Read", {"file_path": "a.py"}),
+ "extra_content": {"google": {"thought_signature": "sig"}},
+ }
+ reply = _completion(
+ {
+ "role": "assistant",
+ "reasoning_content": "thinking it over",
+ "reasoning_details": [{"text": "t"}],
+ "refusal": None,
+ "content": "reading",
+ "tool_calls": [call],
+ },
+ "tool_calls",
+ USAGE_FIRST,
+ )
+ frames = StreamingCCRHandler.openai_chat_sse_frames(reply, include_usage=False)
+ deltas = [
+ chunk["choices"][0]["delta"]
+ for chunk in _chunks(b"".join(frames).decode())
+ if chunk["choices"]
+ ]
+
+ assert {"reasoning_content": "thinking it over", "reasoning_details": [{"text": "t"}]} in deltas
+ tool_delta = next(delta["tool_calls"][0] for delta in deltas if "tool_calls" in delta)
+ assert tool_delta["extra_content"] == {"google": {"thought_signature": "sig"}}
+ assert tool_delta["index"] == 0
+ assert frames[-1] == b"data: [DONE]\n\n"