Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(ccr): answer headroom_retrieve on streaming chat completions (#4631)
SK Arin31dcaf41f7fa
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdxindex 819328dad..4f8c00bb6 100644--- a/docs/content/docs/proxy.mdx+++ b/docs/content/docs/proxy.mdx@@ -349,6 +349,7 @@ By default Headroom stores originals so the model can recover them via `headroom|---|---|---|| `--no-ccr` / `HEADROOM_NO_CCR` | CCR on | Disable retrieval markers **and** the injected `headroom_retrieve` tool. || `--lossless` / `HEADROOM_LOSSLESS` | `false` | Format-native lossless compaction only — no CCR marker, no retrieval tool. |+| `--ccr-buffered-chat-streaming` / `HEADROOM_CCR_BUFFERED_CHAT_STREAMING` | `false` | Answer `headroom_retrieve` on streaming `/v1/chat/completions`. A turn carrying a redeemable marker runs as `stream: false` upstream and is re-emitted as `chat.completion.chunk` frames, so its text arrives all at once; other turns keep their real stream || `--no-ccr-proactive-expansion` | expansion on | Stop proactively re-expanding compressed content when the model appears to need it. |### File-read handlingdiff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.pyindex f9bcab143..c027d2c13 100644--- a/headroom/ccr/response_handler.py+++ b/headroom/ccr/response_handler.py@@ -54,6 +54,20 @@ RESIDUAL_CCR_SKIPPED_MIXED = "skipped_mixed_tools"RESIDUAL_CCR_ERROR = "error"+def _openai_message_extras(message: dict[str, Any]) -> dict[str, Any]:+ """The fields a provider put on a chat message besides the standard three.++ Whoever replays the turn may have to send these back (DeepSeek's+ ``reasoning_content``, OpenRouter's ``reasoning_details``) and gets rejected+ if they went missing. Empty ones (``refusal: null``) stay behind.+ """+ return {+ key: value+ for key, value in message.items()+ if key not in ("role", "content", "tool_calls") and value+ }++@dataclassclass CCRToolResult:"""Result of handling a CCR tool call."""@@ -397,6 +411,7 @@ class CCRResponseHandler:first = choices[0] if isinstance(choices, list) and choices else {}message = first.get("message", {}) if isinstance(first, dict) else {}return {+ **_openai_message_extras(message),"role": "assistant","content": message.get("content"),"tool_calls": message.get("tool_calls"),@@ -951,7 +966,8 @@ class StreamingCCRHandler:response["usage"] = usagereturn response- def _openai_response_to_chunks(self, response: dict[str, Any]) -> list[bytes]:+ @staticmethod+ def _openai_response_to_chunks(response: dict[str, Any]) -> list[bytes]:"""Split a non-streaming ``chat.completion`` body into SSE chunk frames.A streaming client reads ``choices[].delta``, not ``choices[].message``.@@ -968,10 +984,7 @@ class StreamingCCRHandler:message = {}finish_reason = choice.get("finish_reason") or "stop"- base: dict[str, Any] = {"object": "chat.completion.chunk"}- for key in ("id", "created", "model", "system_fingerprint"):- if response.get(key) is not None:- base[key] = response[key]+ base = StreamingCCRHandler._openai_chunk_envelope(response)def frame(delta: dict[str, Any], reason: str | None) -> bytes:payload = {@@ -982,6 +995,10 @@ class StreamingCCRHandler:frames = [frame({"role": message.get("role") or "assistant"}, None)]+ extras = _openai_message_extras(message)+ if extras:+ frames.append(frame(extras, None))+content = message.get("content")if content:frames.append(frame({"content": content}, None))@@ -999,10 +1016,15 @@ class StreamingCCRHandler:{"tool_calls": [{+ # Spread first: gateways hang opaque fields+ # off a tool call that must survive the+ # round trip (Gemini's thought signatures).+ **tool_call,"index": index,"id": tool_call.get("id", ""),"type": tool_call.get("type", "function"),"function": {+ **function,"name": function.get("name", ""),"arguments": function.get("arguments", ""),},@@ -1016,6 +1038,31 @@ class StreamingCCRHandler:frames.append(frame({}, finish_reason))return frames+ @staticmethod+ def _openai_chunk_envelope(response: dict[str, Any]) -> dict[str, Any]:+ """The fields every ``chat.completion.chunk`` frame of a reply repeats."""+ envelope: dict[str, Any] = {"object": "chat.completion.chunk"}+ for key in ("id", "created", "model", "system_fingerprint"):+ if response.get(key) is not None:+ envelope[key] = response[key]+ return envelope++ @staticmethod+ def openai_chat_sse_frames(response: dict[str, Any], *, include_usage: bool) -> list[bytes]:+ """Render a whole ``chat.completion`` body as the SSE a stream would be.++ ``include_usage`` adds the trailing usage-only chunk (``choices: []``)+ that ``stream_options.include_usage`` asks for.+ """+ frames = StreamingCCRHandler._openai_response_to_chunks(response)+ usage = response.get("usage")+ if include_usage and usage is not None:+ envelope = StreamingCCRHandler._openai_chunk_envelope(response)+ payload = {**envelope, "choices": [], "usage": usage}+ frames.append(f"data: {json.dumps(payload)}\n\n".encode())+ frames.append(b"data: [DONE]\n\n")+ return frames+async def _response_to_sse(self,response: dict[str, Any],diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex b49702456..a18f6101d 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -528,6 +528,17 @@ def dashboard(port: int, no_open: bool) -> None:"Env: HEADROOM_CCR_INLINE_RESOLVE."),)+ "--ccr-buffered-chat-streaming",+ is_flag=True,+ envvar="HEADROOM_CCR_BUFFERED_CHAT_STREAMING",+ help=(+ "Answer headroom_retrieve on streaming /v1/chat/completions. A turn that "+ "carries a redeemable marker runs as stream:false upstream and is "+ "re-emitted as chat.completion.chunk frames, so that turn's text arrives "+ "all at once. Off by default. Env: HEADROOM_CCR_BUFFERED_CHAT_STREAMING."+ ),+)@click.option("--no-ccr-proactive-expansion",is_flag=True,@@ -1178,6 +1189,7 @@ def proxy(no_ccr: bool,lossless: bool,ccr_inline_resolve: bool,+ ccr_buffered_chat_streaming: bool,no_ccr_proactive_expansion: bool,proxy_extension: tuple[str, ...],compressor: tuple[str, ...],@@ -1518,6 +1530,7 @@ def proxy(ccr_inject_marker=not no_ccr,ccr_handle_responses=not no_ccr,ccr_resolve_markers_inline=ccr_inline_resolve,+ ccr_buffered_chat_streaming=ccr_buffered_chat_streaming,lossless=lossless,ccr_proactive_expansion=not no_ccr_proactive_expansion,# Flatten repeat-flag tuple AND any comma-separated values inside it.diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex cd627c461..8d7dcc533 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -1399,16 +1399,171 @@ def _should_buffer_openai_responses_stream_ccr()-def _should_inject_openai_chat_ccr_tool(*, ccr_inject_tool: bool, stream: bool) -> bool:+def _should_inject_openai_chat_ccr_tool(+ *, ccr_inject_tool: bool, stream: bool, buffered_eligible: bool = False+) -> bool:"""Return whether chat-completions can redeem an injected CCR tool.- The chat streaming path forwards SSE events immediately and deliberately- does not run the response continuation loop. Injecting ``headroom_retrieve``- there makes OpenAI-compatible clients attempt an unknown tool call. The+ A plain chat streaming turn forwards SSE events immediately and does not run+ the response continuation loop, so injecting ``headroom_retrieve`` there+ makes OpenAI-compatible clients attempt an unknown tool call. Thenon-streaming path can intercept and resolve it; Responses streaming has aseparate buffered-CCR path and is unaffected by this predicate.++ ``buffered_eligible`` is the third case: if this turn carries a marker it+ will run the continuation loop before replying, so the tool is redeemable.+ If it carries none, the model has no hash to call it with and the tool stays+ resident only to keep the tools array byte-stable.+ """+ return bool(ccr_inject_tool and (buffered_eligible or not stream))+++def _is_openai_chat_stream_ccr_eligible(+ *,+ body: dict[str, Any],+ enabled: bool,+ ccr_response_handler_enabled: bool,+ ccr_inject_tool: bool,+ routed_to_backend: bool,+ upstream_base_url: str | None = None,+) -> bool:+ """Return whether a streaming chat turn may answer retrievals by buffering.++ Unlike :func:`_should_buffer_openai_responses_stream_ccr` this cannot key on+ ``headroom_retrieve`` already being in ``tools``: on this path that test is+ what decides injection, so it would never pass.++ Requiring a non-empty client ``tools`` array keeps the rule from #3810: a+ request with no tools is not an agent harness and must not be handed a tools+ array it never sent. The rest are turns buffering cannot serve:++ - A backend route streams through its own path, with no continuation loop.+ - ``n > 1`` and ``logprobs`` ask for output that resynthesis does not carry.+ - OpenCode Zen rejects requests Headroom has reshaped (``stream:false`` plus+ ``accept: application/json``) with ``403 FreeTierError`` (#3656).+ """+ n = body.get("n")+ return bool(+ body.get("stream")+ and enabled+ and ccr_response_handler_enabled+ and ccr_inject_tool+ and body.get("tools")+ and not routed_to_backend+ and not (isinstance(n, int) and n > 1)+ and not body.get("logprobs")+ and not is_opencode_zen_base(upstream_base_url)+ )+++def _chat_body_has_ccr_marker(body: dict[str, Any]) -> bool:+ """Is there anything in the outgoing messages a resident tool could be called for?++ Goes by marker shape alone, without the ownership check that decides+ injection (#2836). An expired hash can still draw a retrieval call once the+ tool is resident, and only the continuation loop can answer that call.+ """+ from headroom.ccr.tool_injection import CCRToolInjector++ messages = body.get("messages")+ if not isinstance(messages, list):+ return False+ probe = CCRToolInjector(provider="openai", inject_tool=False, inject_system_instructions=False)+ try:+ return bool(probe.scan_for_markers(messages))+ except Exception: # pragma: no cover - defensive+ logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)+ return True+++def _take_stream_options_for_buffered_chat(body: dict[str, Any]) -> bool:+ """Drop ``stream_options`` from a body going out as ``stream: false``.++ OpenAI answers 400 to ``stream_options`` on a non-streaming request. Returns+ whether the client should still get the usage chunk, decided by the same+ rule a plain streamed turn uses so the two cannot drift apart.+ """+ _apply_stream_usage_option(body)+ stream_options = body.pop("stream_options", None)+ return isinstance(stream_options, dict) and bool(stream_options.get("include_usage"))+++def _buffered_chat_reply_as_sse(+ result: Response,+ *,+ ccr_response_handler: Any,+ include_usage: bool,+ continuation_status: int | None,+ request_id: str,+) -> Response:+ """Turn the direct chat path's JSON reply into what a streaming client reads.++ Anything but a 200 goes back untouched. Inside the grace window that hands+ the client the real status, and past it :func:`buffered_ccr_asgi_call` turns+ it into a typed stream error (#2997, #3079)."""- return bool(ccr_inject_tool and not stream)+ from fastapi.responses import JSONResponse, StreamingResponse++ from headroom.ccr.response_handler import RESIDUAL_CCR_ERROR, StreamingCCRHandler++ if result.status_code != 200:+ return result++ try:+ reply = json.loads(result.body)+ except (TypeError, ValueError):+ reply = None+ choices = reply.get("choices") if isinstance(reply, dict) else None+ first_choice = choices[0] if isinstance(choices, list) and choices else None+ if not isinstance(first_choice, dict) or not isinstance(first_choice.get("message"), dict):+ logger.warning(f"[{request_id}] CCR: rejecting malformed buffered chat 200 reply")+ return JSONResponse(+ status_code=502,+ content={+ "error": {+ "type": "upstream_protocol_error",+ "message": "Upstream returned an invalid buffered response.",+ }+ },+ )++ sse_headers = {+ k: v+ for k, v in result.headers.items()+ if k.lower() not in ("content-length", "content-type")+ }++ # A retrieval call standing alone means the continuation failed, and the+ # client cannot act on a tool it never declared. Next to a client tool it is+ # the turn handle_response gives back when a continuation fails, because the+ # model made no newer decision, so that goes to the client as is. A plain JSON+ # error works on both sides of the commit: the wrapper turns it into a typed+ # stream error if SSE is already on the wire, so a rate limit on the+ # continuation still reads as a rate limit.+ if ccr_response_handler.residual_ccr_status(reply, "openai") == RESIDUAL_CCR_ERROR:+ status = continuation_status or 502+ logger.warning(+ f"[{request_id}] CCR: buffered chat reply still contains headroom_retrieve "+ f"after handling; failing closed with {status}"+ )+ return JSONResponse(+ status_code=status,+ headers=sse_headers,+ content={+ "error": {+ "message": "Unable to safely complete streamed CCR retrieval.",+ "type": OPENAI_ERROR_FORMAT.error_type_for_status(status),+ }+ },+ )++ frames = StreamingCCRHandler.openai_chat_sse_frames(reply, include_usage=include_usage)++ async def _frames():+ for frame in frames:+ yield frame++ return StreamingResponse(_frames(), media_type="text/event-stream", headers=sse_headers)def _responses_input_to_items(input_data: Any) -> list[dict[str, Any]]:@@ -4237,14 +4392,34 @@ class OpenAIHandlerMixin:# anchored on the previous turn's tool list never busts.tools = body.get("tools")_original_tools = tools # Preserve for diagnostic / future retry+ # Decided before injection: a turn that can buffer can answer the tool,+ # so it may carry it even though the client asked for a stream. Whether+ # it actually buffers is settled at dispatch, from the final body.+ from headroom.proxy.route_advice import advice_from++ _ccr_handler = getattr(self, "ccr_response_handler", None)+ chat_stream_ccr_eligible = _is_openai_chat_stream_ccr_eligible(+ body=body,+ enabled=self.config.ccr_buffered_chat_streaming,+ ccr_response_handler_enabled=bool(+ _ccr_handler and getattr(_ccr_handler.config, "enabled", True)+ ),+ ccr_inject_tool=self.config.ccr_inject_tool,+ routed_to_backend=(+ getattr(self, "anthropic_backend", None) is not None+ or advice_from(request) is not None+ ),+ upstream_base_url=upstream_base_url,+ )can_inject_ccr_tool = _should_inject_openai_chat_ccr_tool(ccr_inject_tool=self.config.ccr_inject_tool,stream=stream,+ buffered_eligible=chat_stream_ccr_eligible,)if (self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions) and not _bypass:- if self.config.ccr_inject_tool and stream:+ if self.config.ccr_inject_tool and stream and not chat_stream_ccr_eligible:logger.info(f"[{request_id}] CCR: skipping retrieval-tool injection for ""OpenAI chat streaming; this path cannot intercept tool calls"@@ -4253,7 +4428,8 @@ class OpenAIHandlerMixin:provider="openai",inject_tool=False, # routed through sticky helper belowinject_system_instructions=(- self.config.ccr_inject_system_instructions and not stream+ self.config.ccr_inject_system_instructions+ and (not stream or chat_stream_ccr_eligible)),)injector.scan_for_markers(optimized_messages)@@ -4263,7 +4439,7 @@ class OpenAIHandlerMixin:injector.verify_ownership()if (self.config.ccr_inject_system_instructions- and not stream+ and (not stream or chat_stream_ccr_eligible)and injector.has_compressed_content):optimized_messages = injector.inject_into_system_message(@@ -5175,8 +5351,31 @@ class OpenAIHandlerMixin:)url = _append_request_query(url, request.url.query)+ # The tool stays resident for the session so the tools array is+ # byte-stable, but the model has nothing to call it with unless this body+ # carries a marker. A turn with none keeps its real stream: buffering it+ # would make time-to-first-byte the whole generation for nothing (#3071).+ # Checking for the tool too keeps a bypassed turn, which got none, streaming.+ buffered_chat_ccr = (+ chat_stream_ccr_eligible+ and has_ccr_retrieve_tool(body.get("tools"))+ and _chat_body_has_ccr_marker(body)+ )+ buffered_chat_include_usage = False+ buffered_chat_continuation_status: int | None = None+ if buffered_chat_ccr:+ buffered_chat_include_usage = _take_stream_options_for_buffered_chat(body)+ body["stream"] = False+ # A strict gateway rejects `stream: false` with `Accept: text/event-stream` (#3078).+ _accept_key = next((k for k in headers if k.lower() == "accept"), "accept")+ headers[_accept_key] = "application/json"+ logger.info(+ f"[{request_id}] CCR: buffering streaming chat turn "+ "(stream:false upstream, resynthesized as chat.completion.chunk)"+ )+try:- if stream:+ if stream and not buffered_chat_ccr:# Request usage stats for accurate token counting instead of# byte-based estimation — without overriding an explicit client# choice (see _apply_stream_usage_option).@@ -5210,7 +5409,12 @@ class OpenAIHandlerMixin:original_messages=original_client_messages,outcome_provider=openai_chat_outcome_provider,)- else:++ async def _direct_chat_turn() -> Response:+ # Usage, the prefix tracker, savings and CCR continuations all+ # live in here, so a buffered turn is accounted for exactly like+ # a non-streaming one instead of through a second copy.+ nonlocal headers, buffered_chat_continuation_statusheaders = await apply_copilot_api_auth(headers, url=url)response = await self._retry_request("POST", url, headers, body)@@ -5232,7 +5436,13 @@ class OpenAIHandlerMixin:# hook actually calls the model again._hook_usage = TurnHookUsage()- if _registered_turn_hooks() and response.status_code == 200:+ # A buffered turn only ran stream-safe on_request hooks (that is+ # decided before dispatch), so it skips their response half too.+ if (+ _registered_turn_hooks()+ and response.status_code == 200+ and not buffered_chat_ccr+ ):try:_hook_resp_json = response.json()except (ValueError, json.JSONDecodeError):@@ -5512,6 +5722,7 @@ class OpenAIHandlerMixin:# means on a path that changed nothing.if _ccr_cont_error is not None or _ccr_final_json is _pre_ccr_json:if _ccr_cont_error is not None:+ buffered_chat_continuation_status = _ccr_cont_errorlogger.warning(f"[{request_id}] CCR: continuation returned HTTP "f"{_ccr_cont_error}; forwarding the upstream reply "@@ -5668,13 +5879,9 @@ class OpenAIHandlerMixin:# the live (hooked) `messages` (#327).## ``not stream`` mirrors the read gate at the cache lookup- # above. It is currently redundant here — a streaming chat- # request returns via ``_stream_response`` well before this- # point — but the Anthropic handler had the same shape until a- # buffered-CCR branch started falling through to its store- # site, which let a response built for a stream:true request- # answer a later non-streaming caller (#3019). Stating the- # invariant keeps that from being reintroduced silently.+ # above. A buffered turn reaches this point with ``stream`` still+ # true, and a response built for a stream:true request must not+ # answer a later non-streaming caller (#3019).if (self.cacheand not stream@@ -5795,6 +6002,37 @@ class OpenAIHandlerMixin:status_code=response.status_code,headers=response_headers,)++ if not buffered_chat_ccr:+ return await _direct_chat_turn()++ async def _buffered_chat_turn() -> Response:+ return _buffered_chat_reply_as_sse(+ await _direct_chat_turn(),+ ccr_response_handler=_ccr_handler,+ include_usage=buffered_chat_include_usage,+ continuation_status=buffered_chat_continuation_status,+ request_id=request_id,+ )++ async def _record_buffered_failure(**_: Any) -> None:+ # The shared wrapper only knows "openai"; keep the label this+ # handler uses for xAI and custom upstreams.+ await self.metrics.record_failed(provider=openai_chat_outcome_provider)++ _buffered_call = buffered_ccr_asgi_call(+ operation=asyncio.create_task(_buffered_chat_turn()),+ fmt=OPENAI_ERROR_FORMAT,+ grace_seconds=self.config.buffered_ccr_grace_seconds,+ record_failed=_record_buffered_failure,+ request_id=request_id,+ )++ class _BufferedCCRResponse(Response):+ async def __call__(self, scope, receive, send): # noqa: ANN001+ await _buffered_call(scope, receive, send)++ return _BufferedCCRResponse(media_type="text/event-stream")except Exception as e:await self.metrics.record_failed(provider=openai_chat_outcome_provider)# Error names the failure; text and traceback go to debug.diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 18bb2d364..894acfc6e 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -422,6 +422,16 @@ class ProxyConfig:# stream-idle watchdog fires. 0 or less disables the keepalive entirely.# See headroom/proxy/buffered_ccr_response.py (#3079).buffered_ccr_grace_seconds: float = DEFAULT_BUFFERED_CCR_GRACE_SECONDS+ # Buffered CCR for /v1/chat/completions streaming. Without it this is the+ # one streaming path with no way to answer a retrieval: content carries+ # markers but `headroom_retrieve` is withheld, and the tool toggles in and+ # out for sessions that mix streaming with non-streaming turns, busting the+ # tools cache segment each way. Enabling it flips turns that carry a+ # redeemable marker to `stream: false` upstream and resynthesizes+ # chat.completion.chunk frames, as Responses streaming already does. Off by+ # default: this is the most incident-prone path in the proxy (#2465, #2997,+ # #3079), so the default flips separately, with measurements (#3815).+ ccr_buffered_chat_streaming: bool = False# Connection poolmax_connections: int = 500diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 24c3d1a03..b3afc278c 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -6497,6 +6497,7 @@ def _proxy_config_from_env() -> ProxyConfig:"HEADROOM_BUFFERED_CCR_GRACE_SECONDS",DEFAULT_BUFFERED_CCR_GRACE_SECONDS,),+ ccr_buffered_chat_streaming=_get_env_bool("HEADROOM_CCR_BUFFERED_CHAT_STREAMING", False),vertex_api_url=os.environ.get("VERTEX_TARGET_API_URL"),backend=_get_env_str("HEADROOM_BACKEND", "anthropic"),bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", "us-west-2"),diff --git a/headroom/settings_store.py b/headroom/settings_store.pyindex ffc0c73a3..014740f4b 100644--- a/headroom/settings_store.py+++ b/headroom/settings_store.py@@ -165,6 +165,19 @@ SETTINGS: tuple[SettingField, ...] = (),tier="advanced",),+ SettingField(+ "HEADROOM_CCR_BUFFERED_CHAT_STREAMING",+ "ccr_buffered_chat_streaming",+ "Buffered CCR for streaming chat",+ "Compression",+ "bool",+ default=False,+ help=(+ "Answer retrieval calls on streaming Chat Completions by buffering "+ "turns that carry CCR markers. Buffered text arrives all at once."+ ),+ tier="advanced",+ ),# --- Limits ---SettingField("HEADROOM_RPM",diff --git a/tests/test_cli_proxy_env.py b/tests/test_cli_proxy_env.pyindex c47497ec0..33724d992 100644--- a/tests/test_cli_proxy_env.py+++ b/tests/test_cli_proxy_env.py@@ -797,6 +797,27 @@ class TestCLIProxyEnvVars:assert captured_config["config"].openai_api_url == "http://my-vllm:4000"assert captured_config["config"].gemini_api_url == "http://my-gemini:5000"+ @pytest.mark.parametrize(+ ("args", "env", "expected"),+ [+ ([], {}, False),+ (["--ccr-buffered-chat-streaming"], {}, True),+ ([], {"HEADROOM_CCR_BUFFERED_CHAT_STREAMING": "1"}, True),+ ],+ )+ def test_ccr_buffered_chat_streaming_reaches_config(self, runner, args, env, expected):+ """`headroom proxy` builds its own ProxyConfig, so the flag has to be wired here too."""+ captured_config = {}++ def mock_run_server(config, **kwargs):+ captured_config["config"] = config++ with patch("headroom.proxy.server.run_server", mock_run_server):+ result = runner.invoke(main, ["proxy", *args], env=env, catch_exceptions=False)++ assert result.exit_code == 0, result.output+ assert captured_config["config"].ccr_buffered_chat_streaming is expected+@pytest.mark.parametrize("timeout", [-1, 0, 1, 10000])def test_request_timeout_cli_flags(self, runner, timeout):"""Fast-fail CLI flags should map into ProxyConfig."""diff --git a/tests/test_openai_chat_buffered_ccr.py b/tests/test_openai_chat_buffered_ccr.pynew file mode 100644index 000000000..5cdffc0c7--- /dev/null+++ b/tests/test_openai_chat_buffered_ccr.py@@ -0,0 +1,718 @@+"""Buffered CCR on streaming /v1/chat/completions (#3815).++Compression still runs on a streaming chat turn, so content carries retrieval+markers, but this path had no way to answer ``headroom_retrieve``. With+``ccr_buffered_chat_streaming`` on, a turn whose outgoing body carries a+redeemable marker runs as ``stream: false`` upstream through the same code as a+non-streaming turn, then goes back to the client as ``chat.completion.chunk``+frames. Everything else keeps its real stream.+"""++from __future__ import annotations++import json+from typing import Any++import pytest++fastapi = pytest.importorskip("fastapi")+httpx = pytest.importorskip("httpx")++from fastapi.responses import Response # noqa: E402+from fastapi.testclient import TestClient # noqa: E402++from headroom.cache.compression_store import ( # noqa: E402+ get_compression_store,+ reset_compression_store,+)+from headroom.ccr.response_handler import CCRResponseHandler, StreamingCCRHandler # noqa: E402+from headroom.ccr.tool_injection import CCR_TOOL_NAME # noqa: E402+from headroom.proxy.handlers.openai import ( # noqa: E402+ _buffered_chat_reply_as_sse,+ _is_openai_chat_stream_ccr_eligible,+ _take_stream_options_for_buffered_chat,+)+from headroom.proxy.helpers import _reset_session_ccr_tracker_for_test # noqa: E402+from headroom.proxy.server import ProxyConfig, _proxy_config_from_env, create_app # noqa: E402++ORIGINAL = "row 1: the original uncompressed rows\nrow 2: more of them"+CLIENT_TOOL = {"type": "function", "function": {"name": "Read", "description": "read a file"}}+USAGE_FIRST = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}+USAGE_FINAL = {"prompt_tokens": 400, "completion_tokens": 10, "total_tokens": 410}++[email protected](autouse=True)+def reset_store():+ reset_compression_store()+ _reset_session_ccr_tracker_for_test()+ yield+ reset_compression_store()+ _reset_session_ccr_tracker_for_test()+++def _stored_hash() -> str:+ return get_compression_store().store(+ original=ORIGINAL,+ compressed="[2 items compressed to 0]",+ original_item_count=2,+ compressed_item_count=0,+ )+++def _completion(message: dict[str, Any], finish_reason: str, usage: dict[str, int]) -> dict:+ return {+ "id": "chatcmpl-1",+ "object": "chat.completion",+ "created": 1,+ "model": "gpt-4o",+ "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],+ "usage": usage,+ }+++def _tool_call(name: str, arguments: dict[str, Any], call_id: str = "call_1") -> dict:+ return {+ "id": call_id,+ "type": "function",+ "function": {"name": name, "arguments": json.dumps(arguments)},+ }+++def _retrieve_reply(hash_key: str) -> dict:+ message = {+ "role": "assistant",+ "content": None,+ "tool_calls": [_tool_call(CCR_TOOL_NAME, {"hash": hash_key})],+ }+ return _completion(message, "tool_calls", USAGE_FIRST)+++def _final_reply() -> dict:+ return _completion(+ {"role": "assistant", "content": "there are 2 rows"},+ "stop",+ USAGE_FINAL,+ )+++def _messages(hash_key: str | None) -> list[dict[str, Any]]:+ content = "how many rows?"+ if hash_key is not None:+ content += f" [2 items compressed to 0. Retrieve more: hash={hash_key}]"+ return [{"role": "user", "content": content}]+++def _request(hash_key: str | None, **overrides: Any) -> dict[str, Any]:+ return {+ "model": "gpt-4o",+ "messages": _messages(hash_key),+ "stream": True,+ "tools": [CLIENT_TOOL],+ **overrides,+ }+++def _events(text: str) -> list[Any]:+ """Parse an SSE body into its JSON payloads, keeping the [DONE] sentinel."""+ events: list[Any] = []+ for line in text.splitlines():+ if line.startswith("data: "):+ data = line[len("data: ") :]+ events.append(data if data == "[DONE]" else json.loads(data))+ return events+++def _chunks(text: str) -> list[dict[str, Any]]:+ return [event for event in _events(text) if event != "[DONE]"]+++class _Proxy:+ """A live app whose upstream calls are served from a script."""++ def __init__(self, upstream: list[Any], **config: Any) -> None:+ self._remaining = list(upstream)+ self.sent: list[dict[str, Any]] = []+ self.sent_headers: list[dict[str, str]] = []+ self.outcomes: list[Any] = []+ self.plain_streams = 0+ settings = {+ "optimize": False,+ "cache_enabled": False,+ "rate_limit_enabled": False,+ "ccr_handle_responses": True,+ "ccr_inject_tool": True,+ "ccr_buffered_chat_streaming": True,+ **config,+ }+ self._app = create_app(ProxyConfig(**settings))++ async def _retry(self, method, url, headers, body, *args, **kwargs): # noqa: ANN001+ self.sent.append(json.loads(json.dumps(body)))+ self.sent_headers.append(dict(headers))+ entry = self._remaining.pop(0)+ if isinstance(entry, Exception):+ raise entry+ status, payload = entry if isinstance(entry, tuple) else (200, entry)+ return httpx.Response(status, json=payload, headers={"content-type": "application/json"})++ def __enter__(self) -> _Proxy:+ self._client = TestClient(self._app)+ self._client.__enter__()+ proxy = self._client.app.state.proxy+ proxy._retry_request = self._retry+ self.proxy = proxy++ async def _plain_stream(*args: Any, **kwargs: Any) -> Response:+ self.plain_streams += 1+ return Response(b"data: plain\n\n", media_type="text/event-stream")++ proxy._stream_response = _plain_stream++ real_record = proxy._record_request_outcome++ async def _capture(outcome, *args, **kwargs): # noqa: ANN001+ self.outcomes.append(outcome)+ return await real_record(outcome, *args, **kwargs)++ proxy._record_request_outcome = _capture+ return self++ def __exit__(self, *exc: Any) -> None:+ self._client.__exit__(*exc)++ def post(self, body: dict[str, Any], **extra_headers: str) -> httpx.Response:+ return self._client.post(+ "/v1/chat/completions",+ json=body,+ headers={+ "Authorization": "Bearer test-key",+ "x-api-key": "test-key",+ "Accept": "text/event-stream",+ **{name.replace("_", "-"): value for name, value in extra_headers.items()},+ },+ )+++def test_flag_off_keeps_the_plain_stream_and_withholds_the_tool():+ hash_key = _stored_hash()+ with _Proxy([], ccr_buffered_chat_streaming=False) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.text == "data: plain\n\n"+ assert proxy.plain_streams == 1+ assert proxy.sent == [], "flag off must not take the buffered path"+++def test_flag_defaults_off():+ assert ProxyConfig().ccr_buffered_chat_streaming is False+++def test_env_var_turns_the_flag_on(monkeypatch):+ monkeypatch.setenv("HEADROOM_CCR_BUFFERED_CHAT_STREAMING", "1")+ assert _proxy_config_from_env().ccr_buffered_chat_streaming is True+++def test_retrieval_is_answered_and_the_client_reads_a_real_stream():+ hash_key = _stored_hash()+ with _Proxy([_retrieve_reply(hash_key), _final_reply()]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ assert resp.headers["content-type"].startswith("text/event-stream")+ assert "content-length" not in resp.headers or int(resp.headers["content-length"]) == len(+ resp.content+ )+ events = _events(resp.text)+ assert events[-1] == "[DONE]"++ chunks = _chunks(resp.text)+ assert {chunk["object"] for chunk in chunks} == {"chat.completion.chunk"}+ choices = [chunk for chunk in chunks if chunk["choices"]]+ assert choices[0]["choices"][0]["delta"]["role"] == "assistant"+ text = "".join(chunk["choices"][0]["delta"].get("content") or "" for chunk in choices)+ assert text == "there are 2 rows"+ assert choices[-1]["choices"][0]["finish_reason"] == "stop"+ assert CCR_TOOL_NAME not in resp.text++ first, continuation = proxy.sent+ assert first["stream"] is False+ assert "stream_options" not in first, "OpenAI answers 400 to stream_options without a stream"+ assert CCR_TOOL_NAME in {tool["function"]["name"] for tool in first["tools"]}+ assert ORIGINAL.splitlines()[0] in json.dumps(continuation)+ assert {k.lower(): v for k, v in proxy.sent_headers[0].items()}["accept"] == "application/json"+++def test_a_turn_with_no_marker_keeps_its_real_stream():+ """Buffering a turn retrieval cannot help only costs time to first byte (#3071)."""+ with _Proxy([]) as proxy:+ resp = proxy.post(_request(None))++ assert resp.text == "data: plain\n\n"+ assert proxy.plain_streams == 1+ assert proxy.sent == []+++def test_a_foreign_marker_does_not_buffer():+ """Another context tool can use the same marker shape; only owned hashes count (#2836)."""+ with _Proxy([]) as proxy:+ resp = proxy.post(_request("0123456789abcdef0123456789abcdef"))++ assert resp.text == "data: plain\n\n"+ assert proxy.sent == []+++def test_a_client_tool_call_still_ends_in_tool_calls():+ hash_key = _stored_hash()+ client_call = _tool_call("Read", {"file_path": "a.py"})+ reply = _completion(+ {"role": "assistant", "content": None, "tool_calls": [client_call]},+ "tool_calls",+ USAGE_FIRST,+ )+ with _Proxy([reply]) as proxy:+ resp = proxy.post(_request(hash_key))++ choices = [chunk["choices"][0] for chunk in _chunks(resp.text) if chunk["choices"]]+ assert choices[-1]["finish_reason"] == "tool_calls"+ tool_deltas = [+ choice["delta"]["tool_calls"][0] for choice in choices if "tool_calls" in choice["delta"]+ ]+ assert [delta["function"]["name"] for delta in tool_deltas] == ["Read"]+ assert tool_deltas[0]["index"] == 0+++ ("stream_options", "expect_usage"),+ [+ (None, True),+ ({"include_usage": True}, True),+ ({"include_usage": False}, False),+ ({}, True),+ ],+)+def test_usage_chunk_follows_the_plain_stream_rule(stream_options, expect_usage):+ hash_key = _stored_hash()+ overrides = {} if stream_options is None else {"stream_options": stream_options}+ with _Proxy([_final_reply()]) as proxy:+ resp = proxy.post(_request(hash_key, **overrides))++ usage_chunks = [chunk for chunk in _chunks(resp.text) if chunk.get("usage")]+ if expect_usage:+ assert len(usage_chunks) == 1+ assert usage_chunks[0]["choices"] == []+ assert usage_chunks[0]["usage"] == USAGE_FINAL+ assert _chunks(resp.text)[-1] is not None and _chunks(resp.text)[-1]["usage"]+ else:+ assert usage_chunks == []+ assert "stream_options" not in proxy.sent[0]+++def test_upstream_error_inside_the_grace_window_keeps_its_status():+ """A 429 must reach the client as a 429, not a 200 with no frames (#2997)."""+ hash_key = _stored_hash()+ error = {"error": {"message": "slow down", "type": "rate_limit_error"}}+ with _Proxy([(429, error)]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 429+ assert resp.json()["error"]["message"] == "slow down"+++class _Slow(_Proxy):+ async def _retry(self, method, url, headers, body, *args, **kwargs): # noqa: ANN001+ import asyncio++ await asyncio.sleep(0.05)+ return await super()._retry(method, url, headers, body, *args, **kwargs)+++def test_a_slow_turn_gets_a_first_byte_before_the_answer():+ """Past the grace window the stream commits and a heartbeat runs (#2465, #3079)."""+ hash_key = _stored_hash()+ with _Slow([_final_reply()], buffered_ccr_grace_seconds=0.01) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200+ assert resp.text.startswith(": ping\n\n")+ assert _events(resp.text)[-1] == "[DONE]"+++def test_an_error_after_the_commit_is_a_typed_stream_error():+ hash_key = _stored_hash()+ error = {"error": {"message": "slow down", "type": "rate_limit_error"}}+ with _Slow([(429, error)], buffered_ccr_grace_seconds=0.01) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, "the status was already committed"+ events = _events(resp.text)+ assert events[-1] == {"error": {"message": "slow down", "type": "rate_limit_error"}}+++def test_a_transport_failure_inside_the_grace_window_is_a_clean_502():+ hash_key = _stored_hash()+ with _Proxy([httpx.ConnectError("boom")], ccr_buffered_chat_streaming=True) as proxy:+ before = proxy.proxy.metrics.requests_failed+ resp = proxy.post(_request(hash_key))+ after = proxy.proxy.metrics.requests_failed++ assert resp.status_code == 502+ assert resp.json()["error"]["type"] == "server_error"+ assert after == before + 1+++def test_buffered_and_non_streaming_turns_account_identically():+ """Same turn both ways: the recorded numbers have to match (#3615, #3699)."""+ fields = (+ "status_code",+ "provider_input_tokens",+ "output_tokens",+ "tokens_saved",+ "cache_read_tokens",+ "cache_write_tokens",+ "uncached_input_tokens",+ )++ def run(stream: bool) -> dict[str, Any]:+ reset_compression_store()+ hash_key = _stored_hash()+ with _Proxy([_retrieve_reply(hash_key), _final_reply()]) as proxy:+ resp = proxy.post(_request(hash_key, stream=stream))+ assert resp.status_code == 200+ assert len(proxy.outcomes) == 1+ return {field: getattr(proxy.outcomes[0], field) for field in fields}++ buffered = run(stream=True)+ assert buffered == run(stream=False)+ assert buffered["provider_input_tokens"] == 500, "both upstream calls must be counted"+++def test_tools_stay_byte_stable_across_a_session_that_mixes_modes():+ """The toggle from #3815: stream and non-stream turns must send the same tools."""+ hash_key = _stored_hash()+ upstream = [_final_reply(), _final_reply(), _final_reply()]+ messages = _messages(hash_key)+ with _Proxy(upstream) as proxy:+ for stream in (False, True, False):+ resp = proxy.post(_request(hash_key, messages=messages, stream=stream))+ assert resp.status_code == 200, resp.text++ assert len(proxy.sent) == 3+ tools = [json.dumps(body["tools"]) for body in proxy.sent]+ assert tools[0] == tools[1] == tools[2]+ assert CCR_TOOL_NAME in tools[0]+++ ("status", "error_type"),+ [(429, "rate_limit_error"), (400, "invalid_request_error"), (500, "server_error")],+)+def test_failed_continuation_fails_closed_with_its_status(status, error_type):+ """The client must not get a tool it never declared, and backoff still has to fire."""+ hash_key = _stored_hash()+ upstream = [_retrieve_reply(hash_key), (status, {"error": {"message": "nope"}})]+ with _Proxy(upstream) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == status+ assert CCR_TOOL_NAME not in resp.text+ assert resp.json()["error"]["type"] == error_type+ assert "nope" not in resp.text, "the upstream error body is untrusted and stays in the proxy"+++def test_failed_continuation_after_the_commit_is_a_typed_stream_error():+ hash_key = _stored_hash()+ upstream = [_retrieve_reply(hash_key), (429, {"error": {"message": "nope"}})]+ with _Slow(upstream, buffered_ccr_grace_seconds=0.01) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, "the status was already committed"+ assert CCR_TOOL_NAME not in resp.text+ assert _events(resp.text)[-1]["error"]["type"] == "rate_limit_error"+++def test_an_expired_marker_is_still_answered_while_the_tool_is_resident():+ """The hash is gone but the model can still ask for it, and the client cannot answer."""+ hash_key = _stored_hash()+ messages = _messages(hash_key)+ upstream = [_final_reply(), _retrieve_reply(hash_key), _final_reply()]+ with _Proxy(upstream) as proxy:+ first = proxy.post(_request(hash_key, messages=messages, stream=False))+ assert first.status_code == 200, first.text++ reset_compression_store()+ resp = proxy.post(_request(hash_key, messages=messages))++ assert resp.status_code == 200, resp.text+ assert proxy.plain_streams == 0+ assert CCR_TOOL_NAME not in resp.text+ assert "there are 2 rows" in resp.text+++def test_a_bypassed_turn_keeps_its_stream():+ """Bypass skips injection, so there is nothing to answer and no reason to buffer."""+ hash_key = _stored_hash()+ with _Proxy([]) as proxy:+ resp = proxy.post(_request(hash_key), x_headroom_bypass="true")++ assert resp.text == "data: plain\n\n"+ assert proxy.sent == []+++def test_response_hooks_do_not_run_on_a_buffered_turn(monkeypatch):+ """Their on_request half never ran; it is decided before dispatch, like a stream."""+ import headroom.proxy.turn_hooks as turn_hooks++ calls: list[str] = []++ class _Hook:+ name = "buffered-only"++ def on_request(self, ctx): # noqa: ANN001+ calls.append("on_request")++ async def on_response(self, ctx, response, call_model): # noqa: ANN001+ calls.append("on_response")+ return None++ hash_key = _stored_hash()+ with _Proxy([_final_reply()]) as proxy:+ monkeypatch.setattr(turn_hooks, "_hooks", [_Hook()], raising=False)+ monkeypatch.setattr(turn_hooks, "registered_turn_hooks", lambda: [_Hook()])+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ assert calls == []+++def _mixed_reply(hash_key: str) -> dict:+ return _completion(+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ _tool_call(CCR_TOOL_NAME, {"hash": hash_key}),+ _tool_call("Read", {"file_path": "a.py"}, call_id="call_2"),+ ],+ },+ "tool_calls",+ USAGE_FIRST,+ )+++def test_retrieval_next_to_a_client_tool_lets_the_model_decide_again():+ """The unrun client call is dropped, and the continuation replaces the turn."""+ hash_key = _stored_hash()+ with _Proxy([_mixed_reply(hash_key), _final_reply()]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ assert CCR_TOOL_NAME not in resp.text+ assert "there are 2 rows" in resp.text+ replayed = proxy.sent[1]["messages"][-2]+ assert [call["function"]["name"] for call in replayed["tool_calls"]] == [CCR_TOOL_NAME]+++def test_mixed_turn_goes_back_whole_when_its_continuation_fails():+ """The model made no newer decision, so the client gets the turn it did make."""+ hash_key = _stored_hash()+ upstream = [_mixed_reply(hash_key), (500, {"error": {"message": "nope"}})]+ with _Proxy(upstream) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ choices = [chunk["choices"][0] for chunk in _chunks(resp.text) if chunk["choices"]]+ names = [+ choice["delta"]["tool_calls"][0]["function"]["name"]+ for choice in choices+ if "tool_calls" in choice["delta"]+ ]+ assert names == [CCR_TOOL_NAME, "Read"]+ assert choices[-1]["finish_reason"] == "tool_calls"+++def test_continuation_replays_what_the_provider_put_on_the_assistant_message():+ """DeepSeek rejects a tool-call turn replayed without its reasoning_content."""+ hash_key = _stored_hash()+ reply = _retrieve_reply(hash_key)+ message = reply["choices"][0]["message"]+ message["reasoning_content"] = "thinking it over"+ message["reasoning_details"] = [{"text": "t"}]+ message["refusal"] = None+ message["annotations"] = []+ with _Proxy([reply, _final_reply()]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ replayed = proxy.sent[1]["messages"][-2]+ assert replayed["role"] == "assistant"+ assert replayed["reasoning_content"] == "thinking it over"+ assert replayed["reasoning_details"] == [{"text": "t"}]+ assert "refusal" not in replayed, "empty fields stay out of the replayed message"+ assert "annotations" not in replayed+++class _RequiresReasoning(_Proxy):+ """An upstream that rejects a tool-call turn replayed without its reasoning."""++ async def _retry(self, method, url, headers, body, *args, **kwargs): # noqa: ANN001+ for message in body["messages"]:+ if message.get("tool_calls") and "reasoning_content" not in message:+ self.sent.append(json.loads(json.dumps(body)))+ return httpx.Response(400, json={"error": {"message": "reasoning_content missing"}})+ return await super()._retry(method, url, headers, body, *args, **kwargs)+++def test_a_provider_that_requires_reasoning_on_replay_gets_it():+ hash_key = _stored_hash()+ reply = _retrieve_reply(hash_key)+ reply["choices"][0]["message"]["reasoning_content"] = "required-replay"+ with _RequiresReasoning([reply, _final_reply()]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ assert "there are 2 rows" in resp.text+++ "payload",+ [+ {"error": {"message": "gateway said no"}},+ {"choices": []},+ {"choices": None},+ {"choices": "nope"},+ {"choices": [None]},+ {"choices": [{}]},+ {"choices": [{"finish_reason": "stop"}]},+ {"choices": [{"message": None, "finish_reason": "stop"}]},+ ],+)+def test_a_200_with_no_usable_choice_is_not_replayed_as_an_empty_stream(payload):+ hash_key = _stored_hash()+ with _Proxy([payload]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 502+ assert resp.json()["error"]["type"] == "upstream_protocol_error"+++def _as_sse(payload: Any) -> Response:+ return _buffered_chat_reply_as_sse(+ Response(json.dumps(payload).encode(), status_code=200),+ ccr_response_handler=CCRResponseHandler(),+ include_usage=False,+ continuation_status=None,+ request_id="test",+ )+++ "payload",+ [+ None,+ [],+ {"choices": [None]},+ {"choices": [{}]},+ {"choices": [{"message": "text"}]},+ ],+)+def test_the_helper_rejects_a_reply_with_no_usable_choice(payload):+ result = _as_sse(payload)++ assert result.status_code == 502+ assert json.loads(result.body)["error"]["type"] == "upstream_protocol_error"+++def test_a_refusal_is_a_valid_reply():+ hash_key = _stored_hash()+ reply = _completion(+ {"role": "assistant", "content": None, "refusal": "no"}, "stop", USAGE_FIRST+ )+ with _Proxy([reply]) as proxy:+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200, resp.text+ deltas = [chunk["choices"][0]["delta"] for chunk in _chunks(resp.text) if chunk["choices"]]+ assert {"refusal": "no"} in deltas+++def test_a_buffered_reply_never_lands_in_the_response_cache():+ """It was built for a stream:true request; a later caller must not get it (#3019)."""+ hash_key = _stored_hash()+ with _Proxy([_final_reply()], cache_enabled=True) as proxy:+ calls: list[Any] = []++ async def _set(*args: Any, **kwargs: Any) -> None:+ calls.append(args)++ proxy.proxy.cache.set = _set+ resp = proxy.post(_request(hash_key))++ assert resp.status_code == 200+ assert calls == []+++ ("overrides", "expected"),+ [+ ({}, True),+ ({"body": {"stream": False, "tools": [CLIENT_TOOL]}}, False),+ ({"body": {"stream": True, "tools": []}}, False),+ ({"body": {"stream": True}}, False),+ ({"body": {"stream": True, "tools": [CLIENT_TOOL], "n": 2}}, False),+ ({"body": {"stream": True, "tools": [CLIENT_TOOL], "n": 1}}, True),+ ({"body": {"stream": True, "tools": [CLIENT_TOOL], "logprobs": True}}, False),+ ({"enabled": False}, False),+ ({"ccr_response_handler_enabled": False}, False),+ ({"ccr_inject_tool": False}, False),+ ({"routed_to_backend": True}, False),+ ({"upstream_base_url": "https://opencode.ai/zen/v1"}, False),+ ],+)+def test_eligibility(overrides, expected):+ args: dict[str, Any] = {+ "body": {"stream": True, "tools": [CLIENT_TOOL]},+ "enabled": True,+ "ccr_response_handler_enabled": True,+ "ccr_inject_tool": True,+ "routed_to_backend": False,+ "upstream_base_url": "https://api.openai.com",+ **overrides,+ }+ assert _is_openai_chat_stream_ccr_eligible(**args) is expected+++def test_taking_stream_options_leaves_other_fields_alone():+ body = {"stream": True, "stream_options": {"include_usage": True}, "temperature": 0}+ assert _take_stream_options_for_buffered_chat(body) is True+ assert body == {"stream": True, "temperature": 0}+++def test_frames_keep_the_fields_a_client_replays_on_its_next_turn():+ call = {+ **_tool_call("Read", {"file_path": "a.py"}),+ "extra_content": {"google": {"thought_signature": "sig"}},+ }+ reply = _completion(+ {+ "role": "assistant",+ "reasoning_content": "thinking it over",+ "reasoning_details": [{"text": "t"}],+ "refusal": None,+ "content": "reading",+ "tool_calls": [call],+ },+ "tool_calls",+ USAGE_FIRST,+ )+ frames = StreamingCCRHandler.openai_chat_sse_frames(reply, include_usage=False)+ deltas = [+ chunk["choices"][0]["delta"]+ for chunk in _chunks(b"".join(frames).decode())+ if chunk["choices"]+ ]++ assert {"reasoning_content": "thinking it over", "reasoning_details": [{"text": "t"}]} in deltas+ tool_delta = next(delta["tool_calls"][0] for delta in deltas if "tool_calls" in delta)+ assert tool_delta["extra_content"] == {"google": {"thought_signature": "sig"}}+ assert tool_delta["index"] == 0+ assert frames[-1] == b"data: [DONE]\n\n"