Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): make StreamingCCRHandler work on OpenAI streams (#3069)
Parideboy7ef736fb1a88
diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.pyindex 7cdea7a49..a4acfeb3d 100644--- a/headroom/ccr/response_handler.py+++ b/headroom/ccr/response_handler.py@@ -571,11 +571,23 @@ class StreamingCCRBuffer:chunks: list[bytes] = field(default_factory=list)detected_ccr: bool = Falsecomplete_response: dict[str, Any] | None = None+ provider: str = "anthropic"- # Patterns to detect tool_use in stream+ # Wire markers for the start of a tool call. Anthropic streams+ # `"type":"tool_use"` content blocks; OpenAI-compatible streams carry a+ # `"tool_calls"` array inside `choices[].delta` and never emit the+ # Anthropic marker, so scanning only for the latter meant CCR was never+ # detected on an OpenAI stream._tool_use_start: bytes = b'"type":"tool_use"'+ _openai_tool_use_start: bytes = b'"tool_calls"'_ccr_tool_pattern: bytes = f'"{CCR_TOOL_NAME}"'.encode()+ def _tool_call_marker(self) -> bytes:+ """The provider's on-the-wire marker for the start of a tool call."""+ if self.provider == "anthropic":+ return self._tool_use_start+ return self._openai_tool_use_start+def add_chunk(self, chunk: bytes) -> bool:"""Add a chunk and check for CCR tool calls.@@ -587,7 +599,7 @@ class StreamingCCRBuffer:# Quick check: does accumulated content contain CCR tool?accumulated = b"".join(self.chunks)- if self._tool_use_start in accumulated and self._ccr_tool_pattern in accumulated:+ if self._tool_call_marker() in accumulated and self._ccr_tool_pattern in accumulated:self.detected_ccr = Truereturn True@@ -622,7 +634,7 @@ class StreamingCCRHandler:) -> None:self.response_handler = response_handlerself.provider = provider- self.buffer = StreamingCCRBuffer()+ self.buffer = StreamingCCRBuffer(provider=provider)async def process_stream(self,@@ -648,8 +660,12 @@ class StreamingCCRHandler:Response chunks (possibly from continuation response)."""# Phase 1: Initial detection- # Buffer chunks until we can determine if there's a CCR call- detection_complete = False+ # Buffer chunks until we can determine if there's a CCR call.+ #+ # The end-of-stream marker is provider-specific. Anthropic signals the+ # terminal state with `stop_reason` in `message_delta`; OpenAI-compatible+ # streams have no such field and terminate with the `[DONE]` sentinel.+ end_marker = b'"stop_reason"' if self.provider == "anthropic" else b"data: [DONE]"async for chunk in stream_iterator:self.buffer.add_chunk(chunk)@@ -660,9 +676,7 @@ class StreamingCCRHandler:accumulated = self.buffer.get_accumulated()# Look for stream end markers- if b'"stop_reason"' in accumulated:- detection_complete = True-+ if end_marker in accumulated:if self.buffer.detected_ccr:# CCR detected - need to handlebreak@@ -679,13 +693,15 @@ class StreamingCCRHandler:yield buffered_chunkself.buffer.clear()- # Continue streaming rest of response- if not detection_complete and not self.buffer.detected_ccr:- async for chunk in stream_iterator:- if self.buffer.detected_ccr:- self.buffer.add_chunk(chunk)- else:- yield chunk+ # The end marker is not guaranteed to arrive: upstream can truncate, a+ # provider can omit the sentinel, or the stream can be a shape this+ # detector does not recognise. Anything still buffered once the source+ # iterator is exhausted is real response data the client has never+ # seen, so flush it instead of dropping it.+ if not self.buffer.detected_ccr and self.buffer.chunks:+ for buffered_chunk in self.buffer.chunks:+ yield buffered_chunk+ self.buffer.clear()# Phase 2: Handle CCR if detectedif self.buffer.detected_ccr:@@ -903,13 +919,38 @@ class StreamingCCRHandler:}tool_calls_map: dict[int, dict[str, Any]] = {}+ finish_reason: str | None = None+ envelope: dict[str, Any] = {}+ usage: Any = Nonefor event in events:- choices = event.get("choices", [])- if not choices:+ # Carry the chunk envelope through. Dropping it left the+ # reconstructed body without `id`, `model`, `created` or `usage`,+ # which downstream middleware reads for routing and metering.+ for key in ("id", "created", "model", "system_fingerprint"):+ value = event.get(key)+ if value is not None:+ envelope[key] = value+ if event.get("usage") is not None:+ usage = event["usage"]++ choices = event.get("choices")+ if not isinstance(choices, list) or not choices:continue+ choice = choices[0]+ if not isinstance(choice, dict):+ continue++ # `finish_reason` is null on every chunk but the last, so keep the+ # most recent non-null value rather than the first one seen.+ if choice.get("finish_reason") is not None:+ finish_reason = choice["finish_reason"]- delta = choices[0].get("delta", {})+ # Some OpenAI-compatible providers send `"delta": null` on the+ # terminal chunk instead of an empty object.+ delta = choice.get("delta")+ if not isinstance(delta, dict):+ delta = {}if "content" in delta and delta["content"]:message["content"] = (message.get("content") or "") + delta["content"]@@ -944,14 +985,94 @@ class StreamingCCRHandler:tc["function"]["arguments"] += fn["arguments"]message["tool_calls"] = [tool_calls_map[i] for i in sorted(tool_calls_map.keys())]- if not message["tool_calls"]:+ has_tool_calls = bool(message["tool_calls"])+ if not has_tool_calls:del message["tool_calls"]if not message["content"]:message["content"] = None- return {- "choices": [{"message": message, "finish_reason": "stop"}],+ # OpenAI requires `finish_reason: "tool_calls"` whenever the message+ # carries tool calls. This was hardcoded to "stop", which tells any+ # client that drives its agent loop off `finish_reason` that the turn+ # is over, so the reconstructed tool calls were never executed.+ if has_tool_calls:+ finish_reason = "tool_calls"+ elif finish_reason is None:+ finish_reason = "stop"++ response: dict[str, Any] = {+ "object": "chat.completion",+ **envelope,+ "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],}+ if usage is not None:+ response["usage"] = usage+ return response++ def _openai_response_to_chunks(self, response: dict[str, Any]) -> list[bytes]:+ """Split a non-streaming ``chat.completion`` body into SSE chunk frames.++ A streaming client reads ``choices[].delta``, not ``choices[].message``.+ Serialising the reconstructed non-streaming body into a single SSE frame+ produced a stream in which both the text and the tool calls were+ invisible to the client.+ """+ choices = response.get("choices")+ choice = choices[0] if isinstance(choices, list) and choices else {}+ if not isinstance(choice, dict):+ choice = {}+ message = choice.get("message")+ if not isinstance(message, dict):+ message = {}+ finish_reason = choice.get("finish_reason") or "stop"++ base: dict[str, Any] = {"object": "chat.completion.chunk"}+ for key in ("id", "created", "model", "system_fingerprint"):+ if response.get(key) is not None:+ base[key] = response[key]++ def frame(delta: dict[str, Any], reason: str | None) -> bytes:+ payload = {+ **base,+ "choices": [{"index": 0, "delta": delta, "finish_reason": reason}],+ }+ return f"data: {json.dumps(payload)}\n\n".encode()++ frames = [frame({"role": message.get("role") or "assistant"}, None)]++ content = message.get("content")+ if content:+ frames.append(frame({"content": content}, None))++ tool_calls = message.get("tool_calls")+ if isinstance(tool_calls, list):+ for index, tool_call in enumerate(tool_calls):+ if not isinstance(tool_call, dict):+ continue+ function = tool_call.get("function")+ if not isinstance(function, dict):+ function = {}+ frames.append(+ frame(+ {+ "tool_calls": [+ {+ "index": index,+ "id": tool_call.get("id", ""),+ "type": tool_call.get("type", "function"),+ "function": {+ "name": function.get("name", ""),+ "arguments": function.get("arguments", ""),+ },+ }+ ]+ },+ None,+ )+ )++ frames.append(frame({}, finish_reason))+ return framesasync def _response_to_sse(self,@@ -968,6 +1089,7 @@ class StreamingCCRHandler:for chunk in StreamingMixin()._response_to_sse(response, "anthropic"):yield chunkelse:- # OpenAI SSE format- yield f"data: {json.dumps(response)}\n\n".encode()+ # OpenAI SSE format: `chat.completion.chunk` frames, then [DONE].+ for chunk in self._openai_response_to_chunks(response):+ yield chunkyield b"data: [DONE]\n\n"diff --git a/tests/test_ccr_response_handler_extra.py b/tests/test_ccr_response_handler_extra.pyindex 1e8d97d93..c2754c413 100644--- a/tests/test_ccr_response_handler_extra.py+++ b/tests/test_ccr_response_handler_extra.py@@ -445,7 +445,15 @@ async def test_streaming_handler_falls_back_to_buffer_on_processing_error(lambda data: (_ for _ in ()).throw(RuntimeError("parse failed")),)- chunks = [b'{"type":"tool_use","name":"headroom_retrieve"', b',"stop_reason":"tool_use"}']+ # Real OpenAI wire shape: a `tool_calls` delta naming the CCR tool, then+ # the `[DONE]` sentinel. This test previously fed Anthropic-shaped bytes to+ # an ``openai`` handler, so it never reached the OpenAI detection path.+ chunks = [+ b'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,'+ b'"id":"call_1","function":{"name":"headroom_retrieve",'+ b'"arguments":"{}"}}]}}]}\n\n',+ b"data: [DONE]\n\n",+ ]streamed = [chunkasync for chunk in handler.process_stream(_async_iter(chunks), [], None, lambda m, t: None)@@ -462,7 +470,14 @@ async def test_response_to_sse_formats() -> None:openai = StreamingCCRHandler(CCRResponseHandler(), provider="openai")openai_chunks = [chunk async for chunk in openai._response_to_sse({"choices": []})]- assert openai_chunks == [b'data: {"choices": []}\n\n', b"data: [DONE]\n\n"]+ # An empty body still produces well-formed chunk frames (role, then a+ # terminal frame carrying finish_reason) rather than a single non-streaming+ # body a streaming client cannot read.+ assert openai_chunks[-1] == b"data: [DONE]\n\n"+ frames = [json.loads(chunk.decode()[len("data: ") :]) for chunk in openai_chunks[:-1]]+ assert [frame["object"] for frame in frames] == ["chat.completion.chunk"] * 2+ assert frames[0]["choices"][0]["delta"] == {"role": "assistant"}+ assert frames[-1]["choices"][0]["finish_reason"] == "stop"@pytest.mark.asyncio@@ -533,3 +548,229 @@ def test_reconstruct_server_tool_use_input_from_partial_json() -> None:assert block["type"] == "server_tool_use"assert block["input"] == {"query": "x"}assert "_partial_json" not in block+++def _openai_chunk(delta: dict[str, Any], finish_reason: str | None = None) -> bytes:+ """One `chat.completion.chunk` SSE frame in the shape a real backend sends."""+ payload = {+ "id": "chatcmpl_1",+ "object": "chat.completion.chunk",+ "created": 1700000000,+ "model": "gpt-4o-mini",+ "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],+ }+ return f"data: {json.dumps(payload)}\n\n".encode()+++def test_streaming_buffer_detects_ccr_in_openai_tool_calls_delta() -> None:+ # An OpenAI-compatible stream never emits Anthropic's `"type":"tool_use"`+ # marker; its tool calls arrive as a `tool_calls` array inside+ # `choices[].delta`. Scanning only for the Anthropic marker meant CCR was+ # never detected on this provider.+ buffer = StreamingCCRBuffer(provider="openai")++ assert buffer.add_chunk(_openai_chunk({"role": "assistant"})) is False+ detected = buffer.add_chunk(+ _openai_chunk(+ {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_1",+ "function": {"name": CCR_TOOL_NAME, "arguments": ""},+ }+ ]+ }+ )+ )++ assert detected is True+ assert buffer.detected_ccr is True++ # A non-CCR tool call on the same provider must not trip detection.+ other = StreamingCCRBuffer(provider="openai")+ assert (+ other.add_chunk(+ _openai_chunk(+ {"tool_calls": [{"index": 0, "id": "c", "function": {"name": "other_tool"}}]}+ )+ )+ is False+ )+ assert other.detected_ccr is False+++async def test_openai_stream_without_ccr_yields_every_chunk() -> None:+ # A short OpenAI stream with no CCR call must pass through byte for byte.+ # End-of-stream was detected by scanning for Anthropic's `stop_reason`,+ # which an OpenAI stream never contains, so nothing was ever flushed and+ # the client received an empty response.+ handler = StreamingCCRHandler(CCRResponseHandler(), provider="openai")+ chunks = [+ _openai_chunk({"role": "assistant"}),+ _openai_chunk({"content": "hello "}),+ _openai_chunk({"content": "world"}),+ _openai_chunk({}, finish_reason="stop"),+ b"data: [DONE]\n\n",+ ]++ streamed = [+ chunk+ async for chunk in handler.process_stream(_async_iter(chunks), [], None, lambda m, t: None)+ ]++ assert streamed == chunks+++async def test_openai_stream_past_flush_threshold_keeps_the_tail() -> None:+ # Past 10 000 buffered bytes the handler flushes in batches. Without an+ # end-of-stream match the final sub-threshold batch was never flushed, so+ # a long response visibly stopped mid-sentence.+ handler = StreamingCCRHandler(CCRResponseHandler(), provider="openai")+ chunks = [+ _openai_chunk({"role": "assistant"}),+ _openai_chunk({"content": "x" * 11000}),+ _openai_chunk({"content": "the tail that used to be dropped"}),+ _openai_chunk({}, finish_reason="stop"),+ b"data: [DONE]\n\n",+ ]++ streamed = [+ chunk+ async for chunk in handler.process_stream(_async_iter(chunks), [], None, lambda m, t: None)+ ]++ assert streamed == chunks+ assert b"the tail that used to be dropped" in b"".join(streamed)+++async def test_openai_stream_without_done_sentinel_still_flushes() -> None:+ # Upstream can truncate before `[DONE]`, and some gateways omit it. Bytes+ # left in the buffer when the source iterator is exhausted are real+ # response data, so they are flushed rather than discarded.+ handler = StreamingCCRHandler(CCRResponseHandler(), provider="openai")+ chunks = [_openai_chunk({"role": "assistant"}), _openai_chunk({"content": "partial answer"})]++ streamed = [+ chunk+ async for chunk in handler.process_stream(_async_iter(chunks), [], None, lambda m, t: None)+ ]++ assert streamed == chunks+++def test_reconstruct_openai_response_marks_tool_calls_finish_reason() -> None:+ # OpenAI requires `finish_reason: "tool_calls"` when the message carries+ # tool calls. It was hardcoded to "stop", so a client driving its agent+ # loop off `finish_reason` ended the turn instead of running the tools.+ handler = StreamingCCRHandler(CCRResponseHandler(), provider="openai")++ parsed = handler._reconstruct_openai_response(+ [+ {"id": "chatcmpl_1", "model": "gpt-4o-mini", "created": 1700000000},+ {+ "choices": [+ {+ "delta": {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_1",+ "function": {+ "name": CCR_TOOL_NAME,+ "arguments": '{"hash":"abc"}',+ },+ }+ ]+ },+ "finish_reason": None,+ }+ ]+ },+ {+ "choices": [{"delta": None, "finish_reason": "tool_calls"}],+ "usage": {"prompt_tokens": 12, "completion_tokens": 3},+ },+ ]+ )++ assert parsed["choices"][0]["finish_reason"] == "tool_calls"+ # The chunk envelope is carried through so the reconstructed body is a+ # valid `chat.completion` rather than a bare `choices` list.+ assert parsed["object"] == "chat.completion"+ assert parsed["id"] == "chatcmpl_1"+ assert parsed["model"] == "gpt-4o-mini"+ assert parsed["created"] == 1700000000+ assert parsed["usage"] == {"prompt_tokens": 12, "completion_tokens": 3}+++def test_reconstruct_openai_response_keeps_upstream_finish_reason() -> None:+ # With no tool calls, the upstream reason is preserved instead of being+ # rewritten to "stop": a truncated turn must stay reported as truncated.+ handler = StreamingCCRHandler(CCRResponseHandler(), provider="openai")++ parsed = handler._reconstruct_openai_response(+ [+ {"choices": [{"delta": {"content": "half an ans"}, "finish_reason": None}]},+ {"choices": [{"delta": {}, "finish_reason": "length"}]},+ ]+ )++ assert parsed["choices"][0]["finish_reason"] == "length"+ assert parsed["choices"][0]["message"]["content"] == "half an ans"++ # And an absent reason still defaults to "stop".+ defaulted = handler._reconstruct_openai_response(+ [{"choices": [{"delta": {"content": "hi"}}]}],+ )+ assert defaulted["choices"][0]["finish_reason"] == "stop"+++async def test_response_to_sse_emits_openai_chunk_frames() -> None:+ # A streaming client reads `choices[].delta`. Re-serialising the+ # reconstructed non-streaming body (`choices[].message`) into one SSE frame+ # made both the content and the tool calls invisible to it.+ handler = StreamingCCRHandler(CCRResponseHandler(), provider="openai")+ response = {+ "id": "chatcmpl_2",+ "object": "chat.completion",+ "created": 1700000001,+ "model": "gpt-4o-mini",+ "choices": [+ {+ "index": 0,+ "message": {+ "role": "assistant",+ "content": "done",+ "tool_calls": [+ {+ "id": "call_9",+ "type": "function",+ "function": {"name": "do_thing", "arguments": '{"a":1}'},+ }+ ],+ },+ "finish_reason": "tool_calls",+ }+ ],+ }++ chunks = [chunk async for chunk in handler._response_to_sse(response)]++ assert chunks[-1] == b"data: [DONE]\n\n"+ frames = [json.loads(chunk.decode()[len("data: ") :]) for chunk in chunks[:-1]]+ assert all(frame["object"] == "chat.completion.chunk" for frame in frames)+ assert all("delta" in frame["choices"][0] for frame in frames)+ assert all(frame["id"] == "chatcmpl_2" for frame in frames)++ deltas = [frame["choices"][0]["delta"] for frame in frames]+ assert deltas[0] == {"role": "assistant"}+ assert deltas[1] == {"content": "done"}+ assert deltas[2]["tool_calls"][0]["id"] == "call_9"+ assert deltas[2]["tool_calls"][0]["index"] == 0+ assert deltas[2]["tool_calls"][0]["function"] == {"name": "do_thing", "arguments": '{"a":1}'}+ assert frames[-1]["choices"][0]["finish_reason"] == "tool_calls"