Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): buffer a stream only for a marker new this turn (#4607)
SD946a70cfcbdc
diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.pyindex c027d2c13..e9e1554bb 100644--- a/headroom/ccr/response_handler.py+++ b/headroom/ccr/response_handler.py@@ -132,6 +132,23 @@ class CCRResponseHandler:self._retrieval_count = 0self._retrieval_count_lock = __import__("threading").Lock()+ def serve_withheld_retrievals(self, tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Run Anthropic ``headroom_retrieve`` calls withheld from a stream.++ Returns one ``tool_result`` block per call that names a hash, in call+ order, for the continuation round the streaming path sends upstream.+ A call without a parsable hash gets no result, so the caller can tell+ the round cannot be replayed.+ """+ ccr_calls, _ = self._parse_ccr_tool_calls({"content": tool_calls}, "anthropic")+ if not ccr_calls:+ return []+ with self._retrieval_count_lock:+ self._retrieval_count += len(ccr_calls)+ results = [self._execute_retrieval(call) for call in ccr_calls]+ message = self._create_tool_result_message(results, "anthropic")+ return list(message.get("content", []))+def has_ccr_tool_calls(self,response: dict[str, Any],diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex b6c5e10a3..f10d86c7b 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -630,28 +630,41 @@ class AnthropicHandlerMixin:return False@staticmethod- def _outgoing_body_has_redeemable_marker(body: Any) -> bool:- """Does the body about to be sent carry a marker retrieval could expand?+ def _outgoing_redeemable_markers(+ body: Any,+ previous_forwarded_messages: list[dict[str, Any]] | None = None,+ ) -> tuple[bool, bool]:+ """Which owned markers does the outgoing body carry?++ Returns ``(any_owned, new_owned)``. ``headroom_retrieve`` exists only to+ expand a ``<<ccr:...>>`` marker, so a request carrying none cannot use+ it (#3071).- ``headroom_retrieve`` exists only to expand a ``<<ccr:...>>`` marker, so- a request carrying none cannot benefit from the buffered path (#3071).+ ``new_owned`` is a marker the model has not seen yet: one absent from+ what this session forwarded last turn. With no forwarded history to+ compare against, every owned marker counts as new. "New" is decided+ against the forwarded history, not by message position: token mode can+ compress an earlier tool result on this request, so a marker first+ seen now may sit anywhere in the history.Ownership is verified rather than shape-matched: the marker shape is notunique to Headroom, and adopting another context tool's hash would sendthe model to an endpoint that is guaranteed to miss (#2836). A hash thatsurvives ``verify_ownership`` is redeemable right now.- Errors are swallowed deliberately and answered ``True``. This gates a- wire-format decision, and the safe direction on an unexpected message- shape is the long-standing buffered behavior, not a silent change.+ Errors are swallowed deliberately and answered ``(True, True)``. This+ gates a wire-format decision, and the safe direction on an unexpected+ message shape is the long-standing buffered behavior, not a silent+ change."""if not isinstance(body, dict):- return True+ return True, Truemessages = body.get("messages")if not isinstance(messages, list) or not messages:- return False+ return False, Falsetry:from headroom.ccr.tool_injection import CCRToolInjector+ from headroom.proxy.helpers import has_new_ccr_markersprobe = CCRToolInjector(provider="anthropic",@@ -660,12 +673,27 @@ class AnthropicHandlerMixin:)probe.scan_for_markers(messages)if not probe.detected_hashes:- return False+ return False, Falseprobe.verify_ownership()- return bool(probe.detected_hashes)+ if not probe.detected_hashes:+ return False, False+ return True, has_new_ccr_markers(+ current_detected_hashes=list(probe.detected_hashes),+ previous_forwarded_messages=previous_forwarded_messages,+ provider="anthropic",+ )except Exception: # pragma: no cover - defensivelogger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)- return True+ return True, True++ @classmethod+ def _outgoing_body_has_redeemable_marker(+ cls,+ body: Any,+ previous_forwarded_messages: list[dict[str, Any]] | None = None,+ ) -> bool:+ """Does the outgoing body show the model an owned marker it has not seen?"""+ return cls._outgoing_redeemable_markers(body, previous_forwarded_messages)[1]@staticmethoddef _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:@@ -4340,19 +4368,34 @@ class AnthropicHandlerMixin:and ccr_response_handler_enabledand has_ccr_retrieve_tool(tools if tools is not None else body.get("tools")))- buffered_retrieval_can_help = (- retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body)+ #+ # A marker the model already saw on an earlier turn can still be+ # retrieved, so it does not justify buffering every later turn+ # (#3587), but it does mean a retrieval call may come. Those+ # turns stream with ``headroom_retrieve`` withheld from the+ # client and served by the proxy in a continuation round.+ owned_marker, new_owned_marker = (+ self._outgoing_redeemable_markers(body, previous_forwarded_messages)+ if retrieve_tool_is_offered+ else (False, False))- wants_buffered_stream_ccr = bool(buffered_retrieval_can_help)- if retrieve_tool_is_offered and not buffered_retrieval_can_help:+ wants_buffered_stream_ccr = bool(new_owned_marker)+ if retrieve_tool_is_offered and not owned_marker:logger.info(f"[{request_id}] CCR: headroom_retrieve is resident but this ""request carries no redeemable marker, so server-side ""retrieval cannot fire; keeping the streaming path (#3071)")+ elif retrieve_tool_is_offered and not new_owned_marker:+ logger.info(+ f"[{request_id}] CCR: every redeemable marker was already "+ "forwarded; streaming with headroom_retrieve served by the "+ "proxy (#3587)"+ )buffered_stream_ccr = (wants_buffered_stream_ccr and not outbound_locked_to_client_bytes)+ serve_ccr_retrieval_in_stream = bool(owned_marker) and not buffered_stream_ccrif wants_buffered_stream_ccr and outbound_locked_to_client_bytes:logger.info(f"[{request_id}] CCR: signed thinking blocks force byte-faithful "@@ -4526,6 +4569,7 @@ class AnthropicHandlerMixin:session_key=session_key,server_memory_tool_names=server_memory_tool_names,client_beta=client_anthropic_beta or "",+ serve_ccr_retrieval=serve_ccr_retrieval_in_stream,)else:# Whatever set it — the client's own ``stream: false`` ordiff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex d51b3e2fb..ddd1680f7 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -520,17 +520,26 @@ class StreamingMixin:server_memory_tool_names: frozenset[str],stream_state: dict[str, Any],request_id: str,+ serve_retrieval: bool = False,) -> AsyncIterator[bytes]:- """Execute withheld memory tool calls and stream continuation rounds.+ """Execute withheld proxy tool calls and stream continuation rounds.``memory_filter`` has forwarded one upstream round to the client minus- the proxy-owned memory tool calls and the message tail. When the round- stopped on memory tool calls alone, run them, send the results upstream+ the proxy-owned tool calls and the message tail. When the round+ stopped on proxy tool calls alone, run them, send the results upstreamas the next turn, and stream that round's blocks into the same clientmessage. Otherwise end the client message. A round that also called aclient tool still has its memory calls executed, but the turn goes backto the client, which cannot carry the memory results.++ With ``serve_retrieval``, ``headroom_retrieve`` is a proxy tool too: the+ client never declared it, so a call that reached it would fail (#3587).+ The filter then holds client tool calls until the round ends, and a+ round that retrieved alongside a client tool is continued without the+ unrun client calls, for the model to re-issue with the content in+ hand. That is what the buffered path does with the same round."""+ from headroom.ccr.tool_injection import CCR_TOOL_NAMEfrom headroom.proxy.helpers import MAX_SSE_BUFFER_SIZEtry:@@ -541,18 +550,47 @@ class StreamingMixin:if not isinstance(messages, list):messages = Noneheaders = {k: v for k, v in outbound_headers.items() if k.lower() != "content-length"}+ proxy_tool_names = server_memory_tool_names | (+ frozenset({CCR_TOOL_NAME}) if serve_retrieval else frozenset()+ )- def is_memory_call(block: Any) -> bool:+ def is_proxy_call(block: Any) -> bool:return (isinstance(block, dict)and block.get("type") == "tool_use"- and block.get("name") in server_memory_tool_names+ and block.get("name") in proxy_tool_names+ )++ def is_client_call(block: Any) -> bool:+ return (+ isinstance(block, dict)+ and block.get("type") == "tool_use"+ and not is_proxy_call(block))rounds = 0# What the client received across rounds; the prefix tracker must see# this, not round one, because it is what the client sends back.visible_content: list[Any] = []++ def record_visible(+ round_response: dict[str, Any] | None,+ round_content: list[Any],+ *,+ dropped_client_calls: bool,+ ) -> None:+ if round_response is None:+ return+ visible_content.extend(+ block+ for block in round_content+ if not is_proxy_call(block) and not (dropped_client_calls and is_client_call(block))+ )+ stream_state["memory_client_message"] = {+ **round_response,+ "content": list(visible_content),+ }+# Usage of rounds whose message_delta the client never receives.prior_usage: dict[str, int] = {}while True:@@ -560,13 +598,9 @@ class StreamingMixin:if response is not None:round_content = response.get("content")content = round_content if isinstance(round_content, list) else []- visible_content.extend(block for block in content if not is_memory_call(block))- stream_state["memory_client_message"] = {- **response,- "content": list(visible_content),- }if not memory_filter.hid_tool_calls:+ record_visible(response, content, dropped_client_calls=False)for frame in memory_filter.closing_frames(prior_usage=prior_usage):yield framereturn@@ -574,7 +608,11 @@ class StreamingMixin:# Run the calls the filter recorded, so they are not lost when the# round could not be reconstructed. Only the proxy's own calls: a# memory-named tool the client declared is the client's to run.- memory_calls = memory_filter.hidden_calls()+ hidden_calls = memory_filter.hidden_calls()+ memory_calls = [c for c in hidden_calls if c.get("name") in server_memory_tool_names]+ retrieval_calls = [+ c for c in hidden_calls if serve_retrieval and c.get("name") == CCR_TOOL_NAME+ ]tool_results: list[dict[str, Any]] = []if memory_calls and memory_user_id is not None and self.memory_handler is not None:tool_results = await self.memory_handler.handle_memory_tool_calls(@@ -583,24 +621,38 @@ class StreamingMixin:"anthropic",request_context=memory_request_ctx,)- logger.info(- f"[{request_id}] Memory: Executed {len(tool_results)}/"- f"{len(memory_filter.hidden_tool_names)} proxy-handled tool call(s) "- "withheld from the SSE stream"- )+ logger.info(+ f"[{request_id}] Memory: Executed {len(tool_results)}/"+ f"{len(memory_calls)} proxy-handled tool call(s) "+ "withheld from the SSE stream"+ )+ ccr_handler = getattr(self, "ccr_response_handler", None)+ if retrieval_calls and ccr_handler is not None:+ retrieval_results = ccr_handler.serve_withheld_retrievals(retrieval_calls)+ tool_results = [*tool_results, *retrieval_results]+ logger.info(+ f"[{request_id}] CCR: Served {len(retrieval_results)}/"+ f"{len(retrieval_calls)} headroom_retrieve call(s) withheld from "+ "the SSE stream"+ )replayable = response is not None and len(tool_results) == sum(- 1 for block in content if is_memory_call(block)+ 1 for block in content if is_proxy_call(block))+ # Unrun client calls can be dropped only while the client has not+ # seen them, and only for a retrieval: memory results alone do not+ # give the model a reason to decide again.+ drop_client_calls = bool(retrieval_calls) and memory_filter.holds_client_tool_usecan_continue = (memory_filter.stop_reason == "tool_use"and not memory_filter.visible_tool_use+ and (drop_client_calls or not memory_filter.holds_client_tool_use)and messages is not Noneand replayableand rounds < self._MEMORY_CONTINUATION_MAX_ROUNDS)if not can_continue:- if memory_filter.visible_tool_use:+ if memory_filter.visible_tool_use or memory_filter.holds_client_tool_use:logger.info(f"[{request_id}] Memory: Round also called a client tool; ""returning the turn to the client"@@ -608,18 +660,31 @@ class StreamingMixin:elif rounds >= self._MEMORY_CONTINUATION_MAX_ROUNDS:logger.warning(f"[{request_id}] Memory: Stopped after {rounds} continuation "- "round(s) that only called memory tools"+ "round(s) that only called proxy tools")elif not replayable:logger.warning(f"[{request_id}] Memory: Could not rebuild the round for a "- "continuation; ending the turn after running its memory calls"+ "continuation; ending the turn after running its proxy calls")+ record_visible(response, content, dropped_client_calls=False)for frame in memory_filter.closing_frames(prior_usage=prior_usage):yield framereturnassert response is not None and messages is not None and isinstance(base_body, dict)+ record_visible(response, content, dropped_client_calls=drop_client_calls)+ # Blocks held after the first tool call (rarely any but the calls+ # themselves) still belong to the client's message.+ for frame in memory_filter.release_held(drop_tool_use=drop_client_calls):+ yield frame+ if drop_client_calls:+ logger.info(+ f"[{request_id}] CCR: Round called headroom_retrieve alongside a "+ "client tool; serving the retrieval and dropping the unrun client "+ "call(s) for the model to re-issue"+ )+ content = [block for block in content if not is_client_call(block)]round_usage = response.get("usage")if isinstance(round_usage, dict):for key in _ROUND_USAGE_KEYS:@@ -647,9 +712,10 @@ class StreamingMixin:yield _upstream_error_frame(error_bytes, request_id)returnmemory_filter = MemoryToolStreamFilter(- server_memory_tool_names,+ proxy_tool_names,index_offset=memory_filter.next_index,forward_message_start=False,+ hold_tool_use=serve_retrieval,)# Same cap as the first round's buffer; past it the round is# still streamed, just not rebuilt for a further continuation.@@ -1082,6 +1148,7 @@ class StreamingMixin:savings_measure: ItemMeasure | None = None,server_memory_tool_names: frozenset[str] | None = None,client_beta: str | None = None,+ serve_ccr_retrieval: bool = False,) -> Response | StreamingResponse:"""Stream response with metrics tracking and memory tool handling.@@ -1130,6 +1197,7 @@ class StreamingMixin:savings_measure=savings_measure,server_memory_tool_names=server_memory_tool_names,client_beta=client_beta,+ serve_ccr_retrieval=serve_ccr_retrieval,)except (Exception, asyncio.CancelledError):self._cleanup_mid_turn_stream(session_key)@@ -1165,6 +1233,7 @@ class StreamingMixin:savings_measure: ItemMeasure | None = None,server_memory_tool_names: frozenset[str] | None = None,client_beta: str | None = None,+ serve_ccr_retrieval: bool = False,) -> Response | StreamingResponse:"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""from fastapi.responses import Response, StreamingResponse@@ -1339,6 +1408,9 @@ class StreamingMixin:and self.memory_handler is not Noneand provider == "anthropic")+ # Withholding headroom_retrieve needs the Anthropic SSE shape the+ # filter and the continuation round are written for.+ serve_ccr_retrieval = serve_ccr_retrieval and provider == "anthropic"# Open connection before generator to capture upstream response headers# (needed to forward ratelimit headers to the client via StreamingResponse)@@ -1672,11 +1744,23 @@ class StreamingMixin:pending_messages: list[dict] = []# Proxy-injected memory tools are invisible to the client, so their# tool_use blocks must never reach it (GH #2195).+ from headroom.ccr.tool_injection import CCR_TOOL_NAME++ # Memory tools are withheld only when this request can run them;+ # the continuation must classify calls exactly as the filter did.+ filter_memory_tool_names = (+ server_memory_tool_names or frozenset() if memory_enabled else frozenset()+ )+ proxy_tool_names = filter_memory_tool_names | (+ frozenset({CCR_TOOL_NAME}) if serve_ccr_retrieval else frozenset()+ )memory_filter = (- MemoryToolStreamFilter(server_memory_tool_names)- if memory_enabled and server_memory_tool_names+ MemoryToolStreamFilter(proxy_tool_names, hold_tool_use=serve_ccr_retrieval)+ if proxy_tool_nameselse None)+ # A withheld retrieval needs the round rebuilt for its continuation.+ retrieval_tracking = serve_ccr_retrievaldef _client_frame(frame: bytes) -> bytes:"""``_guard_client_bytes``, noting when the terminal event goes out."""@@ -1743,6 +1827,7 @@ class StreamingMixin:_track_sse = (_codex_wire_debugor memory_enabled+ or retrieval_trackingor (prefix_tracker is not None and provider == "anthropic"))if _track_sse:@@ -1755,6 +1840,7 @@ class StreamingMixin:"disabling memory detection for this request")memory_enabled = False+ retrieval_tracking = False# Parse complete SSE events from buffer. A failure (e.g.# invalid UTF-8) is raised once the chunk is forwarded:@@ -1815,7 +1901,7 @@ class StreamingMixin:# be an upstream protocol violation — surface loudly.full_sse_data: str = full_sse_bytes.decode("utf-8") if full_sse_bytes else ""- if memory_enabled and full_sse_data:+ if (memory_enabled or retrieval_tracking) and full_sse_data:# Check for Claude Code credential errorif "only authorized for use with Claude Code" in full_sse_data:logger.warning(@@ -1845,9 +1931,10 @@ class StreamingMixin:outbound_bytes=outbound_bytes,memory_user_id=memory_user_id,memory_request_ctx=memory_request_ctx,- server_memory_tool_names=server_memory_tool_names or frozenset(),+ server_memory_tool_names=filter_memory_tool_names,stream_state=stream_state,request_id=request_id,+ serve_retrieval=serve_ccr_retrieval,):guarded_frame = _client_frame(frame)if guarded_frame:@@ -1856,8 +1943,10 @@ class StreamingMixin:parsed_response = (stream_state.pop("memory_client_message", None) or parsed_response)- elif parsed_response and self.memory_handler.has_memory_tool_calls(- parsed_response, provider+ elif (+ memory_enabled+ and parsed_response+ and self.memory_handler.has_memory_tool_calls(parsed_response, provider)):logger.info(f"[{request_id}] Memory: Detected tool calls in streaming response"@@ -1890,9 +1979,10 @@ class StreamingMixin:outbound_bytes=outbound_bytes,memory_user_id=memory_user_id,memory_request_ctx=memory_request_ctx,- server_memory_tool_names=server_memory_tool_names or frozenset(),+ server_memory_tool_names=filter_memory_tool_names,stream_state=stream_state,request_id=request_id,+ serve_retrieval=serve_ccr_retrieval,):guarded_frame = _client_frame(frame)if guarded_frame:diff --git a/headroom/proxy/memory_tool_stream.py b/headroom/proxy/memory_tool_stream.pyindex 850cfabff..7f6a791bf 100644--- a/headroom/proxy/memory_tool_stream.py+++ b/headroom/proxy/memory_tool_stream.py@@ -90,6 +90,12 @@ class MemoryToolStreamFilter:extend the message the client already opened.max_retained_bytes: Limit on the bytes held back at any time;``DEFAULT_MAX_RETAINED_BYTES`` when None.+ hold_tool_use: Hold every block from the round's first ``tool_use``+ on, until the caller decides with ``release_held``. A withheld+ ``headroom_retrieve`` alongside a client tool is served by+ dropping the client call for the model to re-issue, which is+ only possible while that call has not reached the client.+ Text before the first tool call still streams live."""def __init__(@@ -99,8 +105,17 @@ class MemoryToolStreamFilter:index_offset: int = 0,forward_message_start: bool = True,max_retained_bytes: int | None = None,+ hold_tool_use: bool = False,) -> None:self._tool_names = frozenset(tool_names)+ self._hold_tool_use = hold_tool_use+ self._holding = False+ # Frames held from the first tool_use on, with their upstream index;+ # client indices are assigned when they are released.+ self._held: list[tuple[str, dict[str, Any], bytes, int]] = []+ self._held_bytes = 0+ # Upstream indices of held client tool_use blocks.+ self._held_tool_use: set[int] = set()self._max_retained_bytes = (DEFAULT_MAX_RETAINED_BYTES if max_retained_bytes is None else max_retained_bytes)@@ -130,6 +145,39 @@ class MemoryToolStreamFilter:def hidden_tool_names(self) -> list[str]:return [block["name"] for block, _ in self._hidden.values()]+ @property+ def holds_client_tool_use(self) -> bool:+ """A client ``tool_use`` block is held and has not reached the client."""+ return bool(self._held_tool_use)++ def release_held(self, *, drop_tool_use: bool = False) -> list[bytes]:+ """Return the held frames, re-indexed, for the client.++ ``drop_tool_use`` leaves out the held client ``tool_use`` blocks: the+ round goes back to the model with the proxy's results instead, and+ the model re-issues whatever it still needs.+ """+ frames: list[bytes] = []+ dropped = self._held_tool_use if drop_tool_use else set()+ for event_name, payload, frame, upstream_index in self._held:+ if upstream_index in dropped:+ continue+ if event_name == "content_block_start":+ block = payload.get("content_block")+ if isinstance(block, dict) and block.get("type") == "tool_use":+ self.visible_tool_use = True+ self._index_map[upstream_index] = self.next_index+ self.next_index += 1+ client_index = self._index_map.get(upstream_index, upstream_index)+ if client_index == upstream_index:+ frames.append(frame)+ else:+ frames.append(_render_frame(event_name, {**payload, "index": client_index}))+ self._held.clear()+ self._held_bytes = 0+ self._held_tool_use.clear()+ return frames+def hidden_calls(self) -> list[dict[str, Any]]:"""The withheld ``tool_use`` blocks that finished with a complete input.@@ -181,7 +229,12 @@ class MemoryToolStreamFilter:forwarded = self._route(frame)if forwarded is not None:out.append(forwarded)- retained = len(self._buffer) + self._tail_bytes + sum(self._hidden_bytes.values())+ retained = (+ len(self._buffer)+ + self._tail_bytes+ + self._held_bytes+ + sum(self._hidden_bytes.values())+ )if retained > self._max_retained_bytes:self._buffer.clear()self._hidden.clear()@@ -189,6 +242,9 @@ class MemoryToolStreamFilter:self._hidden_stopped.clear()self._tail.clear()self._tail_bytes = 0+ self._held.clear()+ self._held_bytes = 0+ self._held_tool_use.clear()raise MemoryToolStreamOverflowError(f"memory tool stream filter would retain {retained} bytes "f"(limit {self._max_retained_bytes})"@@ -232,11 +288,18 @@ class MemoryToolStreamFilter:self._hidden[upstream_index] = (block, [])self._hidden_bytes[upstream_index] = len(frame)self._hidden_stopped.discard(upstream_index)+ self._holding = self._hold_tool_usereturn None- if isinstance(block, dict) and block.get("type") == "tool_use":- self.visible_tool_use = True- self._index_map[upstream_index] = self.next_index- self.next_index += 1+ is_tool_use = isinstance(block, dict) and block.get("type") == "tool_use"+ if self._hold_tool_use and (self._holding or is_tool_use):+ self._holding = True+ if is_tool_use:+ self._held_tool_use.add(upstream_index)+ else:+ if is_tool_use:+ self.visible_tool_use = True+ self._index_map[upstream_index] = self.next_index+ self.next_index += 1if upstream_index in self._hidden:delta = payload.get("delta")@@ -252,17 +315,26 @@ class MemoryToolStreamFilter:elif event_name == "content_block_stop":self._hidden_stopped.add(upstream_index)return None+ if self._holding and upstream_index not in self._index_map:+ self._held.append((event_name, payload, frame, upstream_index))+ self._held_bytes += len(frame)+ return Noneclient_index = self._index_map.get(upstream_index, upstream_index)if client_index == upstream_index:return framereturn _render_frame(event_name, {**payload, "index": client_index})def closing_frames(self, *, prior_usage: dict[str, int] | None = None) -> list[bytes]:- """End the client message after this round without a continuation."""+ """End the client message after this round without a continuation.++ Held frames are released first, client tool calls included: with no+ continuation, the round is the model's answer as it stands.+ """+ held = self.release_held()ends_on_hidden_call = (self.hid_tool_calls and not self.visible_tool_use and self.stop_reason == "tool_use")- return self.tail_frames(+ return held + self.tail_frames(stop_reason="end_turn" if ends_on_hidden_call else None,prior_usage=prior_usage,)diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.pyindex 82a2fbe52..3571dff22 100644--- a/tests/test_ccr_buffered_stream_signed_thinking.py+++ b/tests/test_ccr_buffered_stream_signed_thinking.py@@ -90,11 +90,13 @@ def ccr_marker() -> str:def _body(*, with_thinking: bool, marker: str | None = None) -> dict:- first = "hi" if marker is None else f"hi — earlier output is at <<ccr:{marker}>>"- messages: list[dict] = [{"role": "user", "content": first}]+ # The marker rides in the latest user turn: only content new this turn+ # gates buffering (#3587).+ latest = "hi" if marker is None else f"hi — earlier output is at <<ccr:{marker}>>"+ messages: list[dict] = [{"role": "user", "content": "hi" if with_thinking else latest}]if with_thinking:messages.append(SIGNED_THINKING_TURN)- messages.append({"role": "user", "content": "continue"})+ messages.append({"role": "user", "content": latest})return {"model": "claude-sonnet-4-20250514","max_tokens": 64,@@ -287,6 +289,116 @@ def test_buffering_is_gated_on_a_redeemable_marker(assert calls["stream_body"]["stream"] is True+def _owned_marker(content: str) -> str:+ """Store one more original in the (fixture-reset) store; return its hash."""+ from headroom.cache.compression_store import get_compression_store++ return get_compression_store().store(+ content,+ "<<ccr:placeholder>>",+ original_tokens=100,+ compressed_tokens=5,+ tool_name="Read",+ )+++def _tool_turn(tool_use_id: str, output: str) -> list[dict]:+ return [+ {+ "role": "assistant",+ "content": [{"type": "tool_use", "id": tool_use_id, "name": "Read", "input": {}}],+ },+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": output}],+ },+ ]+++def _agent_body(messages: list[dict]) -> dict:+ return {+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 64,+ "stream": True,+ "tools": [RETRIEVE_TOOL],+ "messages": messages,+ }+++def _replay_session(turns: list[list[dict]]) -> list[str]:+ """Send each message list as one turn of the same session; report the path taken."""+ paths: list[str] = []++ async def fake_stream_response(url, headers, body, *args, **kwargs): # noqa: ANN001+ paths.append("streamed")+ return fastapi.responses.StreamingResponse(iter([SSE_BODY]), media_type="text/event-stream")++ async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001+ paths.append("buffered")+ return httpx.Response(+ 200,+ json={+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-20250514",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 10, "output_tokens": 5},+ },+ headers={"content-type": "application/json"},+ )++ app = create_app(_config())+ with TestClient(app) as client:+ client.app.state.proxy._stream_response = fake_stream_response+ client.app.state.proxy._retry_request = fake_retry+ for messages in turns:+ resp = client.post("/v1/messages", json=_agent_body(messages), headers=_headers())+ assert resp.status_code == 200, resp.text+ return paths+++def test_a_marker_already_forwarded_does_not_buffer_later_turns(ccr_marker: str) -> None:+ """A marker carried forward in history does not make every later turn buffer.++ The marker was in front of the model on the turn that introduced it, and+ that turn buffered. Buffering every later turn for it is #3587: the+ buffered share climbed toward 90% of turns with zero retrievals.+ """+ first = [+ {"role": "user", "content": "read the logs"},+ *_tool_turn("tu_1", f"earlier output at <<ccr:{ccr_marker}>>"),+ ]+ second = [*first, *_tool_turn("tu_2", "short")]++ assert _replay_session([first, second]) == ["buffered", "streamed"]+++def test_a_marker_first_shown_in_an_earlier_message_still_buffers(ccr_marker: str) -> None:+ """New to the model is what counts, not where the marker sits.++ Token mode can compress an earlier tool result on a later request, so the+ model first sees that marker in the middle of the history. Retrieval has+ to be handled server-side there too, or the call reaches the client.+ """+ newly_compressed = _owned_marker("a different, uncompressed tool output")+ first = [+ {"role": "user", "content": "read the logs"},+ *_tool_turn("tu_1", f"earlier output at <<ccr:{ccr_marker}>>"),+ ]+ second = [+ {"role": "user", "content": "read the logs"},+ *_tool_turn(+ "tu_1",+ f"earlier output at <<ccr:{ccr_marker}>> and <<ccr:{newly_compressed}>>",+ ),+ *_tool_turn("tu_2", "short"),+ ]++ assert _replay_session([first, second]) == ["buffered", "buffered"]++def test_cache_hit_never_replays_a_foreign_content_type() -> None:"""A cache entry cannot hand a caller a wire format it did not ask for."""body = {diff --git a/tests/test_ccr_stream_retrieval.py b/tests/test_ccr_stream_retrieval.pynew file mode 100644index 000000000..0d47c62c2--- /dev/null+++ b/tests/test_ccr_stream_retrieval.py@@ -0,0 +1,320 @@+"""``headroom_retrieve`` on a streamed turn is served by the proxy (#3587).++A turn whose redeemable markers were all forwarded on earlier turns streams+instead of buffering, so the client gets the first token right away. The model+can still ask for one of those originals. The client never declared+``headroom_retrieve`` (a coding agent answers "No such tool available"), so+the call must not reach it: the stream withholds it, the proxy serves it from+the compression store, and the continuation round streams into the same client+message.+"""++from __future__ import annotations++import json+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import httpx+import pytest++fastapi = pytest.importorskip("fastapi")++from fastapi.testclient import TestClient # noqa: E402++from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402+from headroom.proxy.memory_tool_stream import MemoryToolStreamFilter # noqa: E402+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++RETRIEVE_ONLY = frozenset({"headroom_retrieve"})+ORIGINAL = "line 1 of the original build log\nline 2: the error the model needs"+++def _frame(payload: dict[str, Any]) -> bytes:+ return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()+++def _sse(blocks: list[dict[str, Any]], stop_reason: str) -> bytes:+ """An Anthropic SSE response carrying ``blocks``."""+ frames = [+ _frame(+ {+ "type": "message_start",+ "message": {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "sonnet-test",+ "content": [],+ "stop_reason": None,+ "usage": {"input_tokens": 10, "output_tokens": 1},+ },+ }+ )+ ]+ for index, block in enumerate(blocks):+ if block["type"] == "text":+ start = {"type": "text", "text": ""}+ delta = {"type": "text_delta", "text": block["text"]}+ else:+ start = {**block, "input": {}}+ delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}+ frames.append(+ _frame({"type": "content_block_start", "index": index, "content_block": start})+ )+ frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))+ frames.append(_frame({"type": "content_block_stop", "index": index}))+ frames.append(+ _frame(+ {+ "type": "message_delta",+ "delta": {"stop_reason": stop_reason, "stop_sequence": None},+ "usage": {"output_tokens": 5},+ }+ )+ )+ frames.append(_frame({"type": "message_stop"}))+ return b"".join(frames)+++def _events(raw: bytes) -> list[dict[str, Any]]:+ return [+ json.loads(line[len("data: ") :])+ for line in raw.decode().splitlines()+ if line.startswith("data: ")+ ]+++def _tool_names(events: list[dict[str, Any]]) -> list[str]:+ return [+ e["content_block"]["name"]+ for e in events+ if e["type"] == "content_block_start" and e["content_block"].get("type") == "tool_use"+ ]+++TEXT = {"type": "text", "text": "Let me look at the earlier output."}+BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}+++def _retrieve(hash_key: str) -> dict[str, Any]:+ return {+ "type": "tool_use",+ "id": "toolu_ccr",+ "name": "headroom_retrieve",+ "input": {"hash": hash_key},+ }+++class TestHoldingFilter:+ def test_text_before_the_first_tool_call_streams_live(self) -> None:+ flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)+ forwarded = _events(b"".join(flt.feed(_sse([TEXT, BASH], "tool_use"))))+ assert [e["type"] for e in forwarded] == [+ "message_start",+ "content_block_start",+ "content_block_delta",+ "content_block_stop",+ ]+ assert flt.holds_client_tool_use++ def test_without_a_retrieval_the_round_is_byte_identical(self) -> None:+ raw = _sse([TEXT, BASH], "tool_use")+ flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)+ out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())+ assert out == raw++ def test_dropping_held_client_calls(self) -> None:+ flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)+ flt.feed(_sse([BASH, _retrieve("abc")], "tool_use"))+ assert flt.hidden_tool_names == ["headroom_retrieve"]+ assert flt.release_held(drop_tool_use=True) == []+ assert flt.next_index == 0+ assert not flt.visible_tool_use+++def _sse_response(raw: bytes) -> MagicMock:+ response = MagicMock()+ response.headers = httpx.Headers({"content-type": "text/event-stream"})+ response.status_code = 200++ async def aiter_bytes(raw: bytes = raw):+ # Split mid-frame so the filter has to reassemble events.+ yield raw[:41]+ yield raw[41:]++ response.aiter_bytes = aiter_bytes+ response.aclose = AsyncMock()+ return response+++def owned_marker():+ """A marker the proxy owns, with its original in the compression store."""+ from headroom.cache.backends import InMemoryBackend+ from headroom.cache.compression_store import get_compression_store, reset_compression_store++ reset_compression_store()+ store = get_compression_store(backend=InMemoryBackend())+ hash_key = store.store(+ ORIGINAL,+ "<<ccr:placeholder>>",+ original_tokens=100,+ compressed_tokens=5,+ tool_name="Bash",+ )+ try:+ yield hash_key+ finally:+ reset_compression_store()+++def _two_turn_session(+ marker: str, second_turn_rounds: list[bytes]+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:+ """Turn 1 introduces ``marker``; turn 2 only carries it in history.++ Returns the client's SSE events for turn 2, the bodies the proxy sent+ upstream on turn 2's stream (first round, then continuations), and how+ many buffered upstream requests turn 2 made.+ """+ first_turn = [+ {"role": "user", "content": "build it"},+ {+ "role": "assistant",+ "content": [{"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}}],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "tu_1",+ "content": f"build log compressed at <<ccr:{marker}>>",+ }+ ],+ },+ ]+ second_turn = [+ *first_turn,+ {"role": "assistant", "content": [{"type": "text", "text": "The build failed."}]},+ {"role": "user", "content": "what was the exact error?"},+ ]+ buffered_calls: list[int] = []++ async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001+ buffered_calls.append(1)+ return httpx.Response(+ 200,+ json={+ "id": "msg_t1",+ "type": "message",+ "role": "assistant",+ "model": "sonnet-test",+ "content": [{"type": "text", "text": "The build failed."}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 10, "output_tokens": 5},+ },+ headers={"content-type": "application/json"},+ )++ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ memory_enabled=False,+ )+ headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}+ tools = [+ create_ccr_tool_definition("anthropic"),+ {"name": "Bash", "description": "run", "input_schema": {"type": "object"}},+ ]++ def body(messages: list[dict[str, Any]]) -> dict[str, Any]:+ return {+ "model": "sonnet-test",+ "max_tokens": 64,+ "stream": True,+ "tools": tools,+ "messages": messages,+ }++ app = create_app(config)+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._retry_request = fake_retry+ first = client.post("/v1/messages", json=body(first_turn), headers=headers)+ assert first.status_code == 200, first.text+ assert len(buffered_calls) == 1, "the turn that introduces the marker buffers"++ upstream = MagicMock(spec=httpx.AsyncClient)+ upstream.build_request = MagicMock(return_value=MagicMock())+ upstream.send = AsyncMock(side_effect=[_sse_response(r) for r in second_turn_rounds])+ proxy.http_client = upstream+ second = client.post("/v1/messages", json=body(second_turn), headers=headers)+ assert second.status_code == 200, second.text++ sent = [json.loads(c.kwargs["content"]) for c in upstream.build_request.call_args_list]+ return _events(second.content), sent, len(buffered_calls) - 1+++def test_later_turn_retrieval_is_served_by_the_proxy(owned_marker: str) -> None:+ events, sent, buffered = _two_turn_session(+ owned_marker,+ [+ _sse([TEXT, _retrieve(owned_marker)], "tool_use"),+ _sse([{"type": "text", "text": "The error is on line 2."}], "end_turn"),+ ],+ )++ assert buffered == 0, "a turn with only already-forwarded markers keeps streaming"+ assert "headroom_retrieve" not in _tool_names(events)+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]+ assert texts == ["Let me look at the earlier output.", "The error is on line 2."]+ stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]+ assert stops == ["end_turn"]+ assert [e["type"] for e in events].count("message_start") == 1++ assert len(sent) == 2+ continuation = sent[1]["messages"]+ assert continuation[-2]["content"][-1]["name"] == "headroom_retrieve"+ served = continuation[-1]["content"][0]+ assert served["type"] == "tool_result" and served["tool_use_id"] == "toolu_ccr"+ assert json.loads(served["content"])["original_content"] == ORIGINAL++[email protected]("client_call_first", [True, False])+def test_retrieval_mixed_with_a_client_tool(owned_marker: str, client_call_first: bool) -> None:+ calls = [BASH, _retrieve(owned_marker)]+ if not client_call_first:+ calls.reverse()+ reissued = {**BASH, "id": "toolu_bash_2"}+ events, sent, buffered = _two_turn_session(+ owned_marker,+ [+ _sse(calls, "tool_use"),+ _sse([{"type": "text", "text": "Now rerun it."}, reissued], "tool_use"),+ ],+ )++ assert buffered == 0+ # Only the model's decision made with the content in hand reaches the client.+ assert _tool_names(events) == ["Bash"]+ started = [e["content_block"] for e in events if e["type"] == "content_block_start"]+ assert [b.get("id") for b in started if b.get("type") == "tool_use"] == ["toolu_bash_2"]+ stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]+ assert stops == ["tool_use"]+ assert [e["index"] for e in events if e["type"] == "content_block_start"] == [0, 1]++ continuation = sent[1]["messages"]+ assert [b["name"] for b in continuation[-2]["content"]] == ["headroom_retrieve"]+ assert json.loads(continuation[-1]["content"][0]["content"])["original_content"] == ORIGINAL+++def test_old_marker_turn_without_retrieval_streams_unchanged(owned_marker: str) -> None:+ raw = _sse([{"type": "text", "text": "It failed on line 2."}, BASH], "tool_use")+ events, sent, buffered = _two_turn_session(owned_marker, [raw])++ assert buffered == 0+ assert len(sent) == 1, "no continuation round without a retrieval"+ assert events == _events(raw)