Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(ccr): buffer a stream only for a marker new this turn (#4607)

SD946a70cfcbdc
diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.py
index c027d2c13..e9e1554bb 100644
--- a/headroom/ccr/response_handler.py
+++ b/headroom/ccr/response_handler.py
@@ -132,6 +132,23 @@ class CCRResponseHandler:
self._retrieval_count = 0
self._retrieval_count_lock = __import__("threading").Lock()
+ def serve_withheld_retrievals(self, tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Run Anthropic ``headroom_retrieve`` calls withheld from a stream.
+
+ Returns one ``tool_result`` block per call that names a hash, in call
+ order, for the continuation round the streaming path sends upstream.
+ A call without a parsable hash gets no result, so the caller can tell
+ the round cannot be replayed.
+ """
+ ccr_calls, _ = self._parse_ccr_tool_calls({"content": tool_calls}, "anthropic")
+ if not ccr_calls:
+ return []
+ with self._retrieval_count_lock:
+ self._retrieval_count += len(ccr_calls)
+ results = [self._execute_retrieval(call) for call in ccr_calls]
+ message = self._create_tool_result_message(results, "anthropic")
+ return list(message.get("content", []))
+
def has_ccr_tool_calls(
self,
response: dict[str, Any],
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index b6c5e10a3..f10d86c7b 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -630,28 +630,41 @@ class AnthropicHandlerMixin:
return False
@staticmethod
- def _outgoing_body_has_redeemable_marker(body: Any) -> bool:
- """Does the body about to be sent carry a marker retrieval could expand?
+ def _outgoing_redeemable_markers(
+ body: Any,
+ previous_forwarded_messages: list[dict[str, Any]] | None = None,
+ ) -> tuple[bool, bool]:
+ """Which owned markers does the outgoing body carry?
+
+ Returns ``(any_owned, new_owned)``. ``headroom_retrieve`` exists only to
+ expand a ``<<ccr:...>>`` marker, so a request carrying none cannot use
+ it (#3071).
- ``headroom_retrieve`` exists only to expand a ``<<ccr:...>>`` marker, so
- a request carrying none cannot benefit from the buffered path (#3071).
+ ``new_owned`` is a marker the model has not seen yet: one absent from
+ what this session forwarded last turn. With no forwarded history to
+ compare against, every owned marker counts as new. "New" is decided
+ against the forwarded history, not by message position: token mode can
+ compress an earlier tool result on this request, so a marker first
+ seen now may sit anywhere in the history.
Ownership is verified rather than shape-matched: the marker shape is not
unique to Headroom, and adopting another context tool's hash would send
the model to an endpoint that is guaranteed to miss (#2836). A hash that
survives ``verify_ownership`` is redeemable right now.
- Errors are swallowed deliberately and answered ``True``. This gates a
- wire-format decision, and the safe direction on an unexpected message
- shape is the long-standing buffered behavior, not a silent change.
+ Errors are swallowed deliberately and answered ``(True, True)``. This
+ gates a wire-format decision, and the safe direction on an unexpected
+ message shape is the long-standing buffered behavior, not a silent
+ change.
"""
if not isinstance(body, dict):
- return True
+ return True, True
messages = body.get("messages")
if not isinstance(messages, list) or not messages:
- return False
+ return False, False
try:
from headroom.ccr.tool_injection import CCRToolInjector
+ from headroom.proxy.helpers import has_new_ccr_markers
probe = CCRToolInjector(
provider="anthropic",
@@ -660,12 +673,27 @@ class AnthropicHandlerMixin:
)
probe.scan_for_markers(messages)
if not probe.detected_hashes:
- return False
+ return False, False
probe.verify_ownership()
- return bool(probe.detected_hashes)
+ if not probe.detected_hashes:
+ return False, False
+ return True, has_new_ccr_markers(
+ current_detected_hashes=list(probe.detected_hashes),
+ previous_forwarded_messages=previous_forwarded_messages,
+ provider="anthropic",
+ )
except Exception: # pragma: no cover - defensive
logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True)
- return True
+ return True, True
+
+ @classmethod
+ def _outgoing_body_has_redeemable_marker(
+ cls,
+ body: Any,
+ previous_forwarded_messages: list[dict[str, Any]] | None = None,
+ ) -> bool:
+ """Does the outgoing body show the model an owned marker it has not seen?"""
+ return cls._outgoing_redeemable_markers(body, previous_forwarded_messages)[1]
@staticmethod
def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]:
@@ -4340,19 +4368,34 @@ class AnthropicHandlerMixin:
and ccr_response_handler_enabled
and has_ccr_retrieve_tool(tools if tools is not None else body.get("tools"))
)
- buffered_retrieval_can_help = (
- retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body)
+ #
+ # A marker the model already saw on an earlier turn can still be
+ # retrieved, so it does not justify buffering every later turn
+ # (#3587), but it does mean a retrieval call may come. Those
+ # turns stream with ``headroom_retrieve`` withheld from the
+ # client and served by the proxy in a continuation round.
+ owned_marker, new_owned_marker = (
+ self._outgoing_redeemable_markers(body, previous_forwarded_messages)
+ if retrieve_tool_is_offered
+ else (False, False)
)
- wants_buffered_stream_ccr = bool(buffered_retrieval_can_help)
- if retrieve_tool_is_offered and not buffered_retrieval_can_help:
+ wants_buffered_stream_ccr = bool(new_owned_marker)
+ if retrieve_tool_is_offered and not owned_marker:
logger.info(
f"[{request_id}] CCR: headroom_retrieve is resident but this "
"request carries no redeemable marker, so server-side "
"retrieval cannot fire; keeping the streaming path (#3071)"
)
+ elif retrieve_tool_is_offered and not new_owned_marker:
+ logger.info(
+ f"[{request_id}] CCR: every redeemable marker was already "
+ "forwarded; streaming with headroom_retrieve served by the "
+ "proxy (#3587)"
+ )
buffered_stream_ccr = (
wants_buffered_stream_ccr and not outbound_locked_to_client_bytes
)
+ serve_ccr_retrieval_in_stream = bool(owned_marker) and not buffered_stream_ccr
if wants_buffered_stream_ccr and outbound_locked_to_client_bytes:
logger.info(
f"[{request_id}] CCR: signed thinking blocks force byte-faithful "
@@ -4526,6 +4569,7 @@ class AnthropicHandlerMixin:
session_key=session_key,
server_memory_tool_names=server_memory_tool_names,
client_beta=client_anthropic_beta or "",
+ serve_ccr_retrieval=serve_ccr_retrieval_in_stream,
)
else:
# Whatever set it — the client's own ``stream: false`` or
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index d51b3e2fb..ddd1680f7 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -520,17 +520,26 @@ class StreamingMixin:
server_memory_tool_names: frozenset[str],
stream_state: dict[str, Any],
request_id: str,
+ serve_retrieval: bool = False,
) -> AsyncIterator[bytes]:
- """Execute withheld memory tool calls and stream continuation rounds.
+ """Execute withheld proxy tool calls and stream continuation rounds.
``memory_filter`` has forwarded one upstream round to the client minus
- the proxy-owned memory tool calls and the message tail. When the round
- stopped on memory tool calls alone, run them, send the results upstream
+ the proxy-owned tool calls and the message tail. When the round
+ stopped on proxy tool calls alone, run them, send the results upstream
as the next turn, and stream that round's blocks into the same client
message. Otherwise end the client message. A round that also called a
client tool still has its memory calls executed, but the turn goes back
to the client, which cannot carry the memory results.
+
+ With ``serve_retrieval``, ``headroom_retrieve`` is a proxy tool too: the
+ client never declared it, so a call that reached it would fail (#3587).
+ The filter then holds client tool calls until the round ends, and a
+ round that retrieved alongside a client tool is continued without the
+ unrun client calls, for the model to re-issue with the content in
+ hand. That is what the buffered path does with the same round.
"""
+ from headroom.ccr.tool_injection import CCR_TOOL_NAME
from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE
try:
@@ -541,18 +550,47 @@ class StreamingMixin:
if not isinstance(messages, list):
messages = None
headers = {k: v for k, v in outbound_headers.items() if k.lower() != "content-length"}
+ proxy_tool_names = server_memory_tool_names | (
+ frozenset({CCR_TOOL_NAME}) if serve_retrieval else frozenset()
+ )
- def is_memory_call(block: Any) -> bool:
+ def is_proxy_call(block: Any) -> bool:
return (
isinstance(block, dict)
and block.get("type") == "tool_use"
- and block.get("name") in server_memory_tool_names
+ and block.get("name") in proxy_tool_names
+ )
+
+ def is_client_call(block: Any) -> bool:
+ return (
+ isinstance(block, dict)
+ and block.get("type") == "tool_use"
+ and not is_proxy_call(block)
)
rounds = 0
# What the client received across rounds; the prefix tracker must see
# this, not round one, because it is what the client sends back.
visible_content: list[Any] = []
+
+ def record_visible(
+ round_response: dict[str, Any] | None,
+ round_content: list[Any],
+ *,
+ dropped_client_calls: bool,
+ ) -> None:
+ if round_response is None:
+ return
+ visible_content.extend(
+ block
+ for block in round_content
+ if not is_proxy_call(block) and not (dropped_client_calls and is_client_call(block))
+ )
+ stream_state["memory_client_message"] = {
+ **round_response,
+ "content": list(visible_content),
+ }
+
# Usage of rounds whose message_delta the client never receives.
prior_usage: dict[str, int] = {}
while True:
@@ -560,13 +598,9 @@ class StreamingMixin:
if response is not None:
round_content = response.get("content")
content = round_content if isinstance(round_content, list) else []
- visible_content.extend(block for block in content if not is_memory_call(block))
- stream_state["memory_client_message"] = {
- **response,
- "content": list(visible_content),
- }
if not memory_filter.hid_tool_calls:
+ record_visible(response, content, dropped_client_calls=False)
for frame in memory_filter.closing_frames(prior_usage=prior_usage):
yield frame
return
@@ -574,7 +608,11 @@ class StreamingMixin:
# Run the calls the filter recorded, so they are not lost when the
# round could not be reconstructed. Only the proxy's own calls: a
# memory-named tool the client declared is the client's to run.
- memory_calls = memory_filter.hidden_calls()
+ hidden_calls = memory_filter.hidden_calls()
+ memory_calls = [c for c in hidden_calls if c.get("name") in server_memory_tool_names]
+ retrieval_calls = [
+ c for c in hidden_calls if serve_retrieval and c.get("name") == CCR_TOOL_NAME
+ ]
tool_results: list[dict[str, Any]] = []
if memory_calls and memory_user_id is not None and self.memory_handler is not None:
tool_results = await self.memory_handler.handle_memory_tool_calls(
@@ -583,24 +621,38 @@ class StreamingMixin:
"anthropic",
request_context=memory_request_ctx,
)
- logger.info(
- f"[{request_id}] Memory: Executed {len(tool_results)}/"
- f"{len(memory_filter.hidden_tool_names)} proxy-handled tool call(s) "
- "withheld from the SSE stream"
- )
+ logger.info(
+ f"[{request_id}] Memory: Executed {len(tool_results)}/"
+ f"{len(memory_calls)} proxy-handled tool call(s) "
+ "withheld from the SSE stream"
+ )
+ ccr_handler = getattr(self, "ccr_response_handler", None)
+ if retrieval_calls and ccr_handler is not None:
+ retrieval_results = ccr_handler.serve_withheld_retrievals(retrieval_calls)
+ tool_results = [*tool_results, *retrieval_results]
+ logger.info(
+ f"[{request_id}] CCR: Served {len(retrieval_results)}/"
+ f"{len(retrieval_calls)} headroom_retrieve call(s) withheld from "
+ "the SSE stream"
+ )
replayable = response is not None and len(tool_results) == sum(
- 1 for block in content if is_memory_call(block)
+ 1 for block in content if is_proxy_call(block)
)
+ # Unrun client calls can be dropped only while the client has not
+ # seen them, and only for a retrieval: memory results alone do not
+ # give the model a reason to decide again.
+ drop_client_calls = bool(retrieval_calls) and memory_filter.holds_client_tool_use
can_continue = (
memory_filter.stop_reason == "tool_use"
and not memory_filter.visible_tool_use
+ and (drop_client_calls or not memory_filter.holds_client_tool_use)
and messages is not None
and replayable
and rounds < self._MEMORY_CONTINUATION_MAX_ROUNDS
)
if not can_continue:
- if memory_filter.visible_tool_use:
+ if memory_filter.visible_tool_use or memory_filter.holds_client_tool_use:
logger.info(
f"[{request_id}] Memory: Round also called a client tool; "
"returning the turn to the client"
@@ -608,18 +660,31 @@ class StreamingMixin:
elif rounds >= self._MEMORY_CONTINUATION_MAX_ROUNDS:
logger.warning(
f"[{request_id}] Memory: Stopped after {rounds} continuation "
- "round(s) that only called memory tools"
+ "round(s) that only called proxy tools"
)
elif not replayable:
logger.warning(
f"[{request_id}] Memory: Could not rebuild the round for a "
- "continuation; ending the turn after running its memory calls"
+ "continuation; ending the turn after running its proxy calls"
)
+ record_visible(response, content, dropped_client_calls=False)
for frame in memory_filter.closing_frames(prior_usage=prior_usage):
yield frame
return
assert response is not None and messages is not None and isinstance(base_body, dict)
+ record_visible(response, content, dropped_client_calls=drop_client_calls)
+ # Blocks held after the first tool call (rarely any but the calls
+ # themselves) still belong to the client's message.
+ for frame in memory_filter.release_held(drop_tool_use=drop_client_calls):
+ yield frame
+ if drop_client_calls:
+ logger.info(
+ f"[{request_id}] CCR: Round called headroom_retrieve alongside a "
+ "client tool; serving the retrieval and dropping the unrun client "
+ "call(s) for the model to re-issue"
+ )
+ content = [block for block in content if not is_client_call(block)]
round_usage = response.get("usage")
if isinstance(round_usage, dict):
for key in _ROUND_USAGE_KEYS:
@@ -647,9 +712,10 @@ class StreamingMixin:
yield _upstream_error_frame(error_bytes, request_id)
return
memory_filter = MemoryToolStreamFilter(
- server_memory_tool_names,
+ proxy_tool_names,
index_offset=memory_filter.next_index,
forward_message_start=False,
+ hold_tool_use=serve_retrieval,
)
# Same cap as the first round's buffer; past it the round is
# still streamed, just not rebuilt for a further continuation.
@@ -1082,6 +1148,7 @@ class StreamingMixin:
savings_measure: ItemMeasure | None = None,
server_memory_tool_names: frozenset[str] | None = None,
client_beta: str | None = None,
+ serve_ccr_retrieval: bool = False,
) -> Response | StreamingResponse:
"""Stream response with metrics tracking and memory tool handling.
@@ -1130,6 +1197,7 @@ class StreamingMixin:
savings_measure=savings_measure,
server_memory_tool_names=server_memory_tool_names,
client_beta=client_beta,
+ serve_ccr_retrieval=serve_ccr_retrieval,
)
except (Exception, asyncio.CancelledError):
self._cleanup_mid_turn_stream(session_key)
@@ -1165,6 +1233,7 @@ class StreamingMixin:
savings_measure: ItemMeasure | None = None,
server_memory_tool_names: frozenset[str] | None = None,
client_beta: str | None = None,
+ serve_ccr_retrieval: bool = False,
) -> Response | StreamingResponse:
"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""
from fastapi.responses import Response, StreamingResponse
@@ -1339,6 +1408,9 @@ class StreamingMixin:
and self.memory_handler is not None
and provider == "anthropic"
)
+ # Withholding headroom_retrieve needs the Anthropic SSE shape the
+ # filter and the continuation round are written for.
+ serve_ccr_retrieval = serve_ccr_retrieval and provider == "anthropic"
# Open connection before generator to capture upstream response headers
# (needed to forward ratelimit headers to the client via StreamingResponse)
@@ -1672,11 +1744,23 @@ class StreamingMixin:
pending_messages: list[dict] = []
# Proxy-injected memory tools are invisible to the client, so their
# tool_use blocks must never reach it (GH #2195).
+ from headroom.ccr.tool_injection import CCR_TOOL_NAME
+
+ # Memory tools are withheld only when this request can run them;
+ # the continuation must classify calls exactly as the filter did.
+ filter_memory_tool_names = (
+ server_memory_tool_names or frozenset() if memory_enabled else frozenset()
+ )
+ proxy_tool_names = filter_memory_tool_names | (
+ frozenset({CCR_TOOL_NAME}) if serve_ccr_retrieval else frozenset()
+ )
memory_filter = (
- MemoryToolStreamFilter(server_memory_tool_names)
- if memory_enabled and server_memory_tool_names
+ MemoryToolStreamFilter(proxy_tool_names, hold_tool_use=serve_ccr_retrieval)
+ if proxy_tool_names
else None
)
+ # A withheld retrieval needs the round rebuilt for its continuation.
+ retrieval_tracking = serve_ccr_retrieval
def _client_frame(frame: bytes) -> bytes:
"""``_guard_client_bytes``, noting when the terminal event goes out."""
@@ -1743,6 +1827,7 @@ class StreamingMixin:
_track_sse = (
_codex_wire_debug
or memory_enabled
+ or retrieval_tracking
or (prefix_tracker is not None and provider == "anthropic")
)
if _track_sse:
@@ -1755,6 +1840,7 @@ class StreamingMixin:
"disabling memory detection for this request"
)
memory_enabled = False
+ retrieval_tracking = False
# Parse complete SSE events from buffer. A failure (e.g.
# invalid UTF-8) is raised once the chunk is forwarded:
@@ -1815,7 +1901,7 @@ class StreamingMixin:
# be an upstream protocol violation — surface loudly.
full_sse_data: str = full_sse_bytes.decode("utf-8") if full_sse_bytes else ""
- if memory_enabled and full_sse_data:
+ if (memory_enabled or retrieval_tracking) and full_sse_data:
# Check for Claude Code credential error
if "only authorized for use with Claude Code" in full_sse_data:
logger.warning(
@@ -1845,9 +1931,10 @@ class StreamingMixin:
outbound_bytes=outbound_bytes,
memory_user_id=memory_user_id,
memory_request_ctx=memory_request_ctx,
- server_memory_tool_names=server_memory_tool_names or frozenset(),
+ server_memory_tool_names=filter_memory_tool_names,
stream_state=stream_state,
request_id=request_id,
+ serve_retrieval=serve_ccr_retrieval,
):
guarded_frame = _client_frame(frame)
if guarded_frame:
@@ -1856,8 +1943,10 @@ class StreamingMixin:
parsed_response = (
stream_state.pop("memory_client_message", None) or parsed_response
)
- elif parsed_response and self.memory_handler.has_memory_tool_calls(
- parsed_response, provider
+ elif (
+ memory_enabled
+ and parsed_response
+ and self.memory_handler.has_memory_tool_calls(parsed_response, provider)
):
logger.info(
f"[{request_id}] Memory: Detected tool calls in streaming response"
@@ -1890,9 +1979,10 @@ class StreamingMixin:
outbound_bytes=outbound_bytes,
memory_user_id=memory_user_id,
memory_request_ctx=memory_request_ctx,
- server_memory_tool_names=server_memory_tool_names or frozenset(),
+ server_memory_tool_names=filter_memory_tool_names,
stream_state=stream_state,
request_id=request_id,
+ serve_retrieval=serve_ccr_retrieval,
):
guarded_frame = _client_frame(frame)
if guarded_frame:
diff --git a/headroom/proxy/memory_tool_stream.py b/headroom/proxy/memory_tool_stream.py
index 850cfabff..7f6a791bf 100644
--- a/headroom/proxy/memory_tool_stream.py
+++ b/headroom/proxy/memory_tool_stream.py
@@ -90,6 +90,12 @@ class MemoryToolStreamFilter:
extend the message the client already opened.
max_retained_bytes: Limit on the bytes held back at any time;
``DEFAULT_MAX_RETAINED_BYTES`` when None.
+ hold_tool_use: Hold every block from the round's first ``tool_use``
+ on, until the caller decides with ``release_held``. A withheld
+ ``headroom_retrieve`` alongside a client tool is served by
+ dropping the client call for the model to re-issue, which is
+ only possible while that call has not reached the client.
+ Text before the first tool call still streams live.
"""
def __init__(
@@ -99,8 +105,17 @@ class MemoryToolStreamFilter:
index_offset: int = 0,
forward_message_start: bool = True,
max_retained_bytes: int | None = None,
+ hold_tool_use: bool = False,
) -> None:
self._tool_names = frozenset(tool_names)
+ self._hold_tool_use = hold_tool_use
+ self._holding = False
+ # Frames held from the first tool_use on, with their upstream index;
+ # client indices are assigned when they are released.
+ self._held: list[tuple[str, dict[str, Any], bytes, int]] = []
+ self._held_bytes = 0
+ # Upstream indices of held client tool_use blocks.
+ self._held_tool_use: set[int] = set()
self._max_retained_bytes = (
DEFAULT_MAX_RETAINED_BYTES if max_retained_bytes is None else max_retained_bytes
)
@@ -130,6 +145,39 @@ class MemoryToolStreamFilter:
def hidden_tool_names(self) -> list[str]:
return [block["name"] for block, _ in self._hidden.values()]
+ @property
+ def holds_client_tool_use(self) -> bool:
+ """A client ``tool_use`` block is held and has not reached the client."""
+ return bool(self._held_tool_use)
+
+ def release_held(self, *, drop_tool_use: bool = False) -> list[bytes]:
+ """Return the held frames, re-indexed, for the client.
+
+ ``drop_tool_use`` leaves out the held client ``tool_use`` blocks: the
+ round goes back to the model with the proxy's results instead, and
+ the model re-issues whatever it still needs.
+ """
+ frames: list[bytes] = []
+ dropped = self._held_tool_use if drop_tool_use else set()
+ for event_name, payload, frame, upstream_index in self._held:
+ if upstream_index in dropped:
+ continue
+ if event_name == "content_block_start":
+ block = payload.get("content_block")
+ if isinstance(block, dict) and block.get("type") == "tool_use":
+ self.visible_tool_use = True
+ self._index_map[upstream_index] = self.next_index
+ self.next_index += 1
+ client_index = self._index_map.get(upstream_index, upstream_index)
+ if client_index == upstream_index:
+ frames.append(frame)
+ else:
+ frames.append(_render_frame(event_name, {**payload, "index": client_index}))
+ self._held.clear()
+ self._held_bytes = 0
+ self._held_tool_use.clear()
+ return frames
+
def hidden_calls(self) -> list[dict[str, Any]]:
"""The withheld ``tool_use`` blocks that finished with a complete input.
@@ -181,7 +229,12 @@ class MemoryToolStreamFilter:
forwarded = self._route(frame)
if forwarded is not None:
out.append(forwarded)
- retained = len(self._buffer) + self._tail_bytes + sum(self._hidden_bytes.values())
+ retained = (
+ len(self._buffer)
+ + self._tail_bytes
+ + self._held_bytes
+ + sum(self._hidden_bytes.values())
+ )
if retained > self._max_retained_bytes:
self._buffer.clear()
self._hidden.clear()
@@ -189,6 +242,9 @@ class MemoryToolStreamFilter:
self._hidden_stopped.clear()
self._tail.clear()
self._tail_bytes = 0
+ self._held.clear()
+ self._held_bytes = 0
+ self._held_tool_use.clear()
raise MemoryToolStreamOverflowError(
f"memory tool stream filter would retain {retained} bytes "
f"(limit {self._max_retained_bytes})"
@@ -232,11 +288,18 @@ class MemoryToolStreamFilter:
self._hidden[upstream_index] = (block, [])
self._hidden_bytes[upstream_index] = len(frame)
self._hidden_stopped.discard(upstream_index)
+ self._holding = self._hold_tool_use
return None
- if isinstance(block, dict) and block.get("type") == "tool_use":
- self.visible_tool_use = True
- self._index_map[upstream_index] = self.next_index
- self.next_index += 1
+ is_tool_use = isinstance(block, dict) and block.get("type") == "tool_use"
+ if self._hold_tool_use and (self._holding or is_tool_use):
+ self._holding = True
+ if is_tool_use:
+ self._held_tool_use.add(upstream_index)
+ else:
+ if is_tool_use:
+ self.visible_tool_use = True
+ self._index_map[upstream_index] = self.next_index
+ self.next_index += 1
if upstream_index in self._hidden:
delta = payload.get("delta")
@@ -252,17 +315,26 @@ class MemoryToolStreamFilter:
elif event_name == "content_block_stop":
self._hidden_stopped.add(upstream_index)
return None
+ if self._holding and upstream_index not in self._index_map:
+ self._held.append((event_name, payload, frame, upstream_index))
+ self._held_bytes += len(frame)
+ return None
client_index = self._index_map.get(upstream_index, upstream_index)
if client_index == upstream_index:
return frame
return _render_frame(event_name, {**payload, "index": client_index})
def closing_frames(self, *, prior_usage: dict[str, int] | None = None) -> list[bytes]:
- """End the client message after this round without a continuation."""
+ """End the client message after this round without a continuation.
+
+ Held frames are released first, client tool calls included: with no
+ continuation, the round is the model's answer as it stands.
+ """
+ held = self.release_held()
ends_on_hidden_call = (
self.hid_tool_calls and not self.visible_tool_use and self.stop_reason == "tool_use"
)
- return self.tail_frames(
+ return held + self.tail_frames(
stop_reason="end_turn" if ends_on_hidden_call else None,
prior_usage=prior_usage,
)
diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.py
index 82a2fbe52..3571dff22 100644
--- a/tests/test_ccr_buffered_stream_signed_thinking.py
+++ b/tests/test_ccr_buffered_stream_signed_thinking.py
@@ -90,11 +90,13 @@ def ccr_marker() -> str:
def _body(*, with_thinking: bool, marker: str | None = None) -> dict:
- first = "hi" if marker is None else f"hi — earlier output is at <<ccr:{marker}>>"
- messages: list[dict] = [{"role": "user", "content": first}]
+ # The marker rides in the latest user turn: only content new this turn
+ # gates buffering (#3587).
+ latest = "hi" if marker is None else f"hi — earlier output is at <<ccr:{marker}>>"
+ messages: list[dict] = [{"role": "user", "content": "hi" if with_thinking else latest}]
if with_thinking:
messages.append(SIGNED_THINKING_TURN)
- messages.append({"role": "user", "content": "continue"})
+ messages.append({"role": "user", "content": latest})
return {
"model": "claude-sonnet-4-20250514",
"max_tokens": 64,
@@ -287,6 +289,116 @@ def test_buffering_is_gated_on_a_redeemable_marker(
assert calls["stream_body"]["stream"] is True
+def _owned_marker(content: str) -> str:
+ """Store one more original in the (fixture-reset) store; return its hash."""
+ from headroom.cache.compression_store import get_compression_store
+
+ return get_compression_store().store(
+ content,
+ "<<ccr:placeholder>>",
+ original_tokens=100,
+ compressed_tokens=5,
+ tool_name="Read",
+ )
+
+
+def _tool_turn(tool_use_id: str, output: str) -> list[dict]:
+ return [
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": tool_use_id, "name": "Read", "input": {}}],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": tool_use_id, "content": output}],
+ },
+ ]
+
+
+def _agent_body(messages: list[dict]) -> dict:
+ return {
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 64,
+ "stream": True,
+ "tools": [RETRIEVE_TOOL],
+ "messages": messages,
+ }
+
+
+def _replay_session(turns: list[list[dict]]) -> list[str]:
+ """Send each message list as one turn of the same session; report the path taken."""
+ paths: list[str] = []
+
+ async def fake_stream_response(url, headers, body, *args, **kwargs): # noqa: ANN001
+ paths.append("streamed")
+ return fastapi.responses.StreamingResponse(iter([SSE_BODY]), media_type="text/event-stream")
+
+ async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001
+ paths.append("buffered")
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-20250514",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 10, "output_tokens": 5},
+ },
+ headers={"content-type": "application/json"},
+ )
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ client.app.state.proxy._stream_response = fake_stream_response
+ client.app.state.proxy._retry_request = fake_retry
+ for messages in turns:
+ resp = client.post("/v1/messages", json=_agent_body(messages), headers=_headers())
+ assert resp.status_code == 200, resp.text
+ return paths
+
+
+def test_a_marker_already_forwarded_does_not_buffer_later_turns(ccr_marker: str) -> None:
+ """A marker carried forward in history does not make every later turn buffer.
+
+ The marker was in front of the model on the turn that introduced it, and
+ that turn buffered. Buffering every later turn for it is #3587: the
+ buffered share climbed toward 90% of turns with zero retrievals.
+ """
+ first = [
+ {"role": "user", "content": "read the logs"},
+ *_tool_turn("tu_1", f"earlier output at <<ccr:{ccr_marker}>>"),
+ ]
+ second = [*first, *_tool_turn("tu_2", "short")]
+
+ assert _replay_session([first, second]) == ["buffered", "streamed"]
+
+
+def test_a_marker_first_shown_in_an_earlier_message_still_buffers(ccr_marker: str) -> None:
+ """New to the model is what counts, not where the marker sits.
+
+ Token mode can compress an earlier tool result on a later request, so the
+ model first sees that marker in the middle of the history. Retrieval has
+ to be handled server-side there too, or the call reaches the client.
+ """
+ newly_compressed = _owned_marker("a different, uncompressed tool output")
+ first = [
+ {"role": "user", "content": "read the logs"},
+ *_tool_turn("tu_1", f"earlier output at <<ccr:{ccr_marker}>>"),
+ ]
+ second = [
+ {"role": "user", "content": "read the logs"},
+ *_tool_turn(
+ "tu_1",
+ f"earlier output at <<ccr:{ccr_marker}>> and <<ccr:{newly_compressed}>>",
+ ),
+ *_tool_turn("tu_2", "short"),
+ ]
+
+ assert _replay_session([first, second]) == ["buffered", "buffered"]
+
+
def test_cache_hit_never_replays_a_foreign_content_type() -> None:
"""A cache entry cannot hand a caller a wire format it did not ask for."""
body = {
diff --git a/tests/test_ccr_stream_retrieval.py b/tests/test_ccr_stream_retrieval.py
new file mode 100644
index 000000000..0d47c62c2
--- /dev/null
+++ b/tests/test_ccr_stream_retrieval.py
@@ -0,0 +1,320 @@
+"""``headroom_retrieve`` on a streamed turn is served by the proxy (#3587).
+
+A turn whose redeemable markers were all forwarded on earlier turns streams
+instead of buffering, so the client gets the first token right away. The model
+can still ask for one of those originals. The client never declared
+``headroom_retrieve`` (a coding agent answers "No such tool available"), so
+the call must not reach it: the stream withholds it, the proxy serves it from
+the compression store, and the continuation round streams into the same client
+message.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402
+from headroom.proxy.memory_tool_stream import MemoryToolStreamFilter # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+RETRIEVE_ONLY = frozenset({"headroom_retrieve"})
+ORIGINAL = "line 1 of the original build log\nline 2: the error the model needs"
+
+
+def _frame(payload: dict[str, Any]) -> bytes:
+ return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
+def _sse(blocks: list[dict[str, Any]], stop_reason: str) -> bytes:
+ """An Anthropic SSE response carrying ``blocks``."""
+ frames = [
+ _frame(
+ {
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "sonnet-test",
+ "content": [],
+ "stop_reason": None,
+ "usage": {"input_tokens": 10, "output_tokens": 1},
+ },
+ }
+ )
+ ]
+ for index, block in enumerate(blocks):
+ if block["type"] == "text":
+ start = {"type": "text", "text": ""}
+ delta = {"type": "text_delta", "text": block["text"]}
+ else:
+ start = {**block, "input": {}}
+ delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}
+ frames.append(
+ _frame({"type": "content_block_start", "index": index, "content_block": start})
+ )
+ frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))
+ frames.append(_frame({"type": "content_block_stop", "index": index}))
+ frames.append(
+ _frame(
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": stop_reason, "stop_sequence": None},
+ "usage": {"output_tokens": 5},
+ }
+ )
+ )
+ frames.append(_frame({"type": "message_stop"}))
+ return b"".join(frames)
+
+
+def _events(raw: bytes) -> list[dict[str, Any]]:
+ return [
+ json.loads(line[len("data: ") :])
+ for line in raw.decode().splitlines()
+ if line.startswith("data: ")
+ ]
+
+
+def _tool_names(events: list[dict[str, Any]]) -> list[str]:
+ return [
+ e["content_block"]["name"]
+ for e in events
+ if e["type"] == "content_block_start" and e["content_block"].get("type") == "tool_use"
+ ]
+
+
+TEXT = {"type": "text", "text": "Let me look at the earlier output."}
+BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}
+
+
+def _retrieve(hash_key: str) -> dict[str, Any]:
+ return {
+ "type": "tool_use",
+ "id": "toolu_ccr",
+ "name": "headroom_retrieve",
+ "input": {"hash": hash_key},
+ }
+
+
+class TestHoldingFilter:
+ def test_text_before_the_first_tool_call_streams_live(self) -> None:
+ flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
+ forwarded = _events(b"".join(flt.feed(_sse([TEXT, BASH], "tool_use"))))
+ assert [e["type"] for e in forwarded] == [
+ "message_start",
+ "content_block_start",
+ "content_block_delta",
+ "content_block_stop",
+ ]
+ assert flt.holds_client_tool_use
+
+ def test_without_a_retrieval_the_round_is_byte_identical(self) -> None:
+ raw = _sse([TEXT, BASH], "tool_use")
+ flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
+ out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())
+ assert out == raw
+
+ def test_dropping_held_client_calls(self) -> None:
+ flt = MemoryToolStreamFilter(RETRIEVE_ONLY, hold_tool_use=True)
+ flt.feed(_sse([BASH, _retrieve("abc")], "tool_use"))
+ assert flt.hidden_tool_names == ["headroom_retrieve"]
+ assert flt.release_held(drop_tool_use=True) == []
+ assert flt.next_index == 0
+ assert not flt.visible_tool_use
+
+
+def _sse_response(raw: bytes) -> MagicMock:
+ response = MagicMock()
+ response.headers = httpx.Headers({"content-type": "text/event-stream"})
+ response.status_code = 200
+
+ async def aiter_bytes(raw: bytes = raw):
+ # Split mid-frame so the filter has to reassemble events.
+ yield raw[:41]
+ yield raw[41:]
+
+ response.aiter_bytes = aiter_bytes
+ response.aclose = AsyncMock()
+ return response
+
+
+def owned_marker():
+ """A marker the proxy owns, with its original in the compression store."""
+ from headroom.cache.backends import InMemoryBackend
+ from headroom.cache.compression_store import get_compression_store, reset_compression_store
+
+ reset_compression_store()
+ store = get_compression_store(backend=InMemoryBackend())
+ hash_key = store.store(
+ ORIGINAL,
+ "<<ccr:placeholder>>",
+ original_tokens=100,
+ compressed_tokens=5,
+ tool_name="Bash",
+ )
+ try:
+ yield hash_key
+ finally:
+ reset_compression_store()
+
+
+def _two_turn_session(
+ marker: str, second_turn_rounds: list[bytes]
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int]:
+ """Turn 1 introduces ``marker``; turn 2 only carries it in history.
+
+ Returns the client's SSE events for turn 2, the bodies the proxy sent
+ upstream on turn 2's stream (first round, then continuations), and how
+ many buffered upstream requests turn 2 made.
+ """
+ first_turn = [
+ {"role": "user", "content": "build it"},
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": "tu_1", "name": "Bash", "input": {}}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "tu_1",
+ "content": f"build log compressed at <<ccr:{marker}>>",
+ }
+ ],
+ },
+ ]
+ second_turn = [
+ *first_turn,
+ {"role": "assistant", "content": [{"type": "text", "text": "The build failed."}]},
+ {"role": "user", "content": "what was the exact error?"},
+ ]
+ buffered_calls: list[int] = []
+
+ async def fake_retry(method, url, headers, req_body, *args, **kwargs): # noqa: ANN001
+ buffered_calls.append(1)
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_t1",
+ "type": "message",
+ "role": "assistant",
+ "model": "sonnet-test",
+ "content": [{"type": "text", "text": "The build failed."}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 10, "output_tokens": 5},
+ },
+ headers={"content-type": "application/json"},
+ )
+
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ memory_enabled=False,
+ )
+ headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
+ tools = [
+ create_ccr_tool_definition("anthropic"),
+ {"name": "Bash", "description": "run", "input_schema": {"type": "object"}},
+ ]
+
+ def body(messages: list[dict[str, Any]]) -> dict[str, Any]:
+ return {
+ "model": "sonnet-test",
+ "max_tokens": 64,
+ "stream": True,
+ "tools": tools,
+ "messages": messages,
+ }
+
+ app = create_app(config)
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ proxy._retry_request = fake_retry
+ first = client.post("/v1/messages", json=body(first_turn), headers=headers)
+ assert first.status_code == 200, first.text
+ assert len(buffered_calls) == 1, "the turn that introduces the marker buffers"
+
+ upstream = MagicMock(spec=httpx.AsyncClient)
+ upstream.build_request = MagicMock(return_value=MagicMock())
+ upstream.send = AsyncMock(side_effect=[_sse_response(r) for r in second_turn_rounds])
+ proxy.http_client = upstream
+ second = client.post("/v1/messages", json=body(second_turn), headers=headers)
+ assert second.status_code == 200, second.text
+
+ sent = [json.loads(c.kwargs["content"]) for c in upstream.build_request.call_args_list]
+ return _events(second.content), sent, len(buffered_calls) - 1
+
+
+def test_later_turn_retrieval_is_served_by_the_proxy(owned_marker: str) -> None:
+ events, sent, buffered = _two_turn_session(
+ owned_marker,
+ [
+ _sse([TEXT, _retrieve(owned_marker)], "tool_use"),
+ _sse([{"type": "text", "text": "The error is on line 2."}], "end_turn"),
+ ],
+ )
+
+ assert buffered == 0, "a turn with only already-forwarded markers keeps streaming"
+ assert "headroom_retrieve" not in _tool_names(events)
+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
+ assert texts == ["Let me look at the earlier output.", "The error is on line 2."]
+ stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
+ assert stops == ["end_turn"]
+ assert [e["type"] for e in events].count("message_start") == 1
+
+ assert len(sent) == 2
+ continuation = sent[1]["messages"]
+ assert continuation[-2]["content"][-1]["name"] == "headroom_retrieve"
+ served = continuation[-1]["content"][0]
+ assert served["type"] == "tool_result" and served["tool_use_id"] == "toolu_ccr"
+ assert json.loads(served["content"])["original_content"] == ORIGINAL
+
+
[email protected]("client_call_first", [True, False])
+def test_retrieval_mixed_with_a_client_tool(owned_marker: str, client_call_first: bool) -> None:
+ calls = [BASH, _retrieve(owned_marker)]
+ if not client_call_first:
+ calls.reverse()
+ reissued = {**BASH, "id": "toolu_bash_2"}
+ events, sent, buffered = _two_turn_session(
+ owned_marker,
+ [
+ _sse(calls, "tool_use"),
+ _sse([{"type": "text", "text": "Now rerun it."}, reissued], "tool_use"),
+ ],
+ )
+
+ assert buffered == 0
+ # Only the model's decision made with the content in hand reaches the client.
+ assert _tool_names(events) == ["Bash"]
+ started = [e["content_block"] for e in events if e["type"] == "content_block_start"]
+ assert [b.get("id") for b in started if b.get("type") == "tool_use"] == ["toolu_bash_2"]
+ stops = [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"]
+ assert stops == ["tool_use"]
+ assert [e["index"] for e in events if e["type"] == "content_block_start"] == [0, 1]
+
+ continuation = sent[1]["messages"]
+ assert [b["name"] for b in continuation[-2]["content"]] == ["headroom_retrieve"]
+ assert json.loads(continuation[-1]["content"][0]["content"])["original_content"] == ORIGINAL
+
+
+def test_old_marker_turn_without_retrieval_streams_unchanged(owned_marker: str) -> None:
+ raw = _sse([{"type": "text", "text": "It failed on line 2."}, BASH], "tool_use")
+ events, sent, buffered = _two_turn_session(owned_marker, [raw])
+
+ assert buffered == 0
+ assert len(sent) == 1, "no continuation round without a retrieval"
+ assert events == _events(raw)