Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): run injected memory tools server-side on streaming turns (#3947)

Kenneth Wong1cf496612e78
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 2974e602f..6e507c032 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -431,6 +431,25 @@ class AnthropicHandlerMixin:
canonical = str(tool)
return (name, canonical)
+ @staticmethod
+ def _server_memory_tool_names(tools: Any, client_tools: Any) -> frozenset[str]:
+ """Memory tools in ``tools`` that the proxy injected and must run itself.
+
+ A memory tool the client declared stays the client's: its calls are
+ forwarded, not withheld.
+ """
+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES, NATIVE_MEMORY_TOOL_NAME
+
+ client_tool_names = {t.get("name") for t in client_tools or [] if isinstance(t, dict)}
+ return frozenset(
+ name
+ for t in tools or []
+ if isinstance(t, dict)
+ and isinstance(name := t.get("name"), str)
+ and (name in MEMORY_TOOL_NAMES or name == NATIVE_MEMORY_TOOL_NAME)
+ and name not in client_tool_names
+ )
+
@staticmethod
def _has_headroom_retrieve_tool(tools: Any) -> bool:
"""Return True when the final Anthropic tool list includes CCR retrieve."""
@@ -2737,6 +2756,9 @@ class AnthropicHandlerMixin:
# /v1/messages just as on /v1/responses.
memory_context_injected = False
memory_tools_injected = False
+ # Memory tools this proxy injected and must execute itself; the
+ # streaming path withholds their calls from the client.
+ server_memory_tool_names: frozenset[str] = frozenset()
if memory_decision.inject:
# Search and inject memory context
if self.memory_handler.config.inject_context:
@@ -2848,6 +2870,9 @@ class AnthropicHandlerMixin:
)
if mem_tools_injected:
memory_tools_injected = True
+ server_memory_tool_names = self._server_memory_tool_names(
+ tools, _original_tools
+ )
tool_names = [
t.get("name") or t.get("type", "")
for t in tools
@@ -4067,6 +4092,7 @@ class AnthropicHandlerMixin:
memory_request_ctx=memory_request_ctx,
outcome_provider=provider_name,
session_key=session_key,
+ server_memory_tool_names=server_memory_tool_names,
)
else:
# Whatever set it — the client's own ``stream: false`` or
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index d3ec34764..810357dc9 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -10,6 +10,7 @@ import contextlib
import json
import logging
import time
+from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
from headroom.proxy.auth_mode import classify_client, supports_mid_turn_coalescing
@@ -29,12 +30,55 @@ import httpx
from headroom.copilot_auth import apply_copilot_api_auth
from headroom.proxy import public_errors
+from headroom.proxy.memory_tool_stream import (
+ MemoryToolStreamFilter,
+ MemoryToolStreamOverflowError,
+)
from headroom.proxy.stream_output_tokens import estimate_output_tokens
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
from headroom.utils import format_exception_message
logger = logging.getLogger("headroom.proxy")
+_ROUND_USAGE_KEYS = (
+ "input_tokens",
+ "output_tokens",
+ "cache_read_input_tokens",
+ "cache_creation_input_tokens",
+)
+
+
+def _add_round_usage(stream_state: dict[str, Any], response: dict[str, Any] | None) -> None:
+ """Add a server-side continuation round's usage to the turn's totals."""
+ usage = response.get("usage") if isinstance(response, dict) else None
+ if not isinstance(usage, dict):
+ return
+ for key in _ROUND_USAGE_KEYS:
+ value = usage.get(key)
+ if isinstance(value, int):
+ stream_state[key] = (stream_state.get(key) or 0) + value
+
+
+def _upstream_error_frame(error_bytes: bytes, request_id: str) -> bytes:
+ """Render an upstream error response as an Anthropic SSE ``error`` event.
+
+ An Anthropic error envelope is addressed to the caller and passes through
+ as is; anything else becomes the generic public error.
+ """
+ try:
+ payload = json.loads(error_bytes)
+ except (json.JSONDecodeError, UnicodeDecodeError):
+ payload = None
+ if not (
+ isinstance(payload, dict)
+ and payload.get("type") == "error"
+ and isinstance(payload.get("error"), dict)
+ ):
+ payload = public_errors.anthropic_error_body(
+ public_errors.INTERNAL_ERROR, request_id=request_id
+ )
+ return f"event: error\ndata: {json.dumps(payload)}\n\n".encode()
+
def _thinking_for_stream(payload: object) -> ThinkingTokens:
"""Thinking-token split for a reassembled streaming response.
@@ -436,6 +480,174 @@ class StreamingMixin:
return render_anthropic_sse_response(response)
+ async def _continue_memory_tool_stream(
+ self,
+ memory_filter: MemoryToolStreamFilter,
+ response: dict[str, Any] | None,
+ *,
+ url: str,
+ outbound_headers: dict[str, str],
+ outbound_bytes: bytes,
+ memory_user_id: str | None,
+ memory_request_ctx: Any | None,
+ server_memory_tool_names: frozenset[str],
+ stream_state: dict[str, Any],
+ request_id: str,
+ ) -> AsyncIterator[bytes]:
+ """Execute withheld memory tool calls and stream continuation rounds.
+
+ ``memory_filter`` has forwarded one upstream round to the client minus
+ the proxy-owned memory tool calls and the message tail. When the round
+ stopped on memory tool calls alone, run them, send the results upstream
+ as the next turn, and stream that round's blocks into the same client
+ message. Otherwise end the client message. A round that also called a
+ client tool still has its memory calls executed, but the turn goes back
+ to the client, which cannot carry the memory results.
+ """
+ from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE
+
+ try:
+ base_body = json.loads(outbound_bytes)
+ except (json.JSONDecodeError, UnicodeDecodeError):
+ base_body = None
+ messages = base_body.get("messages") if isinstance(base_body, dict) else None
+ if not isinstance(messages, list):
+ messages = None
+ headers = {k: v for k, v in outbound_headers.items() if k.lower() != "content-length"}
+
+ def is_memory_call(block: Any) -> bool:
+ return (
+ isinstance(block, dict)
+ and block.get("type") == "tool_use"
+ and block.get("name") in server_memory_tool_names
+ )
+
+ rounds = 0
+ # What the client received across rounds; the prefix tracker must see
+ # this, not round one, because it is what the client sends back.
+ visible_content: list[Any] = []
+ # Usage of rounds whose message_delta the client never receives.
+ prior_usage: dict[str, int] = {}
+ while True:
+ content: list[Any] = []
+ if response is not None:
+ round_content = response.get("content")
+ content = round_content if isinstance(round_content, list) else []
+ visible_content.extend(block for block in content if not is_memory_call(block))
+ stream_state["memory_client_message"] = {
+ **response,
+ "content": list(visible_content),
+ }
+
+ if not memory_filter.hid_tool_calls:
+ for frame in memory_filter.closing_frames(prior_usage=prior_usage):
+ yield frame
+ return
+
+ # Run the calls the filter recorded, so they are not lost when the
+ # round could not be reconstructed. Only the proxy's own calls: a
+ # memory-named tool the client declared is the client's to run.
+ memory_calls = memory_filter.hidden_calls()
+ tool_results: list[dict[str, Any]] = []
+ if memory_calls and memory_user_id is not None and self.memory_handler is not None:
+ tool_results = await self.memory_handler.handle_memory_tool_calls(
+ {"content": memory_calls},
+ memory_user_id,
+ "anthropic",
+ request_context=memory_request_ctx,
+ )
+ logger.info(
+ f"[{request_id}] Memory: Executed {len(tool_results)}/"
+ f"{len(memory_filter.hidden_tool_names)} proxy-handled tool call(s) "
+ "withheld from the SSE stream"
+ )
+
+ replayable = response is not None and len(tool_results) == sum(
+ 1 for block in content if is_memory_call(block)
+ )
+ can_continue = (
+ memory_filter.stop_reason == "tool_use"
+ and not memory_filter.visible_tool_use
+ and messages is not None
+ and replayable
+ and rounds < self._MEMORY_CONTINUATION_MAX_ROUNDS
+ )
+ if not can_continue:
+ if memory_filter.visible_tool_use:
+ logger.info(
+ f"[{request_id}] Memory: Round also called a client tool; "
+ "returning the turn to the client"
+ )
+ elif rounds >= self._MEMORY_CONTINUATION_MAX_ROUNDS:
+ logger.warning(
+ f"[{request_id}] Memory: Stopped after {rounds} continuation "
+ "round(s) that only called memory tools"
+ )
+ elif not replayable:
+ logger.warning(
+ f"[{request_id}] Memory: Could not rebuild the round for a "
+ "continuation; ending the turn after running its memory calls"
+ )
+ for frame in memory_filter.closing_frames(prior_usage=prior_usage):
+ yield frame
+ return
+
+ assert response is not None and messages is not None and isinstance(base_body, dict)
+ round_usage = response.get("usage")
+ if isinstance(round_usage, dict):
+ for key in _ROUND_USAGE_KEYS:
+ if isinstance(value := round_usage.get(key), int):
+ prior_usage[key] = prior_usage.get(key, 0) + value
+ messages = [
+ *messages,
+ {"role": "assistant", "content": content},
+ {"role": "user", "content": tool_results},
+ ]
+ continuation_bytes = json.dumps({**base_body, "messages": messages}).encode("utf-8")
+ assert self.http_client is not None
+ continuation_request = self.http_client.build_request(
+ "POST", url, content=continuation_bytes, headers=headers
+ )
+ continuation = await self.http_client.send(continuation_request, stream=True)
+ rounds += 1
+ async with contextlib.aclosing(continuation) as upstream:
+ if upstream.status_code >= 400:
+ error_bytes = await upstream.aread()
+ logger.warning(
+ f"[{request_id}] Memory: Continuation round {rounds} failed "
+ f"with upstream status {upstream.status_code}"
+ )
+ yield _upstream_error_frame(error_bytes, request_id)
+ return
+ memory_filter = MemoryToolStreamFilter(
+ server_memory_tool_names,
+ index_offset=memory_filter.next_index,
+ forward_message_start=False,
+ )
+ # Same cap as the first round's buffer; past it the round is
+ # still streamed, just not rebuilt for a further continuation.
+ round_bytes: bytearray | None = bytearray()
+ async for chunk in upstream.aiter_bytes():
+ if round_bytes is not None:
+ round_bytes.extend(chunk)
+ if len(round_bytes) > MAX_SSE_BUFFER_SIZE:
+ logger.warning(
+ f"[{request_id}] Memory: Continuation round {rounds} "
+ "exceeded the SSE buffer cap"
+ )
+ round_bytes = None
+ for frame in memory_filter.feed(chunk):
+ yield frame
+ response = (
+ self._parse_sse_to_response(round_bytes.decode("utf-8"), "anthropic")
+ if round_bytes is not None
+ else None
+ )
+ _add_round_usage(stream_state, response)
+ logger.info(f"[{request_id}] Memory: Continuation round {rounds} streamed")
+
+ _MEMORY_CONTINUATION_MAX_ROUNDS = 4
+
def _record_ccr_feedback_from_response(
self, response: dict, provider: str, request_id: str
) -> None:
@@ -802,17 +1014,17 @@ class StreamingMixin:
session_key: str | None = None,
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
+ server_memory_tool_names: frozenset[str] | None = None,
) -> Response | StreamingResponse:
"""Stream response with metrics tracking and memory tool handling.
Parses SSE events to extract actual usage information from the API response
for accurate token counting and cost calculation.
- When memory is enabled (memory_user_id provided), this method:
- 1. Buffers the response to detect memory tool calls
- 2. Executes memory tools if found
- 3. Makes continuation requests until no memory tools remain
- 4. Streams the final response to the client
+ When the proxy injected memory tools (``server_memory_tool_names``),
+ this method streams live but withholds those tool calls from the
+ client, executes them, and streams server-side continuation rounds
+ into the same client message until no memory tool call remains.
"""
session_key = session_key or self._get_session_key(body)
@@ -848,6 +1060,7 @@ class StreamingMixin:
session_key=session_key,
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
+ server_memory_tool_names=server_memory_tool_names,
)
except (Exception, asyncio.CancelledError):
self._cleanup_mid_turn_stream(session_key)
@@ -880,6 +1093,7 @@ class StreamingMixin:
session_key: str,
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
+ server_memory_tool_names: frozenset[str] | None = None,
) -> Response | StreamingResponse:
"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""
from fastapi.responses import Response, StreamingResponse
@@ -1275,6 +1489,13 @@ class StreamingMixin:
parsed_response = None # Set by memory block; used by CCR + prefix tracker
completed_normally = False
pending_messages: list[dict] = []
+ # Proxy-injected memory tools are invisible to the client, so their
+ # tool_use blocks must never reach it (GH #2195).
+ memory_filter = (
+ MemoryToolStreamFilter(server_memory_tool_names)
+ if memory_enabled and server_memory_tool_names
+ else None
+ )
try:
async with contextlib.aclosing(upstream_response) as response:
@@ -1307,7 +1528,11 @@ class StreamingMixin:
# Always stream immediately — buffering breaks
# real-time clients (LangGraph, LangChain, etc.)
- yield chunk
+ if memory_filter is None:
+ yield chunk
+ else:
+ for frame in memory_filter.feed(chunk):
+ yield frame
if _codex_wire_debug:
capture_codex_wire_debug(
@@ -1384,22 +1609,42 @@ class StreamingMixin:
"do not support custom tool injection. Set ANTHROPIC_API_KEY "
"environment variable or use --no-memory-tools flag."
)
+ if memory_filter is not None:
+ for frame in memory_filter.closing_frames():
+ yield frame
return
# Parse SSE to get response JSON
parsed_response = self._parse_sse_to_response(full_sse_data, provider)
- if parsed_response and self.memory_handler.has_memory_tool_calls(
+ if memory_filter is not None:
+ async for frame in self._continue_memory_tool_stream(
+ memory_filter,
+ parsed_response,
+ url=url,
+ outbound_headers=outbound_headers,
+ outbound_bytes=outbound_bytes,
+ memory_user_id=memory_user_id,
+ memory_request_ctx=memory_request_ctx,
+ server_memory_tool_names=server_memory_tool_names or frozenset(),
+ stream_state=stream_state,
+ request_id=request_id,
+ ):
+ yield frame
+ memory_filter = None
+ parsed_response = (
+ stream_state.pop("memory_client_message", None) or parsed_response
+ )
+ elif parsed_response and self.memory_handler.has_memory_tool_calls(
parsed_response, provider
):
logger.info(
f"[{request_id}] Memory: Detected tool calls in streaming response"
)
- # Execute memory tool calls — response already streamed
- # so results are saved but continuation is not possible
- # in SSE streaming mode. The WS and non-streaming paths
- # handle continuation properly.
+ # The client declared these memory tools itself, so the
+ # tool_use already reached it and the client continues
+ # the turn; only the server-side save happens here.
tool_results = await self.memory_handler.handle_memory_tool_calls(
parsed_response,
memory_user_id,
@@ -1413,6 +1658,23 @@ class StreamingMixin:
"continuation handled by client)"
)
+ if memory_filter is not None:
+ # Memory detection was skipped (empty stream or buffer cap):
+ # still run the withheld calls, then end the message.
+ async for frame in self._continue_memory_tool_stream(
+ memory_filter,
+ None,
+ url=url,
+ outbound_headers=outbound_headers,
+ outbound_bytes=outbound_bytes,
+ memory_user_id=memory_user_id,
+ memory_request_ctx=memory_request_ctx,
+ server_memory_tool_names=server_memory_tool_names or frozenset(),
+ stream_state=stream_state,
+ request_id=request_id,
+ ):
+ yield frame
+
# CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.
# In streaming mode, the client handles actual retrieval, but we
# still need to record the event so TOIN learns which fields matter.
@@ -1459,6 +1721,14 @@ class StreamingMixin:
logger.error(f"[{request_id}] HTTP error from upstream API: {e}")
# Forward the upstream error response
yield e.response.content
+ except MemoryToolStreamOverflowError as e:
+ # The filter dropped what it withheld; end the stream rather
+ # than forward it.
+ logger.error(f"[{request_id}] Memory: {e}; ending the stream")
+ error_event = public_errors.anthropic_error_body(
+ public_errors.INTERNAL_ERROR, request_id=request_id
+ )
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
except Exception as e:
logger.error(f"[{request_id}] Unexpected streaming error: {e}")
error_event = public_errors.anthropic_error_body(
diff --git a/headroom/proxy/memory_tool_stream.py b/headroom/proxy/memory_tool_stream.py
new file mode 100644
index 000000000..850cfabff
--- /dev/null
+++ b/headroom/proxy/memory_tool_stream.py
@@ -0,0 +1,316 @@
+"""Hide proxy-handled memory tool calls from an Anthropic SSE stream.
+
+The proxy injects its own memory tools (``memory_save``, ``memory_search``,
+...) into requests. The client never declared them, so when the model calls
+one the client cannot run it: Claude Code answers ``No such tool available``
+and the model concludes the save or search failed (GH #2195).
+
+``MemoryToolStreamFilter`` sits between the upstream SSE stream and the
+client. It forwards every event live except:
+
+* the ``content_block_*`` events of a ``tool_use`` block whose name is a
+ proxy-owned memory tool, which it withholds;
+* ``message_delta`` / ``message_stop``, which it holds until the caller
+ knows whether the turn ends here or continues with a server-side
+ continuation round.
+
+Content block indices are rewritten so the blocks the client sees stay
+contiguous, including across continuation rounds whose blocks are appended
+to the same client message. Frames that need no rewrite are forwarded as the
+exact upstream bytes.
+
+Everything the filter holds back (a pending partial frame, withheld tool
+blocks and their input, held tail frames) is charged its size on the wire
+against ``max_retained_bytes``; past it the filter drops what it holds and
+raises ``MemoryToolStreamOverflowError`` rather than forwarding withheld
+bytes.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Collection
+from typing import Any
+
+from headroom.proxy.sse_byte_buffer_policy import (
+ SSE_DATA_LINE_PREFIX,
+ SSE_EVENT_LINE_PREFIX,
+ find_sse_event_terminator,
+)
+
+logger = logging.getLogger(__name__)
+
+_BLOCK_EVENTS = ("content_block_start", "content_block_delta", "content_block_stop")
+_TAIL_EVENTS = ("message_delta", "message_stop")
+
+# Same budget as the proxy's SSE reconstruction buffer. Legitimate streams
+# retain a few KiB at most: one partial frame plus the memory calls' input.
+DEFAULT_MAX_RETAINED_BYTES = 10 * 1024 * 1024
+
+
+class MemoryToolStreamOverflowError(Exception):
+ """The filter would have to retain more than ``max_retained_bytes``."""
+
+
+def _parse_frame(frame: bytes) -> tuple[str | None, dict[str, Any] | None]:
+ """Return ``(event_name, payload)`` for one complete SSE frame."""
+ event_name: str | None = None
+ data_lines: list[str] = []
+ for line in frame.decode("utf-8").splitlines():
+ if line.startswith(SSE_EVENT_LINE_PREFIX):
+ event_name = line[len(SSE_EVENT_LINE_PREFIX) :].strip()
+ elif line.startswith(SSE_DATA_LINE_PREFIX):
+ data_lines.append(line[len(SSE_DATA_LINE_PREFIX) :].lstrip())
+ if not data_lines:
+ return event_name, None
+ try:
+ payload = json.loads("\n".join(data_lines))
+ except json.JSONDecodeError:
+ return event_name, None
+ if not isinstance(payload, dict):
+ return event_name, None
+ return event_name or payload.get("type"), payload
+
+
+def _render_frame(event_name: str, payload: dict[str, Any]) -> bytes:
+ return f"event: {event_name}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
+class MemoryToolStreamFilter:
+ """Per-round SSE filter; see the module docstring.
+
+ Args:
+ tool_names: Memory tool names the proxy injected and will execute
+ itself. Tools the client declared are never withheld.
+ index_offset: Client-side index of this round's first visible block.
+ Zero for the first round; ``next_index`` of the previous round
+ for a continuation round.
+ forward_message_start: False for continuation rounds, whose blocks
+ extend the message the client already opened.
+ max_retained_bytes: Limit on the bytes held back at any time;
+ ``DEFAULT_MAX_RETAINED_BYTES`` when None.
+ """
+
+ def __init__(
+ self,
+ tool_names: Collection[str],
+ *,
+ index_offset: int = 0,
+ forward_message_start: bool = True,
+ max_retained_bytes: int | None = None,
+ ) -> None:
+ self._tool_names = frozenset(tool_names)
+ self._max_retained_bytes = (
+ DEFAULT_MAX_RETAINED_BYTES if max_retained_bytes is None else max_retained_bytes
+ )
+ # Wire bytes of the frames behind _hidden (per upstream index) and
+ # _tail; _buffer is counted live.
+ self._hidden_bytes: dict[int, int] = {}
+ self._tail_bytes = 0
+ self._forward_message_start = forward_message_start
+ self._buffer = bytearray()
+ self._index_map: dict[int, int] = {}
+ # Withheld tool_use blocks by upstream index, with their streamed
+ # input JSON, so the calls can run even when the whole response
+ # cannot be reconstructed (buffer cap, unparseable stream).
+ self._hidden: dict[int, tuple[dict[str, Any], list[str]]] = {}
+ # Upstream indices of withheld blocks whose content_block_stop arrived.
+ self._hidden_stopped: set[int] = set()
+ self.next_index = index_offset
+ self.visible_tool_use = False
+ self.stop_reason: str | None = None
+ self._tail: list[tuple[str, dict[str, Any] | None, bytes]] = []
+
+ @property
+ def hid_tool_calls(self) -> bool:
+ return bool(self._hidden)
+
+ @property
+ def hidden_tool_names(self) -> list[str]:
+ return [block["name"] for block, _ in self._hidden.values()]
+
+ def hidden_calls(self) -> list[dict[str, Any]]:
+ """The withheld ``tool_use`` blocks that finished with a complete input.
+
+ A call cut off mid-input (e.g. ``max_tokens``) or whose block never
+ reached ``content_block_stop`` (the stream ended early) is left out
+ rather than run: the model never finished requesting it, and some
+ memory tools mutate state.
+ """
+ calls: list[dict[str, Any]] = []
+ for upstream_index, (block, parts) in self._hidden.items():
+ if upstream_index not in self._hidden_stopped:
+ logger.warning(
+ "Memory: skipping %s call %s whose block never finished",
+ block.get("name"),
+ block.get("id"),
+ )
+ continue
+ tool_input: Any = block.get("input") or {}
+ if parts:
+ try:
+ tool_input = json.loads("".join(parts))
+ except json.JSONDecodeError:
+ logger.warning(
+ "Memory: skipping %s call %s with incomplete input",
+ block.get("name"),
+ block.get("id"),
+ )
+ continue
+ if not isinstance(tool_input, dict):
+ continue
+ calls.append({**block, "input": tool_input})
+ return calls
+
+ def feed(self, chunk: bytes) -> list[bytes]:
+ """Consume upstream bytes; return the frames to forward now.
+
+ Raises:
+ MemoryToolStreamOverflowError: The bytes held back exceed
+ ``max_retained_bytes``. The filter has dropped them; the
+ caller must end the stream instead of forwarding anything
+ further.
+ """
+ self._buffer.extend(chunk)
+ out: list[bytes] = []
+ while (match := find_sse_event_terminator(self._buffer)) is not None:
+ idx, terminator_len = match
+ frame = bytes(self._buffer[: idx + terminator_len])
+ del self._buffer[: idx + terminator_len]
+ forwarded = self._route(frame)
+ if forwarded is not None:
+ out.append(forwarded)
+ retained = len(self._buffer) + self._tail_bytes + sum(self._hidden_bytes.values())
+ if retained > self._max_retained_bytes:
+ self._buffer.clear()
+ self._hidden.clear()
+ self._hidden_bytes.clear()
+ self._hidden_stopped.clear()
+ self._tail.clear()
+ self._tail_bytes = 0
+ raise MemoryToolStreamOverflowError(
+ f"memory tool stream filter would retain {retained} bytes "
+ f"(limit {self._max_retained_bytes})"
+ )
+ return out
+
+ def _route(self, frame: bytes) -> bytes | None:
+ event_name, payload = _parse_frame(frame)
+ if payload is None or event_name is None:
+ return frame
+
+ if event_name == "message_start":
+ return frame if self._forward_message_start else None
+
+ if event_name in _TAIL_EVENTS:
+ if event_name == "message_delta":
+ delta = payload.get("delta")
+ if isinstance(delta, dict) and delta.get("stop_reason"):
+ self.stop_reason = delta["stop_reason"]
+ self._tail.append((event_name, payload, frame))
+ self._tail_bytes += len(frame)
+ return None
+
+ if event_name not in _BLOCK_EVENTS:
+ return frame
+
+ upstream_index = payload.get("index")
+ if not isinstance(upstream_index, int):
+ return frame
+
+ if event_name == "content_block_start":
+ block = payload.get("content_block")
+ if (
+ isinstance(block, dict)
+ and block.get("type") == "tool_use"
+ and block.get("name") in self._tool_names
+ ):
+ # A repeated start for the same index replaces the block,
+ # its charge and any earlier stop. The frame may carry the
+ # whole input inline, so it is charged in full.
+ self._hidden[upstream_index] = (block, [])
+ self._hidden_bytes[upstream_index] = len(frame)
+ self._hidden_stopped.discard(upstream_index)
+ return None
+ if isinstance(block, dict) and block.get("type") == "tool_use":
+ self.visible_tool_use = True
+ self._index_map[upstream_index] = self.next_index
+ self.next_index += 1
+
+ if upstream_index in self._hidden:
+ delta = payload.get("delta")
+ if (
+ event_name == "content_block_delta"
+ and isinstance(delta, dict)
+ and delta.get("type") == "input_json_delta"
+ ):
+ self._hidden[upstream_index][1].append(str(delta.get("partial_json", "")))
+ # The frame's size bounds the fragment's, and needs no
+ # re-encoding (a lone surrogate cannot be encoded).
+ self._hidden_bytes[upstream_index] += len(frame)
+ elif event_name == "content_block_stop":
+ self._hidden_stopped.add(upstream_index)
+ return None
+ client_index = self._index_map.get(upstream_index, upstream_index)
+ if client_index == upstream_index:
+ return frame
+ return _render_frame(event_name, {**payload, "index": client_index})
+
+ def closing_frames(self, *, prior_usage: dict[str, int] | None = None) -> list[bytes]:
+ """End the client message after this round without a continuation."""
+ ends_on_hidden_call = (
+ self.hid_tool_calls and not self.visible_tool_use and self.stop_reason == "tool_use"
+ )
+ return self.tail_frames(
+ stop_reason="end_turn" if ends_on_hidden_call else None,
+ prior_usage=prior_usage,
+ )
+
+ def tail_frames(
+ self,
+ *,
+ stop_reason: str | None = None,
+ prior_usage: dict[str, int] | None = None,
+ ) -> list[bytes]:
+ """Return the held ``message_delta`` / ``message_stop`` frames.
+
+ ``stop_reason`` replaces the upstream stop reason, e.g. ``end_turn``
+ when the only tool calls were withheld memory calls: a ``tool_use``
+ stop with no visible ``tool_use`` block leaves the client waiting on
+ a tool it was never shown.
+
+ ``prior_usage`` holds earlier continuation rounds' usage. Their
+ ``message_delta`` frames never reach the client, so it is added to
+ this one: ``output_tokens`` always, other counters where the delta
+ reports them.
+ """
+ frames: list[bytes] = []
+ for event_name, payload, raw in self._tail:
+ if (
+ event_name != "message_delta"
+ or payload is None
+ or not isinstance(payload.get("delta"), dict)
+ or (stop_reason is None and not prior_usage)
+ ):
+ frames.append(raw)
+ continue
+ rendered = dict(payload)
+ if stop_reason is not None:
+ rendered["delta"] = {**payload["delta"], "stop_reason": stop_reason}
+ if prior_usage:
+ usage = dict(payload.get("usage") or {})
+ for key, value in prior_usage.items():
+ if key == "output_tokens" or isinstance(usage.get(key), int):
+ usage[key] = int(usage.get(key) or 0) + value
+ rendered["usage"] = usage
+ frames.append(_render_frame(event_name, rendered))
+ if self._buffer:
+ # The stream ended mid-frame. It may belong to a withheld block,
+ # so drop it rather than risk exposing it to the client.
+ logger.warning(
+ "Memory: dropping %d bytes of an unterminated SSE frame at end of stream",
+ len(self._buffer),
+ )
+ self._buffer.clear()
+ return frames
diff --git a/tests/test_memory_tool_stream.py b/tests/test_memory_tool_stream.py
new file mode 100644
index 000000000..d2e6e45a4
--- /dev/null
+++ b/tests/test_memory_tool_stream.py
@@ -0,0 +1,877 @@
+"""Proxy-injected memory tool calls never reach a streaming client (GH #2195).
+
+Claude Code streams every request. Before this fix the SSE path forwarded a
+``memory_save`` / ``memory_search`` tool_use to the client, which never
+declared those tools and answered ``No such tool available``, so saves looked
+failed and searches never returned.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+from headroom.proxy.memory_tool_stream import (
+ MemoryToolStreamFilter,
+ MemoryToolStreamOverflowError,
+)
+from headroom.proxy.server import HeadroomProxy
+
+MEMORY_TOOLS = frozenset({"memory_save", "memory_search"})
+
+
+def _frame(payload: dict[str, Any]) -> bytes:
+ return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
+def _sse(blocks: list[dict[str, Any]], stop_reason: str, *, input_tokens: int = 10) -> bytes:
+ """An Anthropic SSE response carrying ``blocks``."""
+ frames = [
+ _frame(
+ {
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-test",
+ "content": [],
+ "stop_reason": None,
+ "usage": {"input_tokens": input_tokens, "output_tokens": 1},
+ },
+ }
+ )
+ ]
+ for index, block in enumerate(blocks):
+ if block["type"] == "text":
+ frames.append(
+ _frame(
+ {
+ "type": "content_block_start",
+ "index": index,
+ "content_block": {"type": "text", "text": ""},
+ }
+ )
+ )
+ delta = {"type": "text_delta", "text": block["text"]}
+ else:
+ frames.append(
+ _frame(
+ {
+ "type": "content_block_start",
+ "index": index,
+ "content_block": {**block, "input": {}},
+ }
+ )
+ )
+ delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}
+ frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))
+ frames.append(_frame({"type": "content_block_stop", "index": index}))
+ frames.append(
+ _frame(
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": stop_reason, "stop_sequence": None},
+ "usage": {"output_tokens": 5},
+ }
+ )
+ )
+ frames.append(_frame({"type": "message_stop"}))
+ return b"".join(frames)
+
+
+def _events(raw: bytes) -> list[dict[str, Any]]:
+ return [
+ json.loads(line[len("data: ") :])
+ for line in raw.decode().splitlines()
+ if line.startswith("data: ")
+ ]
+
+
+TEXT = {"type": "text", "text": "Saving that."}
+SAVE = {
+ "type": "tool_use",
+ "id": "toolu_mem",
+ "name": "memory_save",
+ "input": {"content": "deploy region is ap-southeast-1"},
+}
+BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}
+
+
+class TestMemoryToolStreamFilter:
+ def test_stream_without_memory_calls_is_byte_identical(self) -> None:
+ raw = _sse([TEXT, BASH], "tool_use")
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())
+ assert out == raw
+ assert not flt.hid_tool_calls
+
+ def test_byte_at_a_time_feed_matches_one_shot(self) -> None:
+ raw = _sse([TEXT, SAVE], "tool_use")
+ one_shot = MemoryToolStreamFilter(MEMORY_TOOLS)
+ expected = b"".join(one_shot.feed(raw)) + b"".join(one_shot.closing_frames())
+ trickle = MemoryToolStreamFilter(MEMORY_TOOLS)
+ got = b"".join(b"".join(trickle.feed(raw[i : i + 1])) for i in range(len(raw)))
+ got += b"".join(trickle.closing_frames())
+ assert got == expected
+
+ def test_memory_tool_use_is_withheld_and_turn_ends(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ forwarded = _events(b"".join(flt.feed(_sse([TEXT, SAVE], "tool_use"))))
+ assert not any(e.get("content_block", {}).get("type") == "tool_use" for e in forwarded)
+ assert not any(e["type"] in ("message_delta", "message_stop") for e in forwarded)
+ assert flt.hidden_tool_names == ["memory_save"]
+ assert flt.stop_reason == "tool_use"
+
+ tail = _events(b"".join(flt.closing_frames()))
+ assert [e["type"] for e in tail] == ["message_delta", "message_stop"]
+ assert tail[0]["delta"]["stop_reason"] == "end_turn"
+
+ def test_blocks_after_a_hidden_call_are_reindexed(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ forwarded = _events(b"".join(flt.feed(_sse([SAVE, TEXT], "end_turn"))))
+ assert {e["index"] for e in forwarded if "index" in e} == {0}
+ assert flt.next_index == 1
+
+ def test_continuation_round_extends_the_open_message(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, index_offset=1, forward_message_start=False)
+ forwarded = _events(b"".join(flt.feed(_sse([TEXT], "end_turn"))))
+ assert forwarded[0]["type"] == "content_block_start"
+ assert {e["index"] for e in forwarded if "index" in e} == {1}
+
+ def test_client_tool_alongside_memory_call_keeps_tool_use_stop(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ forwarded = _events(b"".join(flt.feed(_sse([SAVE, BASH], "tool_use"))))
+ names = [e["content_block"].get("name") for e in forwarded if "content_block" in e]
+ assert names == ["Bash"]
+ assert flt.visible_tool_use
+ tail = _events(b"".join(flt.closing_frames()))
+ assert tail[0]["delta"]["stop_reason"] == "tool_use"
+
+ def test_client_declared_memory_tool_is_not_withheld(self) -> None:
+ flt = MemoryToolStreamFilter(frozenset({"memory_search"}))
+ forwarded = _events(b"".join(flt.feed(_sse([SAVE], "tool_use"))))
+ assert any(e.get("content_block", {}).get("name") == "memory_save" for e in forwarded)
+ assert not flt.hid_tool_calls
+
+
+def _proxy(upstream_bodies: list[bytes], tool_results: list[dict[str, Any]]) -> HeadroomProxy:
+ proxy = object.__new__(HeadroomProxy)
+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)
+ proxy.metrics = MagicMock()
+ proxy.metrics.record_request = AsyncMock(return_value=None)
+ proxy.metrics.record_failed = AsyncMock(return_value=None)
+ proxy.cost_tracker = MagicMock()
+ proxy.cost_tracker.estimate_cost.return_value = 0.0
+ proxy.stats = {
+ "requests_total": 0,
+ "requests_optimized": 0,
+ "tokens": {"original": 0, "optimized": 0, "saved": 0},
+ "cost": {"total_usd": 0, "savings_usd": 0},
+ "errors": 0,
+ "active_requests": 0,
+ "requests_per_model": {},
+ }
+ proxy._config = MagicMock()
+ proxy._config.ccr_inject_tool = False
+ proxy._config.retry_max_attempts = 1
+ proxy.config = proxy._config
+ proxy.memory_handler = MagicMock()
+ proxy.memory_handler.handle_memory_tool_calls = AsyncMock(return_value=tool_results)
+
+ responses = []
+ for raw in upstream_bodies:
+ response = MagicMock()
+ response.headers = httpx.Headers({"content-type": "text/event-stream"})
+ response.status_code = 200
+
+ async def aiter_bytes(raw: bytes = raw):
+ # Split mid-frame so the filter has to reassemble events.
+ yield raw[:37]
+ yield raw[37:]
+
+ response.aiter_bytes = aiter_bytes
+ response.aclose = AsyncMock()
+ responses.append(response)
+ proxy.http_client.build_request = MagicMock(return_value=MagicMock())
+ proxy.http_client.send = AsyncMock(side_effect=responses)
+ return proxy
+
+
+async def _client_view(proxy: HeadroomProxy, **kwargs: Any) -> list[dict[str, Any]]:
+ result = await proxy._stream_response(
+ url="https://api.anthropic.com/v1/messages",
+ headers={"x-api-key": "sk-test"},
+ body={
+ "model": "claude-test",
+ "max_tokens": 100,
+ "stream": True,
+ "messages": [{"role": "user", "content": "remember the deploy region"}],
+ },
+ provider="anthropic",
+ model="claude-test",
+ request_id="test-mem",
+ original_tokens=10,
+ optimized_tokens=10,
+ tokens_saved=0,
+ transforms_applied=[],
+ tags={},
+ optimization_latency=0.0,
+ memory_user_id="user-1",
+ **kwargs,
+ )
+ raw = b"".join([chunk async for chunk in result.body_iterator])
+ return _events(raw)
+
+
+SAVE_RESULT = [{"type": "tool_result", "tool_use_id": "toolu_mem", "content": '{"status":"saved"}'}]
+
+
+class TestStreamingMemoryContinuation:
+ @pytest.mark.asyncio
+ async def test_memory_call_runs_server_side_and_answer_streams_on(self) -> None:
+ proxy = _proxy(
+ [
+ _sse([TEXT, SAVE], "tool_use"),
+ _sse([{"type": "text", "text": "Saved."}], "end_turn"),
+ ],
+ SAVE_RESULT,
+ )
+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+
+ assert not any(e.get("content_block", {}).get("type") == "tool_use" for e in events)
+ assert [e["type"] for e in events].count("message_start") == 1
+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
+ assert texts == ["Saving that.", "Saved."]
+ starts = [e["index"] for e in events if e["type"] == "content_block_start"]
+ assert starts == [0, 1]
+ deltas = [e for e in events if e["type"] == "message_delta"]
+ assert len(deltas) == 1 and deltas[0]["delta"]["stop_reason"] == "end_turn"
+ assert events[-1]["type"] == "message_stop"
+
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+ continuation = json.loads(
+ proxy.http_client.build_request.call_args_list[1].kwargs["content"]
+ )
+ assert [m["role"] for m in continuation["messages"]] == ["user", "assistant", "user"]
+ assert continuation["messages"][1]["content"][1]["name"] == "memory_save"
+ assert continuation["messages"][2]["content"] == SAVE_RESULT
+
+ @pytest.mark.asyncio
+ async def test_finalizer_sees_the_message_the_client_received(self) -> None:
+ proxy = _proxy(
+ [
+ _sse([TEXT, SAVE], "tool_use"),
+ _sse([{"type": "text", "text": "Saved."}], "end_turn"),
+ ],
+ SAVE_RESULT,
+ )
+ proxy._finalize_stream_response = AsyncMock(return_value=None)
+ await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+
+ finalized = proxy._finalize_stream_response.await_args.kwargs["parsed_response"]
+ assert [b.get("text") for b in finalized["content"]] == ["Saving that.", "Saved."]
+
+ @pytest.mark.asyncio
+ async def test_client_tool_in_same_round_returns_turn_to_client(self) -> None:
+ proxy = _proxy([_sse([SAVE, BASH], "tool_use")], SAVE_RESULT)
+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+
+ names = [e["content_block"].get("name") for e in events if "content_block" in e]
+ assert names == ["Bash"]
+ assert [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"] == [
+ "tool_use"
+ ]
+ assert proxy.http_client.send.await_count == 1
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_without_injected_tools_stream_passes_through(self) -> None:
+ raw = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([raw], SAVE_RESULT)
+ proxy.memory_handler.has_memory_tool_calls = MagicMock(return_value=False)
+ events = await _client_view(proxy)
+ assert events == _events(raw)
+
+
+class TestRecordedMemoryCalls:
+ def test_hidden_call_input_is_rebuilt_from_stream(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_sse([TEXT, SAVE], "tool_use"))
+ assert flt.hidden_calls() == [SAVE]
+
+ def test_call_with_truncated_input_is_not_run(self) -> None:
+ raw = _sse([SAVE], "max_tokens").replace(b'\\"deploy region', b'\\"deploy', 1)
+ raw = raw.replace(b'ap-southeast-1\\"}', b"ap-southeast-1", 1)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(raw)
+ assert flt.hid_tool_calls
+ assert flt.hidden_calls() == []
+
+ def test_call_whose_block_never_stopped_is_not_run(self) -> None:
+ # Complete, valid input, but the stream ends before content_block_stop.
+ raw = _sse([SAVE], "tool_use")
+ raw = raw[: raw.index(b"event: content_block_stop")]
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(raw)
+ assert flt.hid_tool_calls
+ assert flt.hidden_calls() == []
+
+ def test_prior_rounds_usage_is_added_to_the_final_delta(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_sse([TEXT], "end_turn"))
+ tail = _events(b"".join(flt.closing_frames(prior_usage={"output_tokens": 7})))
+ assert tail[0]["usage"]["output_tokens"] == 12
+
+
+async def _drain(gen: Any) -> list[dict[str, Any]]:
+ return _events(b"".join([frame async for frame in gen]))
+
+
+def _continue(proxy: HeadroomProxy, flt: MemoryToolStreamFilter, response: Any) -> Any:
+ body = {"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}
+ return proxy._continue_memory_tool_stream(
+ flt,
+ response,
+ url="https://api.anthropic.com/v1/messages",
+ outbound_headers={"x-api-key": "sk-test", "content-length": "1"},
+ outbound_bytes=json.dumps(body).encode(),
+ memory_user_id="user-1",
+ memory_request_ctx=None,
+ server_memory_tool_names=MEMORY_TOOLS,
+ stream_state={},
+ request_id="test-mem",
+ )
+
+
+class TestContinuationEdgeCases:
+ @pytest.mark.asyncio
+ async def test_unrebuilt_round_still_runs_its_memory_calls(self) -> None:
+ proxy = _proxy([], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_sse([TEXT, SAVE], "tool_use"))
+
+ events = await _drain(_continue(proxy, flt, None))
+
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+ ran = proxy.memory_handler.handle_memory_tool_calls.await_args.args[0]["content"]
+ assert ran == [SAVE]
+ assert proxy.http_client.send.await_count == 0
+ assert events[0]["delta"]["stop_reason"] == "end_turn"
+
+ @pytest.mark.asyncio
+ async def test_continuation_keeps_streaming_past_the_buffer_cap(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import headroom.proxy.helpers as helpers
+
+ round_one = _sse([TEXT, SAVE], "tool_use")
+ reply = _sse([{"type": "text", "text": "Saved."}], "end_turn")
+ proxy = _proxy([reply], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(round_one)
+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")
+ # _proxy splits each body in two; the second chunk arrives past the cap.
+ monkeypatch.setattr(helpers, "MAX_SSE_BUFFER_SIZE", 16)
+
+ events = await _drain(_continue(proxy, flt, response))
+
+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
+ assert texts == ["Saved."]
+ assert [e["type"] for e in events][-2:] == ["message_delta", "message_stop"]
+
+ @pytest.mark.asyncio
+ async def test_interrupted_call_is_not_run(self) -> None:
+ raw = _sse([TEXT, SAVE], "tool_use")
+ raw = raw[: raw.rindex(b"event: content_block_stop")]
+ proxy = _proxy([], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(raw)
+
+ await _drain(_continue(proxy, flt, None))
+
+ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()
+ assert proxy.http_client.send.await_count == 0
+
+ @pytest.mark.asyncio
+ async def test_continuation_over_the_buffer_cap_is_not_continued_again(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import headroom.proxy.helpers as helpers
+
+ round_one = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([_sse([SAVE], "tool_use")], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(round_one)
+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")
+ monkeypatch.setattr(helpers, "MAX_SSE_BUFFER_SIZE", 64)
+
+ events = await _drain(_continue(proxy, flt, response))
+
+ assert proxy.http_client.send.await_count == 1
+ assert proxy.memory_handler.handle_memory_tool_calls.await_count == 2
+ assert [e["type"] for e in events][-2:] == ["message_delta", "message_stop"]
+ assert events[-2]["delta"]["stop_reason"] == "end_turn"
+
+ @pytest.mark.asyncio
+ async def test_client_sees_output_usage_of_every_round(self) -> None:
+ proxy = _proxy(
+ [
+ _sse([TEXT, SAVE], "tool_use"),
+ _sse([{"type": "text", "text": "Saved."}], "end_turn"),
+ ],
+ SAVE_RESULT,
+ )
+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+ deltas = [e for e in events if e["type"] == "message_delta"]
+ assert deltas[0]["usage"]["output_tokens"] == 10
+
+
+class TestFrameParsingEdgeCases:
+ """Frames the filter cannot interpret pass through to the client untouched."""
+
+ @pytest.mark.parametrize(
+ "frame",
+ [
+ b": keep-alive\n\n",
+ b"event: ping\n\n",
+ b"event: content_block_delta\ndata: {not json\n\n",
+ b"event: content_block_delta\ndata: [1, 2]\n\n",
+ b'event: ping\ndata: {"type": "ping"}\n\n',
+ b'event: content_block_delta\ndata: {"type": "content_block_delta", "index": "0"}\n\n',
+ ],
+ )
+ def test_uninterpretable_frame_is_forwarded_verbatim(self, frame: bytes) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ assert flt.feed(frame) == [frame]
+
+ def test_hidden_call_with_non_object_input_is_not_run(self) -> None:
+ raw = _sse([SAVE], "tool_use").replace(
+ json.dumps(json.dumps(SAVE["input"])).encode(), json.dumps("[1]").encode(), 1
+ )
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(raw)
+ assert flt.hid_tool_calls
+ assert flt.hidden_calls() == []
+
+ def test_hidden_call_with_input_in_its_start_frame(self) -> None:
+ raw = b"".join(
+ [
+ _frame({"type": "content_block_start", "index": 0, "content_block": SAVE}),
+ _frame({"type": "content_block_stop", "index": 0}),
+ ]
+ )
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ assert flt.feed(raw) == []
+ assert flt.hidden_calls() == [SAVE]
+
+ def test_other_delta_on_hidden_block_is_withheld(self) -> None:
+ raw = b"".join(
+ [
+ _frame({"type": "content_block_start", "index": 0, "content_block": SAVE}),
+ _frame({"type": "content_block_delta", "index": 0, "delta": {"type": "x"}}),
+ _frame({"type": "content_block_stop", "index": 0}),
+ ]
+ )
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ assert flt.feed(raw) == []
+ assert flt.hidden_calls() == [SAVE]
+
+ def test_message_delta_without_stop_reason_keeps_none(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_frame({"type": "message_delta", "delta": {}, "usage": {"output_tokens": 1}}))
+ assert flt.stop_reason is None
+
+ def test_unterminated_trailing_frame_is_dropped(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_sse([TEXT], "end_turn"))
+ flt.feed(b"event: ping\ndata: {")
+ assert b"ping" not in b"".join(flt.closing_frames())
+
+ def test_truncated_hidden_delta_never_reaches_the_client(self) -> None:
+ secret = "my private deploy key"
+ start = _frame({"type": "content_block_start", "index": 0, "content_block": SAVE})
+ delta = _frame(
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "input_json_delta", "partial_json": f'{{"content": "{secret}'},
+ }
+ )
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ emitted = flt.feed(start + delta[:-2]) # the stream ends before the terminator
+ emitted += flt.closing_frames()
+ assert secret.encode() not in b"".join(emitted)
+ assert flt.hidden_calls() == []
+
+
+class TestRetainedByteLimit:
+ def test_unterminated_event_stops_growing_at_the_limit(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=4096)
+ chunk = b"event: content_block_delta\ndata: " + b"x" * 1024
+ with pytest.raises(MemoryToolStreamOverflowError):
+ for _ in range(12):
+ flt.feed(chunk)
+ assert flt.closing_frames() == []
+
+ def test_oversized_hidden_input_is_dropped_not_run(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=4096)
+ flt.feed(_frame({"type": "content_block_start", "index": 0, "content_block": SAVE}))
+ fragment = {"type": "input_json_delta", "partial_json": "y" * 1024}
+ with pytest.raises(MemoryToolStreamOverflowError):
+ for _ in range(12):
+ flt.feed(_frame({"type": "content_block_delta", "index": 0, "delta": fragment}))
+ assert not flt.hid_tool_calls
+ assert flt.hidden_calls() == []
+ assert flt.closing_frames() == []
+
+ def test_inline_hidden_input_counts_against_the_limit(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=4096)
+ block = {**SAVE, "input": {"content": "z" * 20000}}
+ with pytest.raises(MemoryToolStreamOverflowError):
+ flt.feed(_frame({"type": "content_block_start", "index": 0, "content_block": block}))
+ assert not flt.hid_tool_calls
+
+ def test_limit_is_in_bytes_at_the_boundary(self) -> None:
+ start = _frame({"type": "content_block_start", "index": 0, "content_block": SAVE})
+ snowmen = {"type": "input_json_delta", "partial_json": "\u2603" * 24}
+ payload = {"type": "content_block_delta", "index": 0, "delta": snowmen}
+ # Raw UTF-8 on the wire: 24 characters, 72 bytes.
+ delta = f"event: {payload['type']}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n"
+ frames = start + delta.encode()
+
+ MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(frames)).feed(frames)
+ with pytest.raises(MemoryToolStreamOverflowError):
+ MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(frames) - 1).feed(frames)
+
+ def test_lone_surrogate_in_hidden_input_is_counted(self) -> None:
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_frame({"type": "content_block_start", "index": 0, "content_block": SAVE}))
+ fragment = {"type": "input_json_delta", "partial_json": '{"content": "\ud800"}'}
+ frame = _frame({"type": "content_block_delta", "index": 0, "delta": fragment})
+ assert flt.feed(frame) == []
+ assert flt.feed(_frame({"type": "content_block_stop", "index": 0})) == []
+ assert flt.hidden_tool_names == ["memory_save"]
+
+ def test_repeated_hidden_start_replaces_its_charge(self) -> None:
+ first = {**SAVE, "input": {"content": "a" * 300}}
+ second = {**SAVE, "input": {"content": "b" * 300}}
+ start_one = _frame({"type": "content_block_start", "index": 0, "content_block": first})
+ start_two = _frame({"type": "content_block_start", "index": 0, "content_block": second})
+ stop = _frame({"type": "content_block_stop", "index": 0})
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(start_two) + 1)
+ flt.feed(start_one)
+ flt.feed(stop)
+ flt.feed(start_two)
+ assert flt.hidden_calls() == [] # the replacement has not stopped yet
+ flt.feed(stop)
+ assert flt.hidden_calls() == [second]
+
+ def test_frames_within_the_limit_pass(self) -> None:
+ raw = _sse([TEXT, SAVE], "tool_use")
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(raw))
+ flt.feed(raw)
+ assert flt.hidden_calls() == [SAVE]
+
+
+class TestUpstreamErrorFrame:
+ def test_anthropic_error_envelope_passes_through(self) -> None:
+ from headroom.proxy.handlers.streaming import _upstream_error_frame
+
+ envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "busy"}}
+ [event] = _events(_upstream_error_frame(json.dumps(envelope).encode(), "req-1"))
+ assert event == envelope
+
+ @pytest.mark.parametrize(
+ "body", [b"<html>Bad Gateway</html>", b"\xff\xfe", b'{"detail": "nope"}']
+ )
+ def test_other_bodies_become_the_public_error(self, body: bytes) -> None:
+ from headroom.proxy.handlers.streaming import _upstream_error_frame
+
+ raw = _upstream_error_frame(body, "req-1")
+ assert raw.startswith(b"event: error\n")
+ [event] = _events(raw)
+ assert event["type"] == "error"
+ assert body.decode("utf-8", "replace") not in json.dumps(event)
+
+
+class TestContinuationFallbacks:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "outbound", [b"not json", json.dumps({"model": "claude-test", "messages": "hi"}).encode()]
+ )
+ async def test_unreadable_request_body_ends_the_turn_after_running_calls(
+ self, outbound: bytes
+ ) -> None:
+ round_one = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(round_one)
+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")
+
+ events = await _drain(
+ proxy._continue_memory_tool_stream(
+ flt,
+ response,
+ url="https://api.anthropic.com/v1/messages",
+ outbound_headers={"x-api-key": "sk-test"},
+ outbound_bytes=outbound,
+ memory_user_id="user-1",
+ memory_request_ctx=None,
+ server_memory_tool_names=MEMORY_TOOLS,
+ stream_state={},
+ request_id="test-mem",
+ )
+ )
+
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+ assert proxy.http_client.send.await_count == 0
+ assert events[0]["delta"]["stop_reason"] == "end_turn"
+
+ @pytest.mark.asyncio
+ async def test_round_limit_stops_continuing(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ round_one = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([], SAVE_RESULT)
+ monkeypatch.setattr(type(proxy), "_MEMORY_CONTINUATION_MAX_ROUNDS", 0)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(round_one)
+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")
+
+ events = await _drain(_continue(proxy, flt, response))
+
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+ assert proxy.http_client.send.await_count == 0
+ assert events[0]["delta"]["stop_reason"] == "end_turn"
+
+ @pytest.mark.asyncio
+ async def test_failed_continuation_round_sends_an_error_event(self) -> None:
+ round_one = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([], SAVE_RESULT)
+ failed = MagicMock()
+ failed.status_code = 529
+ envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "busy"}}
+ failed.aread = AsyncMock(return_value=json.dumps(envelope).encode())
+ failed.aclose = AsyncMock()
+ proxy.http_client.send = AsyncMock(return_value=failed)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(round_one)
+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")
+
+ events = await _drain(_continue(proxy, flt, response))
+
+ assert proxy.http_client.send.await_count == 1
+ assert events == [envelope]
+ failed.aclose.assert_awaited()
+
+ @pytest.mark.asyncio
+ async def test_calls_are_not_run_without_a_memory_user(self) -> None:
+ proxy = _proxy([], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(_sse([TEXT, SAVE], "tool_use"))
+
+ events = await _drain(
+ proxy._continue_memory_tool_stream(
+ flt,
+ None,
+ url="https://api.anthropic.com/v1/messages",
+ outbound_headers={"x-api-key": "sk-test"},
+ outbound_bytes=b"{}",
+ memory_user_id=None,
+ memory_request_ctx=None,
+ server_memory_tool_names=MEMORY_TOOLS,
+ stream_state={},
+ request_id="test-mem",
+ )
+ )
+
+ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()
+ assert events[0]["delta"]["stop_reason"] == "end_turn"
+
+ @pytest.mark.asyncio
+ async def test_round_without_usage_still_continues(self) -> None:
+ round_one = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([_sse([{"type": "text", "text": "Saved."}], "end_turn")], SAVE_RESULT)
+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)
+ flt.feed(round_one)
+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")
+ response.pop("usage", None)
+
+ events = await _drain(_continue(proxy, flt, response))
+
+ assert proxy.http_client.send.await_count == 1
+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]
+ assert texts == ["Saved."]
+ assert events[-2]["delta"]["stop_reason"] == "end_turn"
+
+
+class TestStreamingMemoryFallbacks:
+ @pytest.mark.asyncio
+ async def test_client_handled_memory_calls_still_run_without_server_tools(self) -> None:
+ raw = _sse([TEXT, SAVE], "tool_use")
+ proxy = _proxy([raw], SAVE_RESULT)
+ proxy.memory_handler.has_memory_tool_calls = MagicMock(return_value=True)
+ events = await _client_view(proxy)
+
+ assert events == _events(raw)
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("server_tools", [MEMORY_TOOLS, frozenset()])
+ async def test_subscription_credential_error_ends_the_message(
+ self, server_tools: frozenset[str]
+ ) -> None:
+ refusal = {
+ "type": "text",
+ "text": "This credential is only authorized for use with Claude Code.",
+ }
+ proxy = _proxy([_sse([refusal], "end_turn")], SAVE_RESULT)
+ events = await _client_view(proxy, server_memory_tool_names=server_tools)
+
+ assert proxy.http_client.send.await_count == 1
+ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()
+ assert [e["type"] for e in events][-2:] == ["message_delta", "message_stop"]
+
+ @pytest.mark.asyncio
+ async def test_buffer_cap_still_runs_withheld_calls(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import headroom.proxy.helpers as helpers
+
+ proxy = _proxy([_sse([TEXT, SAVE], "tool_use")], SAVE_RESULT)
+ monkeypatch.setattr(helpers, "MAX_SSE_BUFFER_SIZE", 64)
+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+
+ assert not any(e.get("content_block", {}).get("type") == "tool_use" for e in events)
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+ assert proxy.http_client.send.await_count == 1
+ deltas = [e for e in events if e["type"] == "message_delta"]
+ assert deltas[-1]["delta"]["stop_reason"] == "end_turn"
+
+
+class TestServerMemoryToolNames:
+ def test_injected_memory_tools_are_server_side(self) -> None:
+ from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
+
+ tools = [
+ {"name": "Bash"},
+ {"name": "memory_save"},
+ {"name": "memory_search"},
+ {"type": "memory_20250818", "name": "memory"},
+ {"type": "web_search"},
+ "not-a-tool",
+ ]
+ names = AnthropicHandlerMixin._server_memory_tool_names(tools, [{"name": "Bash"}])
+ assert names == frozenset({"memory_save", "memory_search", "memory"})
+
+ def test_client_declared_memory_tool_stays_with_the_client(self) -> None:
+ from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin
+
+ tools = [{"name": "memory_save"}, {"name": "memory_search"}]
+ names = AnthropicHandlerMixin._server_memory_tool_names(
+ tools, [{"name": "memory_save"}, "junk"]
+ )
+ assert names == frozenset({"memory_search"})
+ assert AnthropicHandlerMixin._server_memory_tool_names(None, None) == frozenset()
+
+
+class TestHandlerPassesServerMemoryTools:
+ def test_injected_memory_tools_reach_the_stream(self) -> None:
+ from types import SimpleNamespace
+
+ from fastapi.responses import StreamingResponse
+ from fastapi.testclient import TestClient
+
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+ captured: dict[str, Any] = {}
+
+ async def fake_stream_response(*args: Any, **kwargs: Any) -> StreamingResponse:
+ captured.update(kwargs)
+
+ async def gen() -> Any:
+ yield b""
+
+ return StreamingResponse(gen(), media_type="text/event-stream")
+
+ memory_tools = [
+ {"name": name, "description": name, "input_schema": {"type": "object"}}
+ for name in sorted(MEMORY_TOOLS)
+ ]
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ proxy.memory_handler = SimpleNamespace(
+ config=SimpleNamespace(inject_context=False, inject_tools=True),
+ compute_memory_tool_definitions=lambda provider: memory_tools,
+ get_beta_headers=lambda: {},
+ has_memory_tool_calls=lambda resp, provider: False,
+ )
+ proxy._stream_response = fake_stream_response
+ client.post(
+ "/v1/messages",
+ headers={
+ "x-api-key": "test-key",
+ "anthropic-version": "2023-06-01",
+ "x-headroom-user-id": "u1",
+ },
+ json={
+ "model": "claude-sonnet-4-6",
+ "max_tokens": 64,
+ "stream": True,
+ "tools": [{"name": "Bash", "input_schema": {"type": "object"}}],
+ "messages": [{"role": "user", "content": "remember the deploy region"}],
+ },
+ )
+
+ assert captured["server_memory_tool_names"] == MEMORY_TOOLS
+
+
+class TestStreamingRetainedByteLimit:
+ SECRET = "s3cr3t-" * 256
+
+ def _oversized_save(self) -> bytes:
+ save = {**SAVE, "input": {"content": self.SECRET}}
+ return _sse([TEXT, save], "tool_use")
+
+ @pytest.mark.asyncio
+ async def test_overflow_ends_the_stream_with_an_error(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import headroom.proxy.memory_tool_stream as mts
+
+ monkeypatch.setattr(mts, "DEFAULT_MAX_RETAINED_BYTES", 256)
+ proxy = _proxy([self._oversized_save()], SAVE_RESULT)
+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+
+ assert events[-1]["type"] == "error"
+ assert self.SECRET not in json.dumps(events)
+ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_continuation_overflow_ends_the_stream_with_an_error(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import headroom.proxy.memory_tool_stream as mts
+
+ monkeypatch.setattr(mts, "DEFAULT_MAX_RETAINED_BYTES", 1024)
+ proxy = _proxy([_sse([TEXT, SAVE], "tool_use"), self._oversized_save()], SAVE_RESULT)
+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)
+
+ assert proxy.http_client.send.await_count == 2
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()
+ assert events[-1]["type"] == "error"
+ assert self.SECRET not in json.dumps(events)