Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): run injected memory tools server-side on streaming turns (#3947)
Kenneth Wong1cf496612e78
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 2974e602f..6e507c032 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -431,6 +431,25 @@ class AnthropicHandlerMixin:canonical = str(tool)return (name, canonical)+ @staticmethod+ def _server_memory_tool_names(tools: Any, client_tools: Any) -> frozenset[str]:+ """Memory tools in ``tools`` that the proxy injected and must run itself.++ A memory tool the client declared stays the client's: its calls are+ forwarded, not withheld.+ """+ from headroom.proxy.memory_handler import MEMORY_TOOL_NAMES, NATIVE_MEMORY_TOOL_NAME++ client_tool_names = {t.get("name") for t in client_tools or [] if isinstance(t, dict)}+ return frozenset(+ name+ for t in tools or []+ if isinstance(t, dict)+ and isinstance(name := t.get("name"), str)+ and (name in MEMORY_TOOL_NAMES or name == NATIVE_MEMORY_TOOL_NAME)+ and name not in client_tool_names+ )+@staticmethoddef _has_headroom_retrieve_tool(tools: Any) -> bool:"""Return True when the final Anthropic tool list includes CCR retrieve."""@@ -2737,6 +2756,9 @@ class AnthropicHandlerMixin:# /v1/messages just as on /v1/responses.memory_context_injected = Falsememory_tools_injected = False+ # Memory tools this proxy injected and must execute itself; the+ # streaming path withholds their calls from the client.+ server_memory_tool_names: frozenset[str] = frozenset()if memory_decision.inject:# Search and inject memory contextif self.memory_handler.config.inject_context:@@ -2848,6 +2870,9 @@ class AnthropicHandlerMixin:)if mem_tools_injected:memory_tools_injected = True+ server_memory_tool_names = self._server_memory_tool_names(+ tools, _original_tools+ )tool_names = [t.get("name") or t.get("type", "")for t in tools@@ -4067,6 +4092,7 @@ class AnthropicHandlerMixin:memory_request_ctx=memory_request_ctx,outcome_provider=provider_name,session_key=session_key,+ server_memory_tool_names=server_memory_tool_names,)else:# Whatever set it — the client's own ``stream: false`` ordiff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex d3ec34764..810357dc9 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -10,6 +10,7 @@ import contextlibimport jsonimport loggingimport time+from collections.abc import AsyncIteratorfrom typing import TYPE_CHECKING, Anyfrom headroom.proxy.auth_mode import classify_client, supports_mid_turn_coalescing@@ -29,12 +30,55 @@ import httpxfrom headroom.copilot_auth import apply_copilot_api_authfrom headroom.proxy import public_errors+from headroom.proxy.memory_tool_stream import (+ MemoryToolStreamFilter,+ MemoryToolStreamOverflowError,+)from headroom.proxy.stream_output_tokens import estimate_output_tokensfrom headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokensfrom headroom.utils import format_exception_messagelogger = logging.getLogger("headroom.proxy")+_ROUND_USAGE_KEYS = (+ "input_tokens",+ "output_tokens",+ "cache_read_input_tokens",+ "cache_creation_input_tokens",+)+++def _add_round_usage(stream_state: dict[str, Any], response: dict[str, Any] | None) -> None:+ """Add a server-side continuation round's usage to the turn's totals."""+ usage = response.get("usage") if isinstance(response, dict) else None+ if not isinstance(usage, dict):+ return+ for key in _ROUND_USAGE_KEYS:+ value = usage.get(key)+ if isinstance(value, int):+ stream_state[key] = (stream_state.get(key) or 0) + value+++def _upstream_error_frame(error_bytes: bytes, request_id: str) -> bytes:+ """Render an upstream error response as an Anthropic SSE ``error`` event.++ An Anthropic error envelope is addressed to the caller and passes through+ as is; anything else becomes the generic public error.+ """+ try:+ payload = json.loads(error_bytes)+ except (json.JSONDecodeError, UnicodeDecodeError):+ payload = None+ if not (+ isinstance(payload, dict)+ and payload.get("type") == "error"+ and isinstance(payload.get("error"), dict)+ ):+ payload = public_errors.anthropic_error_body(+ public_errors.INTERNAL_ERROR, request_id=request_id+ )+ return f"event: error\ndata: {json.dumps(payload)}\n\n".encode()+def _thinking_for_stream(payload: object) -> ThinkingTokens:"""Thinking-token split for a reassembled streaming response.@@ -436,6 +480,174 @@ class StreamingMixin:return render_anthropic_sse_response(response)+ async def _continue_memory_tool_stream(+ self,+ memory_filter: MemoryToolStreamFilter,+ response: dict[str, Any] | None,+ *,+ url: str,+ outbound_headers: dict[str, str],+ outbound_bytes: bytes,+ memory_user_id: str | None,+ memory_request_ctx: Any | None,+ server_memory_tool_names: frozenset[str],+ stream_state: dict[str, Any],+ request_id: str,+ ) -> AsyncIterator[bytes]:+ """Execute withheld memory tool calls and stream continuation rounds.++ ``memory_filter`` has forwarded one upstream round to the client minus+ the proxy-owned memory tool calls and the message tail. When the round+ stopped on memory tool calls alone, run them, send the results upstream+ as the next turn, and stream that round's blocks into the same client+ message. Otherwise end the client message. A round that also called a+ client tool still has its memory calls executed, but the turn goes back+ to the client, which cannot carry the memory results.+ """+ from headroom.proxy.helpers import MAX_SSE_BUFFER_SIZE++ try:+ base_body = json.loads(outbound_bytes)+ except (json.JSONDecodeError, UnicodeDecodeError):+ base_body = None+ messages = base_body.get("messages") if isinstance(base_body, dict) else None+ if not isinstance(messages, list):+ messages = None+ headers = {k: v for k, v in outbound_headers.items() if k.lower() != "content-length"}++ def is_memory_call(block: Any) -> bool:+ return (+ isinstance(block, dict)+ and block.get("type") == "tool_use"+ and block.get("name") in server_memory_tool_names+ )++ rounds = 0+ # What the client received across rounds; the prefix tracker must see+ # this, not round one, because it is what the client sends back.+ visible_content: list[Any] = []+ # Usage of rounds whose message_delta the client never receives.+ prior_usage: dict[str, int] = {}+ while True:+ content: list[Any] = []+ if response is not None:+ round_content = response.get("content")+ content = round_content if isinstance(round_content, list) else []+ visible_content.extend(block for block in content if not is_memory_call(block))+ stream_state["memory_client_message"] = {+ **response,+ "content": list(visible_content),+ }++ if not memory_filter.hid_tool_calls:+ for frame in memory_filter.closing_frames(prior_usage=prior_usage):+ yield frame+ return++ # Run the calls the filter recorded, so they are not lost when the+ # round could not be reconstructed. Only the proxy's own calls: a+ # memory-named tool the client declared is the client's to run.+ memory_calls = memory_filter.hidden_calls()+ tool_results: list[dict[str, Any]] = []+ if memory_calls and memory_user_id is not None and self.memory_handler is not None:+ tool_results = await self.memory_handler.handle_memory_tool_calls(+ {"content": memory_calls},+ memory_user_id,+ "anthropic",+ request_context=memory_request_ctx,+ )+ logger.info(+ f"[{request_id}] Memory: Executed {len(tool_results)}/"+ f"{len(memory_filter.hidden_tool_names)} proxy-handled tool call(s) "+ "withheld from the SSE stream"+ )++ replayable = response is not None and len(tool_results) == sum(+ 1 for block in content if is_memory_call(block)+ )+ can_continue = (+ memory_filter.stop_reason == "tool_use"+ and not memory_filter.visible_tool_use+ and messages is not None+ and replayable+ and rounds < self._MEMORY_CONTINUATION_MAX_ROUNDS+ )+ if not can_continue:+ if memory_filter.visible_tool_use:+ logger.info(+ f"[{request_id}] Memory: Round also called a client tool; "+ "returning the turn to the client"+ )+ elif rounds >= self._MEMORY_CONTINUATION_MAX_ROUNDS:+ logger.warning(+ f"[{request_id}] Memory: Stopped after {rounds} continuation "+ "round(s) that only called memory tools"+ )+ elif not replayable:+ logger.warning(+ f"[{request_id}] Memory: Could not rebuild the round for a "+ "continuation; ending the turn after running its memory calls"+ )+ for frame in memory_filter.closing_frames(prior_usage=prior_usage):+ yield frame+ return++ assert response is not None and messages is not None and isinstance(base_body, dict)+ round_usage = response.get("usage")+ if isinstance(round_usage, dict):+ for key in _ROUND_USAGE_KEYS:+ if isinstance(value := round_usage.get(key), int):+ prior_usage[key] = prior_usage.get(key, 0) + value+ messages = [+ *messages,+ {"role": "assistant", "content": content},+ {"role": "user", "content": tool_results},+ ]+ continuation_bytes = json.dumps({**base_body, "messages": messages}).encode("utf-8")+ assert self.http_client is not None+ continuation_request = self.http_client.build_request(+ "POST", url, content=continuation_bytes, headers=headers+ )+ continuation = await self.http_client.send(continuation_request, stream=True)+ rounds += 1+ async with contextlib.aclosing(continuation) as upstream:+ if upstream.status_code >= 400:+ error_bytes = await upstream.aread()+ logger.warning(+ f"[{request_id}] Memory: Continuation round {rounds} failed "+ f"with upstream status {upstream.status_code}"+ )+ yield _upstream_error_frame(error_bytes, request_id)+ return+ memory_filter = MemoryToolStreamFilter(+ server_memory_tool_names,+ index_offset=memory_filter.next_index,+ forward_message_start=False,+ )+ # Same cap as the first round's buffer; past it the round is+ # still streamed, just not rebuilt for a further continuation.+ round_bytes: bytearray | None = bytearray()+ async for chunk in upstream.aiter_bytes():+ if round_bytes is not None:+ round_bytes.extend(chunk)+ if len(round_bytes) > MAX_SSE_BUFFER_SIZE:+ logger.warning(+ f"[{request_id}] Memory: Continuation round {rounds} "+ "exceeded the SSE buffer cap"+ )+ round_bytes = None+ for frame in memory_filter.feed(chunk):+ yield frame+ response = (+ self._parse_sse_to_response(round_bytes.decode("utf-8"), "anthropic")+ if round_bytes is not None+ else None+ )+ _add_round_usage(stream_state, response)+ logger.info(f"[{request_id}] Memory: Continuation round {rounds} streamed")++ _MEMORY_CONTINUATION_MAX_ROUNDS = 4+def _record_ccr_feedback_from_response(self, response: dict, provider: str, request_id: str) -> None:@@ -802,17 +1014,17 @@ class StreamingMixin:session_key: str | None = None,conversation_key: str | None = None,conversation_tokens_saved: int | None = None,+ server_memory_tool_names: frozenset[str] | None = None,) -> Response | StreamingResponse:"""Stream response with metrics tracking and memory tool handling.Parses SSE events to extract actual usage information from the API responsefor accurate token counting and cost calculation.- When memory is enabled (memory_user_id provided), this method:- 1. Buffers the response to detect memory tool calls- 2. Executes memory tools if found- 3. Makes continuation requests until no memory tools remain- 4. Streams the final response to the client+ When the proxy injected memory tools (``server_memory_tool_names``),+ this method streams live but withholds those tool calls from the+ client, executes them, and streams server-side continuation rounds+ into the same client message until no memory tool call remains."""session_key = session_key or self._get_session_key(body)@@ -848,6 +1060,7 @@ class StreamingMixin:session_key=session_key,conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,+ server_memory_tool_names=server_memory_tool_names,)except (Exception, asyncio.CancelledError):self._cleanup_mid_turn_stream(session_key)@@ -880,6 +1093,7 @@ class StreamingMixin:session_key: str,conversation_key: str | None = None,conversation_tokens_saved: int | None = None,+ server_memory_tool_names: frozenset[str] | None = None,) -> Response | StreamingResponse:"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""from fastapi.responses import Response, StreamingResponse@@ -1275,6 +1489,13 @@ class StreamingMixin:parsed_response = None # Set by memory block; used by CCR + prefix trackercompleted_normally = Falsepending_messages: list[dict] = []+ # Proxy-injected memory tools are invisible to the client, so their+ # tool_use blocks must never reach it (GH #2195).+ memory_filter = (+ MemoryToolStreamFilter(server_memory_tool_names)+ if memory_enabled and server_memory_tool_names+ else None+ )try:async with contextlib.aclosing(upstream_response) as response:@@ -1307,7 +1528,11 @@ class StreamingMixin:# Always stream immediately — buffering breaks# real-time clients (LangGraph, LangChain, etc.)- yield chunk+ if memory_filter is None:+ yield chunk+ else:+ for frame in memory_filter.feed(chunk):+ yield frameif _codex_wire_debug:capture_codex_wire_debug(@@ -1384,22 +1609,42 @@ class StreamingMixin:"do not support custom tool injection. Set ANTHROPIC_API_KEY ""environment variable or use --no-memory-tools flag.")+ if memory_filter is not None:+ for frame in memory_filter.closing_frames():+ yield framereturn# Parse SSE to get response JSONparsed_response = self._parse_sse_to_response(full_sse_data, provider)- if parsed_response and self.memory_handler.has_memory_tool_calls(+ if memory_filter is not None:+ async for frame in self._continue_memory_tool_stream(+ memory_filter,+ parsed_response,+ url=url,+ outbound_headers=outbound_headers,+ outbound_bytes=outbound_bytes,+ memory_user_id=memory_user_id,+ memory_request_ctx=memory_request_ctx,+ server_memory_tool_names=server_memory_tool_names or frozenset(),+ stream_state=stream_state,+ request_id=request_id,+ ):+ yield frame+ memory_filter = None+ parsed_response = (+ stream_state.pop("memory_client_message", None) or parsed_response+ )+ elif parsed_response and self.memory_handler.has_memory_tool_calls(parsed_response, provider):logger.info(f"[{request_id}] Memory: Detected tool calls in streaming response")- # Execute memory tool calls — response already streamed- # so results are saved but continuation is not possible- # in SSE streaming mode. The WS and non-streaming paths- # handle continuation properly.+ # The client declared these memory tools itself, so the+ # tool_use already reached it and the client continues+ # the turn; only the server-side save happens here.tool_results = await self.memory_handler.handle_memory_tool_calls(parsed_response,memory_user_id,@@ -1413,6 +1658,23 @@ class StreamingMixin:"continuation handled by client)")+ if memory_filter is not None:+ # Memory detection was skipped (empty stream or buffer cap):+ # still run the withheld calls, then end the message.+ async for frame in self._continue_memory_tool_stream(+ memory_filter,+ None,+ url=url,+ outbound_headers=outbound_headers,+ outbound_bytes=outbound_bytes,+ memory_user_id=memory_user_id,+ memory_request_ctx=memory_request_ctx,+ server_memory_tool_names=server_memory_tool_names or frozenset(),+ stream_state=stream_state,+ request_id=request_id,+ ):+ yield frame+# CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.# In streaming mode, the client handles actual retrieval, but we# still need to record the event so TOIN learns which fields matter.@@ -1459,6 +1721,14 @@ class StreamingMixin:logger.error(f"[{request_id}] HTTP error from upstream API: {e}")# Forward the upstream error responseyield e.response.content+ except MemoryToolStreamOverflowError as e:+ # The filter dropped what it withheld; end the stream rather+ # than forward it.+ logger.error(f"[{request_id}] Memory: {e}; ending the stream")+ error_event = public_errors.anthropic_error_body(+ public_errors.INTERNAL_ERROR, request_id=request_id+ )+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()except Exception as e:logger.error(f"[{request_id}] Unexpected streaming error: {e}")error_event = public_errors.anthropic_error_body(diff --git a/headroom/proxy/memory_tool_stream.py b/headroom/proxy/memory_tool_stream.pynew file mode 100644index 000000000..850cfabff--- /dev/null+++ b/headroom/proxy/memory_tool_stream.py@@ -0,0 +1,316 @@+"""Hide proxy-handled memory tool calls from an Anthropic SSE stream.++The proxy injects its own memory tools (``memory_save``, ``memory_search``,+...) into requests. The client never declared them, so when the model calls+one the client cannot run it: Claude Code answers ``No such tool available``+and the model concludes the save or search failed (GH #2195).++``MemoryToolStreamFilter`` sits between the upstream SSE stream and the+client. It forwards every event live except:++* the ``content_block_*`` events of a ``tool_use`` block whose name is a+ proxy-owned memory tool, which it withholds;+* ``message_delta`` / ``message_stop``, which it holds until the caller+ knows whether the turn ends here or continues with a server-side+ continuation round.++Content block indices are rewritten so the blocks the client sees stay+contiguous, including across continuation rounds whose blocks are appended+to the same client message. Frames that need no rewrite are forwarded as the+exact upstream bytes.++Everything the filter holds back (a pending partial frame, withheld tool+blocks and their input, held tail frames) is charged its size on the wire+against ``max_retained_bytes``; past it the filter drops what it holds and+raises ``MemoryToolStreamOverflowError`` rather than forwarding withheld+bytes.+"""++from __future__ import annotations++import json+import logging+from collections.abc import Collection+from typing import Any++from headroom.proxy.sse_byte_buffer_policy import (+ SSE_DATA_LINE_PREFIX,+ SSE_EVENT_LINE_PREFIX,+ find_sse_event_terminator,+)++logger = logging.getLogger(__name__)++_BLOCK_EVENTS = ("content_block_start", "content_block_delta", "content_block_stop")+_TAIL_EVENTS = ("message_delta", "message_stop")++# Same budget as the proxy's SSE reconstruction buffer. Legitimate streams+# retain a few KiB at most: one partial frame plus the memory calls' input.+DEFAULT_MAX_RETAINED_BYTES = 10 * 1024 * 1024+++class MemoryToolStreamOverflowError(Exception):+ """The filter would have to retain more than ``max_retained_bytes``."""+++def _parse_frame(frame: bytes) -> tuple[str | None, dict[str, Any] | None]:+ """Return ``(event_name, payload)`` for one complete SSE frame."""+ event_name: str | None = None+ data_lines: list[str] = []+ for line in frame.decode("utf-8").splitlines():+ if line.startswith(SSE_EVENT_LINE_PREFIX):+ event_name = line[len(SSE_EVENT_LINE_PREFIX) :].strip()+ elif line.startswith(SSE_DATA_LINE_PREFIX):+ data_lines.append(line[len(SSE_DATA_LINE_PREFIX) :].lstrip())+ if not data_lines:+ return event_name, None+ try:+ payload = json.loads("\n".join(data_lines))+ except json.JSONDecodeError:+ return event_name, None+ if not isinstance(payload, dict):+ return event_name, None+ return event_name or payload.get("type"), payload+++def _render_frame(event_name: str, payload: dict[str, Any]) -> bytes:+ return f"event: {event_name}\ndata: {json.dumps(payload)}\n\n".encode()+++class MemoryToolStreamFilter:+ """Per-round SSE filter; see the module docstring.++ Args:+ tool_names: Memory tool names the proxy injected and will execute+ itself. Tools the client declared are never withheld.+ index_offset: Client-side index of this round's first visible block.+ Zero for the first round; ``next_index`` of the previous round+ for a continuation round.+ forward_message_start: False for continuation rounds, whose blocks+ extend the message the client already opened.+ max_retained_bytes: Limit on the bytes held back at any time;+ ``DEFAULT_MAX_RETAINED_BYTES`` when None.+ """++ def __init__(+ self,+ tool_names: Collection[str],+ *,+ index_offset: int = 0,+ forward_message_start: bool = True,+ max_retained_bytes: int | None = None,+ ) -> None:+ self._tool_names = frozenset(tool_names)+ self._max_retained_bytes = (+ DEFAULT_MAX_RETAINED_BYTES if max_retained_bytes is None else max_retained_bytes+ )+ # Wire bytes of the frames behind _hidden (per upstream index) and+ # _tail; _buffer is counted live.+ self._hidden_bytes: dict[int, int] = {}+ self._tail_bytes = 0+ self._forward_message_start = forward_message_start+ self._buffer = bytearray()+ self._index_map: dict[int, int] = {}+ # Withheld tool_use blocks by upstream index, with their streamed+ # input JSON, so the calls can run even when the whole response+ # cannot be reconstructed (buffer cap, unparseable stream).+ self._hidden: dict[int, tuple[dict[str, Any], list[str]]] = {}+ # Upstream indices of withheld blocks whose content_block_stop arrived.+ self._hidden_stopped: set[int] = set()+ self.next_index = index_offset+ self.visible_tool_use = False+ self.stop_reason: str | None = None+ self._tail: list[tuple[str, dict[str, Any] | None, bytes]] = []++ @property+ def hid_tool_calls(self) -> bool:+ return bool(self._hidden)++ @property+ def hidden_tool_names(self) -> list[str]:+ return [block["name"] for block, _ in self._hidden.values()]++ def hidden_calls(self) -> list[dict[str, Any]]:+ """The withheld ``tool_use`` blocks that finished with a complete input.++ A call cut off mid-input (e.g. ``max_tokens``) or whose block never+ reached ``content_block_stop`` (the stream ended early) is left out+ rather than run: the model never finished requesting it, and some+ memory tools mutate state.+ """+ calls: list[dict[str, Any]] = []+ for upstream_index, (block, parts) in self._hidden.items():+ if upstream_index not in self._hidden_stopped:+ logger.warning(+ "Memory: skipping %s call %s whose block never finished",+ block.get("name"),+ block.get("id"),+ )+ continue+ tool_input: Any = block.get("input") or {}+ if parts:+ try:+ tool_input = json.loads("".join(parts))+ except json.JSONDecodeError:+ logger.warning(+ "Memory: skipping %s call %s with incomplete input",+ block.get("name"),+ block.get("id"),+ )+ continue+ if not isinstance(tool_input, dict):+ continue+ calls.append({**block, "input": tool_input})+ return calls++ def feed(self, chunk: bytes) -> list[bytes]:+ """Consume upstream bytes; return the frames to forward now.++ Raises:+ MemoryToolStreamOverflowError: The bytes held back exceed+ ``max_retained_bytes``. The filter has dropped them; the+ caller must end the stream instead of forwarding anything+ further.+ """+ self._buffer.extend(chunk)+ out: list[bytes] = []+ while (match := find_sse_event_terminator(self._buffer)) is not None:+ idx, terminator_len = match+ frame = bytes(self._buffer[: idx + terminator_len])+ del self._buffer[: idx + terminator_len]+ forwarded = self._route(frame)+ if forwarded is not None:+ out.append(forwarded)+ retained = len(self._buffer) + self._tail_bytes + sum(self._hidden_bytes.values())+ if retained > self._max_retained_bytes:+ self._buffer.clear()+ self._hidden.clear()+ self._hidden_bytes.clear()+ self._hidden_stopped.clear()+ self._tail.clear()+ self._tail_bytes = 0+ raise MemoryToolStreamOverflowError(+ f"memory tool stream filter would retain {retained} bytes "+ f"(limit {self._max_retained_bytes})"+ )+ return out++ def _route(self, frame: bytes) -> bytes | None:+ event_name, payload = _parse_frame(frame)+ if payload is None or event_name is None:+ return frame++ if event_name == "message_start":+ return frame if self._forward_message_start else None++ if event_name in _TAIL_EVENTS:+ if event_name == "message_delta":+ delta = payload.get("delta")+ if isinstance(delta, dict) and delta.get("stop_reason"):+ self.stop_reason = delta["stop_reason"]+ self._tail.append((event_name, payload, frame))+ self._tail_bytes += len(frame)+ return None++ if event_name not in _BLOCK_EVENTS:+ return frame++ upstream_index = payload.get("index")+ if not isinstance(upstream_index, int):+ return frame++ if event_name == "content_block_start":+ block = payload.get("content_block")+ if (+ isinstance(block, dict)+ and block.get("type") == "tool_use"+ and block.get("name") in self._tool_names+ ):+ # A repeated start for the same index replaces the block,+ # its charge and any earlier stop. The frame may carry the+ # whole input inline, so it is charged in full.+ self._hidden[upstream_index] = (block, [])+ self._hidden_bytes[upstream_index] = len(frame)+ self._hidden_stopped.discard(upstream_index)+ return None+ if isinstance(block, dict) and block.get("type") == "tool_use":+ self.visible_tool_use = True+ self._index_map[upstream_index] = self.next_index+ self.next_index += 1++ if upstream_index in self._hidden:+ delta = payload.get("delta")+ if (+ event_name == "content_block_delta"+ and isinstance(delta, dict)+ and delta.get("type") == "input_json_delta"+ ):+ self._hidden[upstream_index][1].append(str(delta.get("partial_json", "")))+ # The frame's size bounds the fragment's, and needs no+ # re-encoding (a lone surrogate cannot be encoded).+ self._hidden_bytes[upstream_index] += len(frame)+ elif event_name == "content_block_stop":+ self._hidden_stopped.add(upstream_index)+ return None+ client_index = self._index_map.get(upstream_index, upstream_index)+ if client_index == upstream_index:+ return frame+ return _render_frame(event_name, {**payload, "index": client_index})++ def closing_frames(self, *, prior_usage: dict[str, int] | None = None) -> list[bytes]:+ """End the client message after this round without a continuation."""+ ends_on_hidden_call = (+ self.hid_tool_calls and not self.visible_tool_use and self.stop_reason == "tool_use"+ )+ return self.tail_frames(+ stop_reason="end_turn" if ends_on_hidden_call else None,+ prior_usage=prior_usage,+ )++ def tail_frames(+ self,+ *,+ stop_reason: str | None = None,+ prior_usage: dict[str, int] | None = None,+ ) -> list[bytes]:+ """Return the held ``message_delta`` / ``message_stop`` frames.++ ``stop_reason`` replaces the upstream stop reason, e.g. ``end_turn``+ when the only tool calls were withheld memory calls: a ``tool_use``+ stop with no visible ``tool_use`` block leaves the client waiting on+ a tool it was never shown.++ ``prior_usage`` holds earlier continuation rounds' usage. Their+ ``message_delta`` frames never reach the client, so it is added to+ this one: ``output_tokens`` always, other counters where the delta+ reports them.+ """+ frames: list[bytes] = []+ for event_name, payload, raw in self._tail:+ if (+ event_name != "message_delta"+ or payload is None+ or not isinstance(payload.get("delta"), dict)+ or (stop_reason is None and not prior_usage)+ ):+ frames.append(raw)+ continue+ rendered = dict(payload)+ if stop_reason is not None:+ rendered["delta"] = {**payload["delta"], "stop_reason": stop_reason}+ if prior_usage:+ usage = dict(payload.get("usage") or {})+ for key, value in prior_usage.items():+ if key == "output_tokens" or isinstance(usage.get(key), int):+ usage[key] = int(usage.get(key) or 0) + value+ rendered["usage"] = usage+ frames.append(_render_frame(event_name, rendered))+ if self._buffer:+ # The stream ended mid-frame. It may belong to a withheld block,+ # so drop it rather than risk exposing it to the client.+ logger.warning(+ "Memory: dropping %d bytes of an unterminated SSE frame at end of stream",+ len(self._buffer),+ )+ self._buffer.clear()+ return framesdiff --git a/tests/test_memory_tool_stream.py b/tests/test_memory_tool_stream.pynew file mode 100644index 000000000..d2e6e45a4--- /dev/null+++ b/tests/test_memory_tool_stream.py@@ -0,0 +1,877 @@+"""Proxy-injected memory tool calls never reach a streaming client (GH #2195).++Claude Code streams every request. Before this fix the SSE path forwarded a+``memory_save`` / ``memory_search`` tool_use to the client, which never+declared those tools and answered ``No such tool available``, so saves looked+failed and searches never returned.+"""++from __future__ import annotations++import json+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import httpx+import pytest++from headroom.proxy.memory_tool_stream import (+ MemoryToolStreamFilter,+ MemoryToolStreamOverflowError,+)+from headroom.proxy.server import HeadroomProxy++MEMORY_TOOLS = frozenset({"memory_save", "memory_search"})+++def _frame(payload: dict[str, Any]) -> bytes:+ return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()+++def _sse(blocks: list[dict[str, Any]], stop_reason: str, *, input_tokens: int = 10) -> bytes:+ """An Anthropic SSE response carrying ``blocks``."""+ frames = [+ _frame(+ {+ "type": "message_start",+ "message": {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-test",+ "content": [],+ "stop_reason": None,+ "usage": {"input_tokens": input_tokens, "output_tokens": 1},+ },+ }+ )+ ]+ for index, block in enumerate(blocks):+ if block["type"] == "text":+ frames.append(+ _frame(+ {+ "type": "content_block_start",+ "index": index,+ "content_block": {"type": "text", "text": ""},+ }+ )+ )+ delta = {"type": "text_delta", "text": block["text"]}+ else:+ frames.append(+ _frame(+ {+ "type": "content_block_start",+ "index": index,+ "content_block": {**block, "input": {}},+ }+ )+ )+ delta = {"type": "input_json_delta", "partial_json": json.dumps(block["input"])}+ frames.append(_frame({"type": "content_block_delta", "index": index, "delta": delta}))+ frames.append(_frame({"type": "content_block_stop", "index": index}))+ frames.append(+ _frame(+ {+ "type": "message_delta",+ "delta": {"stop_reason": stop_reason, "stop_sequence": None},+ "usage": {"output_tokens": 5},+ }+ )+ )+ frames.append(_frame({"type": "message_stop"}))+ return b"".join(frames)+++def _events(raw: bytes) -> list[dict[str, Any]]:+ return [+ json.loads(line[len("data: ") :])+ for line in raw.decode().splitlines()+ if line.startswith("data: ")+ ]+++TEXT = {"type": "text", "text": "Saving that."}+SAVE = {+ "type": "tool_use",+ "id": "toolu_mem",+ "name": "memory_save",+ "input": {"content": "deploy region is ap-southeast-1"},+}+BASH = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}+++class TestMemoryToolStreamFilter:+ def test_stream_without_memory_calls_is_byte_identical(self) -> None:+ raw = _sse([TEXT, BASH], "tool_use")+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ out = b"".join(flt.feed(raw)) + b"".join(flt.closing_frames())+ assert out == raw+ assert not flt.hid_tool_calls++ def test_byte_at_a_time_feed_matches_one_shot(self) -> None:+ raw = _sse([TEXT, SAVE], "tool_use")+ one_shot = MemoryToolStreamFilter(MEMORY_TOOLS)+ expected = b"".join(one_shot.feed(raw)) + b"".join(one_shot.closing_frames())+ trickle = MemoryToolStreamFilter(MEMORY_TOOLS)+ got = b"".join(b"".join(trickle.feed(raw[i : i + 1])) for i in range(len(raw)))+ got += b"".join(trickle.closing_frames())+ assert got == expected++ def test_memory_tool_use_is_withheld_and_turn_ends(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ forwarded = _events(b"".join(flt.feed(_sse([TEXT, SAVE], "tool_use"))))+ assert not any(e.get("content_block", {}).get("type") == "tool_use" for e in forwarded)+ assert not any(e["type"] in ("message_delta", "message_stop") for e in forwarded)+ assert flt.hidden_tool_names == ["memory_save"]+ assert flt.stop_reason == "tool_use"++ tail = _events(b"".join(flt.closing_frames()))+ assert [e["type"] for e in tail] == ["message_delta", "message_stop"]+ assert tail[0]["delta"]["stop_reason"] == "end_turn"++ def test_blocks_after_a_hidden_call_are_reindexed(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ forwarded = _events(b"".join(flt.feed(_sse([SAVE, TEXT], "end_turn"))))+ assert {e["index"] for e in forwarded if "index" in e} == {0}+ assert flt.next_index == 1++ def test_continuation_round_extends_the_open_message(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, index_offset=1, forward_message_start=False)+ forwarded = _events(b"".join(flt.feed(_sse([TEXT], "end_turn"))))+ assert forwarded[0]["type"] == "content_block_start"+ assert {e["index"] for e in forwarded if "index" in e} == {1}++ def test_client_tool_alongside_memory_call_keeps_tool_use_stop(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ forwarded = _events(b"".join(flt.feed(_sse([SAVE, BASH], "tool_use"))))+ names = [e["content_block"].get("name") for e in forwarded if "content_block" in e]+ assert names == ["Bash"]+ assert flt.visible_tool_use+ tail = _events(b"".join(flt.closing_frames()))+ assert tail[0]["delta"]["stop_reason"] == "tool_use"++ def test_client_declared_memory_tool_is_not_withheld(self) -> None:+ flt = MemoryToolStreamFilter(frozenset({"memory_search"}))+ forwarded = _events(b"".join(flt.feed(_sse([SAVE], "tool_use"))))+ assert any(e.get("content_block", {}).get("name") == "memory_save" for e in forwarded)+ assert not flt.hid_tool_calls+++def _proxy(upstream_bodies: list[bytes], tool_results: list[dict[str, Any]]) -> HeadroomProxy:+ proxy = object.__new__(HeadroomProxy)+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)+ proxy.metrics = MagicMock()+ proxy.metrics.record_request = AsyncMock(return_value=None)+ proxy.metrics.record_failed = AsyncMock(return_value=None)+ proxy.cost_tracker = MagicMock()+ proxy.cost_tracker.estimate_cost.return_value = 0.0+ proxy.stats = {+ "requests_total": 0,+ "requests_optimized": 0,+ "tokens": {"original": 0, "optimized": 0, "saved": 0},+ "cost": {"total_usd": 0, "savings_usd": 0},+ "errors": 0,+ "active_requests": 0,+ "requests_per_model": {},+ }+ proxy._config = MagicMock()+ proxy._config.ccr_inject_tool = False+ proxy._config.retry_max_attempts = 1+ proxy.config = proxy._config+ proxy.memory_handler = MagicMock()+ proxy.memory_handler.handle_memory_tool_calls = AsyncMock(return_value=tool_results)++ responses = []+ for raw in upstream_bodies:+ response = MagicMock()+ response.headers = httpx.Headers({"content-type": "text/event-stream"})+ response.status_code = 200++ async def aiter_bytes(raw: bytes = raw):+ # Split mid-frame so the filter has to reassemble events.+ yield raw[:37]+ yield raw[37:]++ response.aiter_bytes = aiter_bytes+ response.aclose = AsyncMock()+ responses.append(response)+ proxy.http_client.build_request = MagicMock(return_value=MagicMock())+ proxy.http_client.send = AsyncMock(side_effect=responses)+ return proxy+++async def _client_view(proxy: HeadroomProxy, **kwargs: Any) -> list[dict[str, Any]]:+ result = await proxy._stream_response(+ url="https://api.anthropic.com/v1/messages",+ headers={"x-api-key": "sk-test"},+ body={+ "model": "claude-test",+ "max_tokens": 100,+ "stream": True,+ "messages": [{"role": "user", "content": "remember the deploy region"}],+ },+ provider="anthropic",+ model="claude-test",+ request_id="test-mem",+ original_tokens=10,+ optimized_tokens=10,+ tokens_saved=0,+ transforms_applied=[],+ tags={},+ optimization_latency=0.0,+ memory_user_id="user-1",+ **kwargs,+ )+ raw = b"".join([chunk async for chunk in result.body_iterator])+ return _events(raw)+++SAVE_RESULT = [{"type": "tool_result", "tool_use_id": "toolu_mem", "content": '{"status":"saved"}'}]+++class TestStreamingMemoryContinuation:+ @pytest.mark.asyncio+ async def test_memory_call_runs_server_side_and_answer_streams_on(self) -> None:+ proxy = _proxy(+ [+ _sse([TEXT, SAVE], "tool_use"),+ _sse([{"type": "text", "text": "Saved."}], "end_turn"),+ ],+ SAVE_RESULT,+ )+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)++ assert not any(e.get("content_block", {}).get("type") == "tool_use" for e in events)+ assert [e["type"] for e in events].count("message_start") == 1+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]+ assert texts == ["Saving that.", "Saved."]+ starts = [e["index"] for e in events if e["type"] == "content_block_start"]+ assert starts == [0, 1]+ deltas = [e for e in events if e["type"] == "message_delta"]+ assert len(deltas) == 1 and deltas[0]["delta"]["stop_reason"] == "end_turn"+ assert events[-1]["type"] == "message_stop"++ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()+ continuation = json.loads(+ proxy.http_client.build_request.call_args_list[1].kwargs["content"]+ )+ assert [m["role"] for m in continuation["messages"]] == ["user", "assistant", "user"]+ assert continuation["messages"][1]["content"][1]["name"] == "memory_save"+ assert continuation["messages"][2]["content"] == SAVE_RESULT++ @pytest.mark.asyncio+ async def test_finalizer_sees_the_message_the_client_received(self) -> None:+ proxy = _proxy(+ [+ _sse([TEXT, SAVE], "tool_use"),+ _sse([{"type": "text", "text": "Saved."}], "end_turn"),+ ],+ SAVE_RESULT,+ )+ proxy._finalize_stream_response = AsyncMock(return_value=None)+ await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)++ finalized = proxy._finalize_stream_response.await_args.kwargs["parsed_response"]+ assert [b.get("text") for b in finalized["content"]] == ["Saving that.", "Saved."]++ @pytest.mark.asyncio+ async def test_client_tool_in_same_round_returns_turn_to_client(self) -> None:+ proxy = _proxy([_sse([SAVE, BASH], "tool_use")], SAVE_RESULT)+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)++ names = [e["content_block"].get("name") for e in events if "content_block" in e]+ assert names == ["Bash"]+ assert [e["delta"]["stop_reason"] for e in events if e["type"] == "message_delta"] == [+ "tool_use"+ ]+ assert proxy.http_client.send.await_count == 1+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_without_injected_tools_stream_passes_through(self) -> None:+ raw = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([raw], SAVE_RESULT)+ proxy.memory_handler.has_memory_tool_calls = MagicMock(return_value=False)+ events = await _client_view(proxy)+ assert events == _events(raw)+++class TestRecordedMemoryCalls:+ def test_hidden_call_input_is_rebuilt_from_stream(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_sse([TEXT, SAVE], "tool_use"))+ assert flt.hidden_calls() == [SAVE]++ def test_call_with_truncated_input_is_not_run(self) -> None:+ raw = _sse([SAVE], "max_tokens").replace(b'\\"deploy region', b'\\"deploy', 1)+ raw = raw.replace(b'ap-southeast-1\\"}', b"ap-southeast-1", 1)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(raw)+ assert flt.hid_tool_calls+ assert flt.hidden_calls() == []++ def test_call_whose_block_never_stopped_is_not_run(self) -> None:+ # Complete, valid input, but the stream ends before content_block_stop.+ raw = _sse([SAVE], "tool_use")+ raw = raw[: raw.index(b"event: content_block_stop")]+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(raw)+ assert flt.hid_tool_calls+ assert flt.hidden_calls() == []++ def test_prior_rounds_usage_is_added_to_the_final_delta(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_sse([TEXT], "end_turn"))+ tail = _events(b"".join(flt.closing_frames(prior_usage={"output_tokens": 7})))+ assert tail[0]["usage"]["output_tokens"] == 12+++async def _drain(gen: Any) -> list[dict[str, Any]]:+ return _events(b"".join([frame async for frame in gen]))+++def _continue(proxy: HeadroomProxy, flt: MemoryToolStreamFilter, response: Any) -> Any:+ body = {"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}+ return proxy._continue_memory_tool_stream(+ flt,+ response,+ url="https://api.anthropic.com/v1/messages",+ outbound_headers={"x-api-key": "sk-test", "content-length": "1"},+ outbound_bytes=json.dumps(body).encode(),+ memory_user_id="user-1",+ memory_request_ctx=None,+ server_memory_tool_names=MEMORY_TOOLS,+ stream_state={},+ request_id="test-mem",+ )+++class TestContinuationEdgeCases:+ @pytest.mark.asyncio+ async def test_unrebuilt_round_still_runs_its_memory_calls(self) -> None:+ proxy = _proxy([], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_sse([TEXT, SAVE], "tool_use"))++ events = await _drain(_continue(proxy, flt, None))++ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()+ ran = proxy.memory_handler.handle_memory_tool_calls.await_args.args[0]["content"]+ assert ran == [SAVE]+ assert proxy.http_client.send.await_count == 0+ assert events[0]["delta"]["stop_reason"] == "end_turn"++ @pytest.mark.asyncio+ async def test_continuation_keeps_streaming_past_the_buffer_cap(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import headroom.proxy.helpers as helpers++ round_one = _sse([TEXT, SAVE], "tool_use")+ reply = _sse([{"type": "text", "text": "Saved."}], "end_turn")+ proxy = _proxy([reply], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(round_one)+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")+ # _proxy splits each body in two; the second chunk arrives past the cap.+ monkeypatch.setattr(helpers, "MAX_SSE_BUFFER_SIZE", 16)++ events = await _drain(_continue(proxy, flt, response))++ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]+ assert texts == ["Saved."]+ assert [e["type"] for e in events][-2:] == ["message_delta", "message_stop"]++ @pytest.mark.asyncio+ async def test_interrupted_call_is_not_run(self) -> None:+ raw = _sse([TEXT, SAVE], "tool_use")+ raw = raw[: raw.rindex(b"event: content_block_stop")]+ proxy = _proxy([], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(raw)++ await _drain(_continue(proxy, flt, None))++ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()+ assert proxy.http_client.send.await_count == 0++ @pytest.mark.asyncio+ async def test_continuation_over_the_buffer_cap_is_not_continued_again(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import headroom.proxy.helpers as helpers++ round_one = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([_sse([SAVE], "tool_use")], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(round_one)+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")+ monkeypatch.setattr(helpers, "MAX_SSE_BUFFER_SIZE", 64)++ events = await _drain(_continue(proxy, flt, response))++ assert proxy.http_client.send.await_count == 1+ assert proxy.memory_handler.handle_memory_tool_calls.await_count == 2+ assert [e["type"] for e in events][-2:] == ["message_delta", "message_stop"]+ assert events[-2]["delta"]["stop_reason"] == "end_turn"++ @pytest.mark.asyncio+ async def test_client_sees_output_usage_of_every_round(self) -> None:+ proxy = _proxy(+ [+ _sse([TEXT, SAVE], "tool_use"),+ _sse([{"type": "text", "text": "Saved."}], "end_turn"),+ ],+ SAVE_RESULT,+ )+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)+ deltas = [e for e in events if e["type"] == "message_delta"]+ assert deltas[0]["usage"]["output_tokens"] == 10+++class TestFrameParsingEdgeCases:+ """Frames the filter cannot interpret pass through to the client untouched."""++ @pytest.mark.parametrize(+ "frame",+ [+ b": keep-alive\n\n",+ b"event: ping\n\n",+ b"event: content_block_delta\ndata: {not json\n\n",+ b"event: content_block_delta\ndata: [1, 2]\n\n",+ b'event: ping\ndata: {"type": "ping"}\n\n',+ b'event: content_block_delta\ndata: {"type": "content_block_delta", "index": "0"}\n\n',+ ],+ )+ def test_uninterpretable_frame_is_forwarded_verbatim(self, frame: bytes) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ assert flt.feed(frame) == [frame]++ def test_hidden_call_with_non_object_input_is_not_run(self) -> None:+ raw = _sse([SAVE], "tool_use").replace(+ json.dumps(json.dumps(SAVE["input"])).encode(), json.dumps("[1]").encode(), 1+ )+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(raw)+ assert flt.hid_tool_calls+ assert flt.hidden_calls() == []++ def test_hidden_call_with_input_in_its_start_frame(self) -> None:+ raw = b"".join(+ [+ _frame({"type": "content_block_start", "index": 0, "content_block": SAVE}),+ _frame({"type": "content_block_stop", "index": 0}),+ ]+ )+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ assert flt.feed(raw) == []+ assert flt.hidden_calls() == [SAVE]++ def test_other_delta_on_hidden_block_is_withheld(self) -> None:+ raw = b"".join(+ [+ _frame({"type": "content_block_start", "index": 0, "content_block": SAVE}),+ _frame({"type": "content_block_delta", "index": 0, "delta": {"type": "x"}}),+ _frame({"type": "content_block_stop", "index": 0}),+ ]+ )+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ assert flt.feed(raw) == []+ assert flt.hidden_calls() == [SAVE]++ def test_message_delta_without_stop_reason_keeps_none(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_frame({"type": "message_delta", "delta": {}, "usage": {"output_tokens": 1}}))+ assert flt.stop_reason is None++ def test_unterminated_trailing_frame_is_dropped(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_sse([TEXT], "end_turn"))+ flt.feed(b"event: ping\ndata: {")+ assert b"ping" not in b"".join(flt.closing_frames())++ def test_truncated_hidden_delta_never_reaches_the_client(self) -> None:+ secret = "my private deploy key"+ start = _frame({"type": "content_block_start", "index": 0, "content_block": SAVE})+ delta = _frame(+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "input_json_delta", "partial_json": f'{{"content": "{secret}'},+ }+ )+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ emitted = flt.feed(start + delta[:-2]) # the stream ends before the terminator+ emitted += flt.closing_frames()+ assert secret.encode() not in b"".join(emitted)+ assert flt.hidden_calls() == []+++class TestRetainedByteLimit:+ def test_unterminated_event_stops_growing_at_the_limit(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=4096)+ chunk = b"event: content_block_delta\ndata: " + b"x" * 1024+ with pytest.raises(MemoryToolStreamOverflowError):+ for _ in range(12):+ flt.feed(chunk)+ assert flt.closing_frames() == []++ def test_oversized_hidden_input_is_dropped_not_run(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=4096)+ flt.feed(_frame({"type": "content_block_start", "index": 0, "content_block": SAVE}))+ fragment = {"type": "input_json_delta", "partial_json": "y" * 1024}+ with pytest.raises(MemoryToolStreamOverflowError):+ for _ in range(12):+ flt.feed(_frame({"type": "content_block_delta", "index": 0, "delta": fragment}))+ assert not flt.hid_tool_calls+ assert flt.hidden_calls() == []+ assert flt.closing_frames() == []++ def test_inline_hidden_input_counts_against_the_limit(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=4096)+ block = {**SAVE, "input": {"content": "z" * 20000}}+ with pytest.raises(MemoryToolStreamOverflowError):+ flt.feed(_frame({"type": "content_block_start", "index": 0, "content_block": block}))+ assert not flt.hid_tool_calls++ def test_limit_is_in_bytes_at_the_boundary(self) -> None:+ start = _frame({"type": "content_block_start", "index": 0, "content_block": SAVE})+ snowmen = {"type": "input_json_delta", "partial_json": "\u2603" * 24}+ payload = {"type": "content_block_delta", "index": 0, "delta": snowmen}+ # Raw UTF-8 on the wire: 24 characters, 72 bytes.+ delta = f"event: {payload['type']}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n"+ frames = start + delta.encode()++ MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(frames)).feed(frames)+ with pytest.raises(MemoryToolStreamOverflowError):+ MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(frames) - 1).feed(frames)++ def test_lone_surrogate_in_hidden_input_is_counted(self) -> None:+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_frame({"type": "content_block_start", "index": 0, "content_block": SAVE}))+ fragment = {"type": "input_json_delta", "partial_json": '{"content": "\ud800"}'}+ frame = _frame({"type": "content_block_delta", "index": 0, "delta": fragment})+ assert flt.feed(frame) == []+ assert flt.feed(_frame({"type": "content_block_stop", "index": 0})) == []+ assert flt.hidden_tool_names == ["memory_save"]++ def test_repeated_hidden_start_replaces_its_charge(self) -> None:+ first = {**SAVE, "input": {"content": "a" * 300}}+ second = {**SAVE, "input": {"content": "b" * 300}}+ start_one = _frame({"type": "content_block_start", "index": 0, "content_block": first})+ start_two = _frame({"type": "content_block_start", "index": 0, "content_block": second})+ stop = _frame({"type": "content_block_stop", "index": 0})+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(start_two) + 1)+ flt.feed(start_one)+ flt.feed(stop)+ flt.feed(start_two)+ assert flt.hidden_calls() == [] # the replacement has not stopped yet+ flt.feed(stop)+ assert flt.hidden_calls() == [second]++ def test_frames_within_the_limit_pass(self) -> None:+ raw = _sse([TEXT, SAVE], "tool_use")+ flt = MemoryToolStreamFilter(MEMORY_TOOLS, max_retained_bytes=len(raw))+ flt.feed(raw)+ assert flt.hidden_calls() == [SAVE]+++class TestUpstreamErrorFrame:+ def test_anthropic_error_envelope_passes_through(self) -> None:+ from headroom.proxy.handlers.streaming import _upstream_error_frame++ envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "busy"}}+ [event] = _events(_upstream_error_frame(json.dumps(envelope).encode(), "req-1"))+ assert event == envelope++ @pytest.mark.parametrize(+ "body", [b"<html>Bad Gateway</html>", b"\xff\xfe", b'{"detail": "nope"}']+ )+ def test_other_bodies_become_the_public_error(self, body: bytes) -> None:+ from headroom.proxy.handlers.streaming import _upstream_error_frame++ raw = _upstream_error_frame(body, "req-1")+ assert raw.startswith(b"event: error\n")+ [event] = _events(raw)+ assert event["type"] == "error"+ assert body.decode("utf-8", "replace") not in json.dumps(event)+++class TestContinuationFallbacks:+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "outbound", [b"not json", json.dumps({"model": "claude-test", "messages": "hi"}).encode()]+ )+ async def test_unreadable_request_body_ends_the_turn_after_running_calls(+ self, outbound: bytes+ ) -> None:+ round_one = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(round_one)+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")++ events = await _drain(+ proxy._continue_memory_tool_stream(+ flt,+ response,+ url="https://api.anthropic.com/v1/messages",+ outbound_headers={"x-api-key": "sk-test"},+ outbound_bytes=outbound,+ memory_user_id="user-1",+ memory_request_ctx=None,+ server_memory_tool_names=MEMORY_TOOLS,+ stream_state={},+ request_id="test-mem",+ )+ )++ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()+ assert proxy.http_client.send.await_count == 0+ assert events[0]["delta"]["stop_reason"] == "end_turn"++ @pytest.mark.asyncio+ async def test_round_limit_stops_continuing(self, monkeypatch: pytest.MonkeyPatch) -> None:+ round_one = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([], SAVE_RESULT)+ monkeypatch.setattr(type(proxy), "_MEMORY_CONTINUATION_MAX_ROUNDS", 0)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(round_one)+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")++ events = await _drain(_continue(proxy, flt, response))++ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()+ assert proxy.http_client.send.await_count == 0+ assert events[0]["delta"]["stop_reason"] == "end_turn"++ @pytest.mark.asyncio+ async def test_failed_continuation_round_sends_an_error_event(self) -> None:+ round_one = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([], SAVE_RESULT)+ failed = MagicMock()+ failed.status_code = 529+ envelope = {"type": "error", "error": {"type": "overloaded_error", "message": "busy"}}+ failed.aread = AsyncMock(return_value=json.dumps(envelope).encode())+ failed.aclose = AsyncMock()+ proxy.http_client.send = AsyncMock(return_value=failed)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(round_one)+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")++ events = await _drain(_continue(proxy, flt, response))++ assert proxy.http_client.send.await_count == 1+ assert events == [envelope]+ failed.aclose.assert_awaited()++ @pytest.mark.asyncio+ async def test_calls_are_not_run_without_a_memory_user(self) -> None:+ proxy = _proxy([], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(_sse([TEXT, SAVE], "tool_use"))++ events = await _drain(+ proxy._continue_memory_tool_stream(+ flt,+ None,+ url="https://api.anthropic.com/v1/messages",+ outbound_headers={"x-api-key": "sk-test"},+ outbound_bytes=b"{}",+ memory_user_id=None,+ memory_request_ctx=None,+ server_memory_tool_names=MEMORY_TOOLS,+ stream_state={},+ request_id="test-mem",+ )+ )++ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()+ assert events[0]["delta"]["stop_reason"] == "end_turn"++ @pytest.mark.asyncio+ async def test_round_without_usage_still_continues(self) -> None:+ round_one = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([_sse([{"type": "text", "text": "Saved."}], "end_turn")], SAVE_RESULT)+ flt = MemoryToolStreamFilter(MEMORY_TOOLS)+ flt.feed(round_one)+ response = proxy._parse_sse_to_response(round_one.decode(), "anthropic")+ response.pop("usage", None)++ events = await _drain(_continue(proxy, flt, response))++ assert proxy.http_client.send.await_count == 1+ texts = [e["delta"]["text"] for e in events if e["type"] == "content_block_delta"]+ assert texts == ["Saved."]+ assert events[-2]["delta"]["stop_reason"] == "end_turn"+++class TestStreamingMemoryFallbacks:+ @pytest.mark.asyncio+ async def test_client_handled_memory_calls_still_run_without_server_tools(self) -> None:+ raw = _sse([TEXT, SAVE], "tool_use")+ proxy = _proxy([raw], SAVE_RESULT)+ proxy.memory_handler.has_memory_tool_calls = MagicMock(return_value=True)+ events = await _client_view(proxy)++ assert events == _events(raw)+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()++ @pytest.mark.asyncio+ @pytest.mark.parametrize("server_tools", [MEMORY_TOOLS, frozenset()])+ async def test_subscription_credential_error_ends_the_message(+ self, server_tools: frozenset[str]+ ) -> None:+ refusal = {+ "type": "text",+ "text": "This credential is only authorized for use with Claude Code.",+ }+ proxy = _proxy([_sse([refusal], "end_turn")], SAVE_RESULT)+ events = await _client_view(proxy, server_memory_tool_names=server_tools)++ assert proxy.http_client.send.await_count == 1+ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()+ assert [e["type"] for e in events][-2:] == ["message_delta", "message_stop"]++ @pytest.mark.asyncio+ async def test_buffer_cap_still_runs_withheld_calls(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import headroom.proxy.helpers as helpers++ proxy = _proxy([_sse([TEXT, SAVE], "tool_use")], SAVE_RESULT)+ monkeypatch.setattr(helpers, "MAX_SSE_BUFFER_SIZE", 64)+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)++ assert not any(e.get("content_block", {}).get("type") == "tool_use" for e in events)+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()+ assert proxy.http_client.send.await_count == 1+ deltas = [e for e in events if e["type"] == "message_delta"]+ assert deltas[-1]["delta"]["stop_reason"] == "end_turn"+++class TestServerMemoryToolNames:+ def test_injected_memory_tools_are_server_side(self) -> None:+ from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin++ tools = [+ {"name": "Bash"},+ {"name": "memory_save"},+ {"name": "memory_search"},+ {"type": "memory_20250818", "name": "memory"},+ {"type": "web_search"},+ "not-a-tool",+ ]+ names = AnthropicHandlerMixin._server_memory_tool_names(tools, [{"name": "Bash"}])+ assert names == frozenset({"memory_save", "memory_search", "memory"})++ def test_client_declared_memory_tool_stays_with_the_client(self) -> None:+ from headroom.proxy.handlers.anthropic import AnthropicHandlerMixin++ tools = [{"name": "memory_save"}, {"name": "memory_search"}]+ names = AnthropicHandlerMixin._server_memory_tool_names(+ tools, [{"name": "memory_save"}, "junk"]+ )+ assert names == frozenset({"memory_search"})+ assert AnthropicHandlerMixin._server_memory_tool_names(None, None) == frozenset()+++class TestHandlerPassesServerMemoryTools:+ def test_injected_memory_tools_reach_the_stream(self) -> None:+ from types import SimpleNamespace++ from fastapi.responses import StreamingResponse+ from fastapi.testclient import TestClient++ from headroom.proxy.server import ProxyConfig, create_app++ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+ captured: dict[str, Any] = {}++ async def fake_stream_response(*args: Any, **kwargs: Any) -> StreamingResponse:+ captured.update(kwargs)++ async def gen() -> Any:+ yield b""++ return StreamingResponse(gen(), media_type="text/event-stream")++ memory_tools = [+ {"name": name, "description": name, "input_schema": {"type": "object"}}+ for name in sorted(MEMORY_TOOLS)+ ]+ with TestClient(create_app(config)) as client:+ proxy = client.app.state.proxy+ proxy.memory_handler = SimpleNamespace(+ config=SimpleNamespace(inject_context=False, inject_tools=True),+ compute_memory_tool_definitions=lambda provider: memory_tools,+ get_beta_headers=lambda: {},+ has_memory_tool_calls=lambda resp, provider: False,+ )+ proxy._stream_response = fake_stream_response+ client.post(+ "/v1/messages",+ headers={+ "x-api-key": "test-key",+ "anthropic-version": "2023-06-01",+ "x-headroom-user-id": "u1",+ },+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "stream": True,+ "tools": [{"name": "Bash", "input_schema": {"type": "object"}}],+ "messages": [{"role": "user", "content": "remember the deploy region"}],+ },+ )++ assert captured["server_memory_tool_names"] == MEMORY_TOOLS+++class TestStreamingRetainedByteLimit:+ SECRET = "s3cr3t-" * 256++ def _oversized_save(self) -> bytes:+ save = {**SAVE, "input": {"content": self.SECRET}}+ return _sse([TEXT, save], "tool_use")++ @pytest.mark.asyncio+ async def test_overflow_ends_the_stream_with_an_error(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import headroom.proxy.memory_tool_stream as mts++ monkeypatch.setattr(mts, "DEFAULT_MAX_RETAINED_BYTES", 256)+ proxy = _proxy([self._oversized_save()], SAVE_RESULT)+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)++ assert events[-1]["type"] == "error"+ assert self.SECRET not in json.dumps(events)+ proxy.memory_handler.handle_memory_tool_calls.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_continuation_overflow_ends_the_stream_with_an_error(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import headroom.proxy.memory_tool_stream as mts++ monkeypatch.setattr(mts, "DEFAULT_MAX_RETAINED_BYTES", 1024)+ proxy = _proxy([_sse([TEXT, SAVE], "tool_use"), self._oversized_save()], SAVE_RESULT)+ events = await _client_view(proxy, server_memory_tool_names=MEMORY_TOOLS)++ assert proxy.http_client.send.await_count == 2+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()+ assert events[-1]["type"] == "error"+ assert self.SECRET not in json.dumps(events)