Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(ccr): stream the buffered CCR upstream request so idle-killing networks cannot cut it (#4206)

gglucassca6ba3486c09
diff --git a/docs/content/docs/security-model.mdx b/docs/content/docs/security-model.mdx
index dcf93eff6..1ab994d81 100644
--- a/docs/content/docs/security-model.mdx
+++ b/docs/content/docs/security-model.mdx
@@ -284,6 +284,7 @@ accept arbitrary environment variables — writes are restricted to a hardcoded
| `HEADROOM_INTERCEPT_READ_MIN_CHARS` | ast-grep read-interception threshold |
| `HEADROOM_VERIFY_TRUNCATION_ON_DISK` | Verify ast-grep Read truncation against disk when no client banner is found. Always returns `UNKNOWN` on Windows, where `dir_fd` / `O_NOFOLLOW` support is unavailable. |
| `HEADROOM_VERIFY_TRUNCATION_MAX_BYTES` | Max on-disk file size (bytes) the disk-verify fallback will read |
+| `HEADROOM_BUFFERED_UPSTREAM_STREAM` | Send buffered CCR requests (and their continuations) upstream as `stream: true` and fold the reply (default on; `0` sends `stream: false`) |
Keys outside this list are ignored, and `/admin/upstream` is read-only — there is no
write route on the admin surface, specifically so a caller cannot redirect
diff --git a/headroom/proxy/anthropic_wire.py b/headroom/proxy/anthropic_wire.py
index df7b62048..c4066ec70 100644
--- a/headroom/proxy/anthropic_wire.py
+++ b/headroom/proxy/anthropic_wire.py
@@ -59,6 +59,13 @@ _ACCUMULATED_FIELDS = {
"input_json_delta": "_partial_json",
"thinking_delta": "thinking",
}
+# What a non-streaming Message carries for block types whose stream form adds
+# placeholders: a thinking ``content_block_start`` carries
+# ``estimated_tokens: null``, which the Message schema lacks.
+_MESSAGE_BLOCK_FIELDS: dict[str, frozenset[str]] = {
+ "thinking": frozenset({"type", "thinking", "signature"}),
+ "redacted_thinking": frozenset({"type", "data"}),
+}
_KNOWN_MESSAGE_FIELDS = frozenset(
{
"id",
@@ -421,7 +428,11 @@ def _response_from_events(frames: list[_SSEFrame]) -> _Reconstruction:
response[key] = copy.deepcopy(delta[key])
extensions.message_delta_delta.update(_extensions(delta, _MESSAGE_DELTA_FIELDS))
if isinstance(usage, dict):
- response.setdefault("usage", {}).update(copy.deepcopy(usage))
+ # Merged the way the Anthropic SDK's MessageStream does: a null
+ # count here never overwrites the one message_start reported.
+ response.setdefault("usage", {}).update(
+ {key: copy.deepcopy(value) for key, value in usage.items() if value is not None}
+ )
for key in usage:
if key not in extensions.message_delta_usage_keys:
extensions.message_delta_usage_keys.append(key)
@@ -682,3 +693,51 @@ def render_anthropic_sse_response(response: dict[str, Any]) -> list[bytes]:
"""
return _render_known_response(response)
+
+
+def fold_anthropic_stream(raw_sse_bytes: bytes) -> dict[str, Any] | None:
+ """Fold a whole Anthropic event stream into the body ``stream: false`` returns.
+
+ Blocks are accumulated as the Anthropic SDK's ``MessageStream`` does (see
+ ``_response_from_events``), then trimmed to the non-streaming Message
+ shape: a null member leaves every block, and thinking blocks keep only
+ their schema. The proxy's own continuations resend these blocks as
+ history, and Anthropic rejects a block carrying a stream-only member with
+ 400 "Extra inputs are not permitted".
+
+ Args:
+ raw_sse_bytes: The complete Anthropic SSE reply.
+
+ Returns:
+ The Message; the stream's ``error`` event when it carried one (the
+ same shape as an Anthropic HTTP error body); or None when the stream
+ ended before ``message_stop`` or with a block still open.
+
+ Raises:
+ ValueError: The stream is complete but holds a known event this
+ module cannot fold, such as a delta type it does not know.
+ """
+
+ envelope = AnthropicSSEEnvelope.parse(raw_sse_bytes)
+ if envelope.saw_error:
+ for raw in _split_frames(raw_sse_bytes):
+ event_type, payload = _json_payload(raw)
+ if event_type == "error" and payload is not None:
+ return payload
+ if not envelope.is_complete():
+ return None
+ if envelope.saw_unreconstructable_frame:
+ raise ValueError("the event stream holds a frame that cannot be folded")
+ message = envelope.message
+ content = []
+ for block in message["content"]:
+ fields = _MESSAGE_BLOCK_FIELDS.get(block.get("type"))
+ content.append(
+ {
+ key: value
+ for key, value in block.items()
+ if value is not None and (fields is None or key in fields)
+ }
+ )
+ message["content"] = content
+ return message
diff --git a/headroom/proxy/buffered_ccr_response.py b/headroom/proxy/buffered_ccr_response.py
index cd0bc0b9a..95daf130d 100644
--- a/headroom/proxy/buffered_ccr_response.py
+++ b/headroom/proxy/buffered_ccr_response.py
@@ -76,18 +76,21 @@ class BufferedCCRErrorFormat:
heartbeat: bytes
+# The wire types Anthropic documents; clients switch their retry behaviour
+# on these, so a 429 must not arrive labelled `api_error`.
+ANTHROPIC_ERROR_TYPES: dict[int, str] = {
+ 400: "invalid_request_error",
+ 401: "authentication_error",
+ 403: "permission_error",
+ 404: "not_found_error",
+ 413: "request_too_large",
+ 429: "rate_limit_error",
+ 529: "overloaded_error",
+}
+
+
def _anthropic_error_type(status: int) -> str:
- # The wire types Anthropic documents; clients switch their retry behaviour
- # on these, so a 429 must not arrive labelled `api_error`.
- return {
- 400: "invalid_request_error",
- 401: "authentication_error",
- 403: "permission_error",
- 404: "not_found_error",
- 413: "request_too_large",
- 429: "rate_limit_error",
- 529: "overloaded_error",
- }.get(status, "api_error")
+ return ANTHROPIC_ERROR_TYPES.get(status, "api_error")
def _openai_error_type(status: int) -> str:
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 742372ec7..3344e9fef 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -12,9 +12,9 @@ import logging
import os
import time
import uuid
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
from datetime import datetime
-from functools import lru_cache
+from functools import lru_cache, partial
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urlsplit
@@ -40,6 +40,7 @@ from headroom.pipeline import PipelineStage, summarize_routing_markers
from headroom.proxy import public_errors
from headroom.proxy.anthropic_wire import (
build_anthropic_upstream_url,
+ fold_anthropic_stream,
is_safeguard_capable_request,
preserve_opaque_response_fields,
strip_safeguard_payload,
@@ -51,6 +52,7 @@ from headroom.proxy.auth_mode import (
)
from headroom.proxy.buffered_ccr_response import (
ANTHROPIC_ERROR_FORMAT,
+ ANTHROPIC_ERROR_TYPES,
DEFAULT_BUFFERED_CCR_GRACE_SECONDS,
buffered_ccr_asgi_call,
)
@@ -402,6 +404,87 @@ class AnthropicHandlerMixin:
)
return httpx.Response(200, json=parsed, headers=headers)
+ async def _send_buffered_as_stream(
+ self,
+ send: Callable[[dict[str, str], dict[str, Any]], Awaitable[httpx.Response]],
+ headers: dict[str, str],
+ body: dict[str, Any],
+ *,
+ url: str,
+ request_id: str,
+ ) -> httpx.Response:
+ """Send a non-streaming request Headroom built as a stream, and fold the reply.
+
+ The buffered CCR path flips a streaming turn to ``stream: false`` so the
+ whole reply is in hand before server-side retrieval, and the CCR, memory
+ and turn-hook continuations are built from that body. Upstream then
+ sends nothing until the generation is complete, and a network that
+ drops HTTP connections idle for ~60s (TLS-inspecting antivirus, VPN,
+ corporate proxy; TCP keepalive does not reset an HTTP idle timer) cuts
+ every long turn: each retry fails at ~60s and the client gets a 502 it
+ resends forever. Asked for as a stream, upstream sends deltas and pings
+ for the whole generation; folded, the reply is the same Message JSON the
+ caller already reads.
+
+ ``send(headers, body)`` makes one upstream call. A reply that is not a
+ 200 event stream (an HTTP error, or an upstream that answered JSON
+ anyway) is returned unchanged. A connection dropped mid-stream raises
+ inside ``send``, as it does for a ``stream: false`` reply; a stream that
+ ends early gets the 502 of ``_adapt_event_stream_to_json``; an in-band
+ ``error`` event comes back with its HTTP status. A complete stream this
+ proxy cannot fold is answered by sending the original ``stream: false``
+ body instead, so it costs one more generation rather than a 502 the
+ client would resend forever.
+
+ First-party Anthropic only: a gateway's event stream is not something
+ the fold has been checked against. Kill switch:
+ ``HEADROOM_BUFFERED_UPSTREAM_STREAM=0``, a runtime-env knob, so
+ ``headroom wrap`` also pushes it to a reused proxy.
+ """
+ from headroom.providers.claude.runtime import is_custom_anthropic_base_url
+ from headroom.proxy import runtime_env
+
+ if (
+ body.get("stream")
+ or is_custom_anthropic_base_url(url)
+ or runtime_env.getenv("HEADROOM_BUFFERED_UPSTREAM_STREAM", "1").strip().lower()
+ in ("0", "false", "no", "off")
+ ):
+ return await send(headers, body)
+
+ stream_headers = {k: v for k, v in headers.items() if k.lower() != "accept"}
+ stream_headers["accept"] = "text/event-stream"
+ response = await send(stream_headers, {**body, "stream": True})
+ if response.status_code != 200 or not _looks_like_sse_response(response):
+ return response
+ try:
+ folded = fold_anthropic_stream(response.content)
+ except ValueError as exc:
+ logger.warning(
+ f"[{request_id}] Could not fold the upstream event stream ({exc}); "
+ "sending the request again as stream:false"
+ )
+ return await send(headers, body)
+ if folded is None:
+ # Ended early: the strict 502 any truncated reply here gets (#3266).
+ return self._adapt_event_stream_to_json(response, request_id)
+
+ status_code = 200
+ if folded.get("type") == "error":
+ error_type = (folded.get("error") or {}).get("type")
+ status_code = {name: code for code, name in ANTHROPIC_ERROR_TYPES.items()}.get(
+ error_type, 500
+ )
+ logger.warning(
+ f"[{request_id}] Upstream event stream carried an in-band "
+ f"{error_type!r} error; answering {status_code}"
+ )
+ return httpx.Response(
+ status_code,
+ json=folded,
+ headers=sanitize_forwarded_response_headers(response.headers, "content-type"),
+ )
+
async def _count_tokens_offloaded(self, model, messages): # noqa: ANN001, ANN201
from headroom.proxy.token_counting import count_tokens_offloaded
@@ -4212,11 +4295,10 @@ class AnthropicHandlerMixin:
async def _buffered_ccr_operation():
async with stage_timer.measure("upstream_connect"):
- response = await self._retry_request(
+ send_upstream = partial(
+ self._retry_request,
"POST",
url,
- headers,
- body,
original_body_bytes=original_body_bytes,
body_mutated=body_mutation_tracker.mutated,
mutation_reasons=body_mutation_tracker.reasons,
@@ -4225,6 +4307,14 @@ class AnthropicHandlerMixin:
path_for_log="/v1/messages",
timeout=self._anthropic_buffered_request_timeout(),
)
+ # Only the CCR flip goes out as a stream: a client's
+ # own stream:false request keeps its bytes.
+ if buffered_stream_ccr:
+ response = await self._send_buffered_as_stream(
+ send_upstream, headers, body, url=url, request_id=request_id
+ )
+ else:
+ response = await send_upstream(headers, body)
self.pipeline_extensions.emit(
PipelineStage.POST_SEND,
operation="proxy.request",
@@ -4520,13 +4610,6 @@ class AnthropicHandlerMixin:
)
from headroom.proxy.helpers import log_outbound_request
- ccr_outbound_bytes, ccr_outbound_source = (
- prepare_outbound_body_bytes(
- body=continuation_body,
- original_body_bytes=None,
- body_mutated=True,
- )
- )
# A continuation is a non-streaming call, so it
# needs a matching Accept for the same reason the
# buffered flip above does (#3078).
@@ -4537,23 +4620,43 @@ class AnthropicHandlerMixin:
}
ccr_outbound_headers["content-type"] = "application/json"
ccr_outbound_headers["accept"] = "application/json"
- log_outbound_request(
- forwarder="anthropic_ccr_continuation",
- method="POST",
- path=url,
- body_bytes_count=len(ccr_outbound_bytes),
- body_mutated=True,
- mutation_reasons=["ccr_continuation"],
- request_id=request_id,
- source=ccr_outbound_source,
- )
- try:
- cont_response = await self.http_client.post(
+
+ async def _post_continuation(
+ send_headers: dict[str, str], send_body: dict[str, Any]
+ ) -> httpx.Response:
+ assert self.http_client is not None
+ ccr_outbound_bytes, ccr_outbound_source = (
+ prepare_outbound_body_bytes(
+ body=send_body,
+ original_body_bytes=None,
+ body_mutated=True,
+ )
+ )
+ log_outbound_request(
+ forwarder="anthropic_ccr_continuation",
+ method="POST",
+ path=url,
+ body_bytes_count=len(ccr_outbound_bytes),
+ body_mutated=True,
+ mutation_reasons=["ccr_continuation"],
+ request_id=request_id,
+ source=ccr_outbound_source,
+ )
+ return await self.http_client.post(
url,
content=ccr_outbound_bytes,
- headers=ccr_outbound_headers,
+ headers=send_headers,
timeout=self._anthropic_buffered_request_timeout(),
)
+
+ try:
+ cont_response = await self._send_buffered_as_stream(
+ _post_continuation,
+ ccr_outbound_headers,
+ continuation_body,
+ url=url,
+ request_id=request_id,
+ )
logger.info(
f"CCR: Got response status={cont_response.status_code}, "
f"content-encoding={cont_response.headers.get('content-encoding')}"
@@ -4758,12 +4861,17 @@ class AnthropicHandlerMixin:
if tools:
continuation_body["tools"] = tools
- cont_response = await self._retry_request(
- "POST",
- url,
+ cont_response = await self._send_buffered_as_stream(
+ partial(
+ self._retry_request,
+ "POST",
+ url,
+ timeout=self._anthropic_buffered_request_timeout(),
+ ),
headers,
continuation_body,
- timeout=self._anthropic_buffered_request_timeout(),
+ url=url,
+ request_id=request_id,
)
# Update response with continuation
@@ -4800,12 +4908,17 @@ class AnthropicHandlerMixin:
hook_messages: list[dict[str, Any]],
) -> dict[str, Any]:
continuation_body = {**body, "messages": hook_messages}
- continuation_response = await self._retry_request(
- "POST",
- url,
+ continuation_response = await self._send_buffered_as_stream(
+ partial(
+ self._retry_request,
+ "POST",
+ url,
+ timeout=self._anthropic_buffered_request_timeout(),
+ ),
headers,
continuation_body,
- timeout=self._anthropic_buffered_request_timeout(),
+ url=url,
+ request_id=request_id,
)
continuation_json = continuation_response.json()
_hook_usage.record(continuation_json)
diff --git a/headroom/proxy/runtime_env.py b/headroom/proxy/runtime_env.py
index a3415042e..08110776a 100644
--- a/headroom/proxy/runtime_env.py
+++ b/headroom/proxy/runtime_env.py
@@ -84,6 +84,12 @@ RUNTIME_ENV_KNOBS: tuple[Knob, ...] = (
"int",
"Max on-disk file size (bytes) the disk-verify fallback will read.",
),
+ Knob(
+ "HEADROOM_BUFFERED_UPSTREAM_STREAM",
+ "bool",
+ "Send buffered CCR requests upstream as stream:true and fold the reply "
+ "(default on; 0 sends stream:false).",
+ ),
)
_KNOBS_BY_ENV: dict[str, Knob] = {k.env: k for k in RUNTIME_ENV_KNOBS}
diff --git a/tests/test_anthropic_buffered_ccr_usage.py b/tests/test_anthropic_buffered_ccr_usage.py
index 46569cce4..034b48479 100644
--- a/tests/test_anthropic_buffered_ccr_usage.py
+++ b/tests/test_anthropic_buffered_ccr_usage.py
@@ -1,9 +1,9 @@
"""A buffered-CCR turn is a billed call and its usage must reach accounting.
When a ``stream:true`` request carries the ``headroom_retrieve`` tool, the
-Anthropic handler rewrites it to ``stream:false`` upstream so it can resolve
-retrievals server-side, then re-synthesizes SSE for the client. That buffered
-response carries the same ``usage`` block any non-stream reply does, so the
+Anthropic handler buffers the whole reply (upstream streams it, the proxy folds
+the stream into one Message) so it can resolve retrievals server-side, then
+re-synthesizes SSE for the client. That buffered response carries the same ``usage`` block any non-stream reply does, so the
provider's cache-read / cache-write / output counts must land on the outcome.
If they don't, every cached token on the dominant Claude Code path is invisible:
@@ -15,6 +15,7 @@ compression savings on the buffered turns still count in full.
from __future__ import annotations
+import json
from typing import Any
import httpx
@@ -25,6 +26,7 @@ pytest.importorskip("fastapi")
from fastapi.testclient import TestClient # noqa: E402
+from headroom.proxy.anthropic_wire import render_anthropic_sse_response # noqa: E402
from headroom.proxy.loopback_guard import require_loopback # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
@@ -139,15 +141,19 @@ def test_buffered_ccr_turn_records_provider_usage(monkeypatch, ccr_marker: str)
"""A stream:true + headroom_retrieve turn must book the same usage block."""
app, outcomes = _app_and_outcomes(monkeypatch)
route = respx.post("https://api.anthropic.com/v1/messages").mock(
- return_value=httpx.Response(200, json=_RESPONSE)
+ return_value=httpx.Response(
+ 200,
+ content=b"".join(render_anthropic_sse_response(_RESPONSE)),
+ headers={"content-type": "text/event-stream"},
+ )
)
r = _post(app, stream=True, tools=[_RETRIEVE_TOOL], marker=ccr_marker)
assert r.status_code == 200
- # The handler must have buffered it: upstream saw stream:false.
+ # The handler buffered it: upstream was asked for the stream it then folded.
sent = route.calls.last.request
- assert b'"stream": false' in sent.content or b'"stream":false' in sent.content
+ assert json.loads(sent.content)["stream"] is True
assert outcomes, "an outcome must be recorded"
o = outcomes[-1]
diff --git a/tests/test_buffered_ccr_accept_header.py b/tests/test_buffered_ccr_accept_header.py
index 3a663cb62..822cbb7be 100644
--- a/tests/test_buffered_ccr_accept_header.py
+++ b/tests/test_buffered_ccr_accept_header.py
@@ -34,6 +34,17 @@ from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
[email protected](autouse=True)
+def _stream_false_leg(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Pin the ``stream: false`` request #3078 is about.
+
+ A first-party buffered turn now asks upstream for a stream and folds it
+ (``test_buffered_ccr_upstream_stream.py``). This request is still what a
+ custom upstream such as Copilot gets, and what the kill switch restores.
+ """
+ monkeypatch.setenv("HEADROOM_BUFFERED_UPSTREAM_STREAM", "0")
+
+
@pytest.fixture(autouse=True)
def _store():
reset_compression_store()
diff --git a/tests/test_buffered_ccr_upstream_stream.py b/tests/test_buffered_ccr_upstream_stream.py
new file mode 100644
index 000000000..acb3c7822
--- /dev/null
+++ b/tests/test_buffered_ccr_upstream_stream.py
@@ -0,0 +1,540 @@
+"""The buffered CCR request streams from upstream, and the proxy folds the reply.
+
+The buffered CCR path flips a streaming turn to ``stream: false`` so the whole
+reply is in hand before server-side retrieval. Sent that way, upstream sends
+nothing until the generation is complete, and a network that drops HTTP
+connections idle for ~60s (TLS-inspecting antivirus, VPN, corporate proxy)
+cut every long turn: three attempts failed at ~60s each and the client got a
+502 it resent forever. The request now goes out as a stream (as do the CCR,
+memory and turn-hook continuations built from it) and the event stream is
+folded back into the Message the buffered path reads.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+from typing import Any
+
+import pytest
+
+fastapi = pytest.importorskip("fastapi")
+httpx = pytest.importorskip("httpx")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.cache.backends import InMemoryBackend # noqa: E402
+from headroom.cache.compression_store import ( # noqa: E402
+ get_compression_store,
+ reset_compression_store,
+)
+from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402
+from headroom.proxy.anthropic_wire import fold_anthropic_stream # noqa: E402
+from headroom.proxy.server import HeadroomProxy, ProxyConfig, create_app # noqa: E402
+
+URL = "https://api.anthropic.com/v1/messages"
+
+
+def _sse(*events: dict[str, Any]) -> bytes:
+ return b"".join(
+ f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events
+ )
+
+
+def _message_start(model: str = "claude-opus-5") -> dict[str, Any]:
+ return {
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": model,
+ "content": [],
+ "stop_reason": None,
+ "stop_sequence": None,
+ "usage": {
+ "input_tokens": 25,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 100,
+ "output_tokens": 1,
+ },
+ },
+ }
+
+
+def _text_turn(text: str) -> list[dict[str, Any]]:
+ return [
+ _message_start(),
+ {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
+ {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
+ {"type": "content_block_stop", "index": 0},
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+ "usage": {"output_tokens": 7},
+ },
+ {"type": "message_stop"},
+ ]
+
+
+# A turn exercising every delta the fold accumulates, with the members only a
+# stream carries: a thinking block's ``estimated_tokens`` placeholder, a delta's
+# progress count, interleaved pings, and a null usage count in message_delta.
+EVERY_DELTA = [
+ _message_start(),
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {
+ "type": "thinking",
+ "thinking": "",
+ "signature": "",
+ "estimated_tokens": None,
+ },
+ },
+ {"type": "ping"},
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "thinking_delta", "thinking": "Let me ", "estimated_tokens": 3},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "thinking_delta", "thinking": "check."},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "signature_delta", "signature": "sig-1"},
+ },
+ {"type": "content_block_stop", "index": 0},
+ {
+ "type": "content_block_start",
+ "index": 1,
+ "content_block": {"type": "redacted_thinking", "data": "opaque"},
+ },
+ {"type": "content_block_stop", "index": 1},
+ {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}},
+ {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris"}},
+ {
+ "type": "content_block_delta",
+ "index": 2,
+ "delta": {
+ "type": "citations_delta",
+ "citation": {
+ "type": "char_location",
+ "cited_text": "Paris",
+ "document_index": 0,
+ "document_title": "Atlas",
+ "start_char_index": 0,
+ "end_char_index": 5,
+ },
+ },
+ },
+ {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": " it is."}},
+ {"type": "content_block_stop", "index": 2},
+ {
+ "type": "content_block_start",
+ "index": 3,
+ "content_block": {"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {}},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 3,
+ "delta": {"type": "input_json_delta", "partial_json": '{"file_pa'},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 3,
+ "delta": {"type": "input_json_delta", "partial_json": 'th": "a.py"}'},
+ },
+ {"type": "content_block_stop", "index": 3},
+ {"type": "ping"},
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": "tool_use", "stop_sequence": None},
+ "usage": {"output_tokens": 42, "input_tokens": None},
+ },
+ {"type": "message_stop"},
+]
+
+
+# --------------------------------------------------------------------------- #
+# The fold
+# --------------------------------------------------------------------------- #
+def test_fold_matches_the_anthropic_sdk_accumulator() -> None:
+ streaming = pytest.importorskip("anthropic.lib.streaming._messages")
+ accumulate = streaming.accumulate_event
+ # anthropic 1.x keeps each tool block's partial JSON in a buffer the caller
+ # owns (``json_bufs``); 0.x kept it on the block. Same accumulation either way.
+ buffers: dict[str, Any] = (
+ {"json_bufs": {}} if "json_bufs" in inspect.signature(accumulate).parameters else {}
+ )
+ snapshot = None
+ for event in EVERY_DELTA:
+ snapshot = accumulate(event=event, current_snapshot=snapshot, **buffers)
+ assert snapshot is not None
+
+ folded = fold_anthropic_stream(_sse(*EVERY_DELTA))
+
+ assert folded is not None
+ # The SDK keeps nulls (``stop_sequence``, the thinking placeholder); the
+ # non-streaming Message drops the placeholder, so compare without them.
+ assert {k: v for k, v in folded.items() if v is not None} == snapshot.model_dump(
+ exclude_none=True
+ )
+ assert folded["usage"]["input_tokens"] == 25, "a null delta count must not erase it"
+
+
+def test_fold_keeps_stream_only_members_out_of_blocks() -> None:
+ """The proxy resends folded blocks; Anthropic 400s on any extra member."""
+ folded = fold_anthropic_stream(_sse(*EVERY_DELTA))
+
+ assert folded is not None
+ assert folded["content"][0] == {
+ "type": "thinking",
+ "thinking": "Let me check.",
+ "signature": "sig-1",
+ }
+ assert folded["content"][1] == {"type": "redacted_thinking", "data": "opaque"}
+ assert "estimated_tokens" not in json.dumps(folded)
+
+
+def test_fold_reports_a_truncated_stream() -> None:
+ assert fold_anthropic_stream(_sse(*EVERY_DELTA[:-2])) is None
+ assert fold_anthropic_stream(_sse(*EVERY_DELTA[:5])) is None # a block left open
+
+
+def test_fold_returns_an_in_band_error_event() -> None:
+ error = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
+
+ assert fold_anthropic_stream(_sse(*EVERY_DELTA[:7], error)) == error
+
+
+def test_fold_refuses_a_delta_it_does_not_know() -> None:
+ unknown = {"type": "content_block_delta", "index": 2, "delta": {"type": "future_delta"}}
+
+ with pytest.raises(ValueError):
+ fold_anthropic_stream(_sse(*EVERY_DELTA[:10], unknown, *EVERY_DELTA[10:]))
+
+
+# --------------------------------------------------------------------------- #
+# The send helper
+# --------------------------------------------------------------------------- #
+def proxy() -> HeadroomProxy:
+ return HeadroomProxy(ProxyConfig(optimize=False, cache_enabled=False, image_optimize=False))
+
+
+class _Upstream:
+ """Answers a stream:true request with ``stream_reply``, anything else with JSON."""
+
+ def __init__(self, stream_reply: httpx.Response) -> None:
+ self.stream_reply = stream_reply
+ self.calls: list[tuple[dict[str, str], dict[str, Any]]] = []
+
+ async def __call__(self, headers: dict[str, str], body: dict[str, Any]) -> httpx.Response:
+ self.calls.append((dict(headers), json.loads(json.dumps(body))))
+ if body.get("stream"):
+ return self.stream_reply
+ return httpx.Response(200, json={"type": "message", "content": [], "via": "json"})
+
+
+def _stream_reply(*events: dict[str, Any]) -> httpx.Response:
+ return httpx.Response(200, content=_sse(*events), headers={"content-type": "text/event-stream"})
+
+
+async def _send(
+ proxy: HeadroomProxy, upstream: _Upstream, *, url: str = URL, body: dict | None = None
+) -> httpx.Response:
+ return await proxy._send_buffered_as_stream(
+ upstream,
+ {"Accept": "application/json", "x-api-key": "k"},
+ body if body is not None else {"model": "m", "stream": False, "messages": []},
+ url=url,
+ request_id="t",
+ )
+
+
+async def test_a_buffered_request_streams_and_folds(proxy) -> None:
+ upstream = _Upstream(_stream_reply(*_text_turn("hello")))
+ body = {"model": "m", "stream": False, "messages": []}
+
+ response = await _send(proxy, upstream, body=body)
+
+ assert response.status_code == 200
+ assert response.json()["content"] == [{"type": "text", "text": "hello"}]
+ assert len(upstream.calls) == 1
+ headers, sent = upstream.calls[0]
+ assert sent["stream"] is True
+ assert [v for k, v in headers.items() if k.lower() == "accept"] == ["text/event-stream"]
+ assert body["stream"] is False, "the caller's body is not mutated"
+
+
+ ("env", "url", "stream"),
+ [
+ ("0", URL, False), # kill switch
+ ("1", "https://gateway.example/v1/messages", False), # not first-party
+ ("1", URL, True), # already a stream
+ ],
+)
+async def test_requests_it_should_not_touch_go_out_unchanged(
+ proxy, monkeypatch, env: str, url: str, stream: bool
+) -> None:
+ monkeypatch.setenv("HEADROOM_BUFFERED_UPSTREAM_STREAM", env)
+ upstream = _Upstream(_stream_reply(*_text_turn("hello")))
+ body = {"model": "m", "stream": stream, "messages": []}
+
+ await _send(proxy, upstream, url=url, body=body)
+
+ assert upstream.calls == [({"Accept": "application/json", "x-api-key": "k"}, body)]
+
+
+async def test_a_propagated_kill_switch_keeps_the_request_non_streaming(proxy) -> None:
+ # `headroom wrap` pushes this knob to a reused proxy, so the rollback works
+ # without the proxy's own environment or a restart.
+ from headroom.proxy import runtime_env
+
+ assert runtime_env.explicit_env({"HEADROOM_BUFFERED_UPSTREAM_STREAM": "0"}) == {
+ "HEADROOM_BUFFERED_UPSTREAM_STREAM": "0"
+ }
+ upstream = _Upstream(_stream_reply(*_text_turn("hello")))
+ body = {"model": "m", "stream": False, "messages": []}
+ runtime_env.set_overrides({"HEADROOM_BUFFERED_UPSTREAM_STREAM": "0"})
+ try:
+ response = await _send(proxy, upstream, body=body)
+ finally:
+ runtime_env.clear_overrides()
+
+ assert upstream.calls == [({"Accept": "application/json", "x-api-key": "k"}, body)]
+ assert response.json()["via"] == "json"
+
+
+ "reply",
+ [
+ httpx.Response(529, json={"type": "error", "error": {"type": "overloaded_error"}}),
+ httpx.Response(200, json={"type": "message", "content": []}), # answered JSON anyway
+ ],
+)
+async def test_a_reply_that_is_not_an_event_stream_is_returned_as_is(proxy, reply) -> None:
+ upstream = _Upstream(reply)
+
+ assert await _send(proxy, upstream) is reply
+
+
+async def test_an_in_band_error_comes_back_with_its_status(proxy) -> None:
+ error = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
+ upstream = _Upstream(_stream_reply(_message_start(), error))
+
+ response = await _send(proxy, upstream)
+
+ assert response.status_code == 529
+ assert response.json() == error
+
+
+async def test_a_truncated_stream_is_a_502(proxy) -> None:
+ upstream = _Upstream(_stream_reply(*_text_turn("hello")[:-2]))
+
+ response = await _send(proxy, upstream)
+
+ assert response.status_code == 502
+ assert response.json()["error"]["type"] == "upstream_protocol_error"
+ assert len(upstream.calls) == 1
+
+
+async def test_a_stream_it_cannot_fold_is_sent_again_as_stream_false(proxy) -> None:
+ unknown = {"type": "content_block_delta", "index": 0, "delta": {"type": "future_delta"}}
+ events = _text_turn("hello")
+ upstream = _Upstream(_stream_reply(*events[:3], unknown, *events[3:]))
+
+ response = await _send(proxy, upstream)
+
+ assert response.json()["via"] == "json"
+ assert [sent["stream"] for _, sent in upstream.calls] == [True, False]
+ assert upstream.calls[1][0]["Accept"] == "application/json"
+
+
+# --------------------------------------------------------------------------- #
+# Through the handler
+# --------------------------------------------------------------------------- #
[email protected](autouse=True)
+def _store():
+ reset_compression_store()
+ get_compression_store(backend=InMemoryBackend())
+ try:
+ yield
+ finally:
+ reset_compression_store()
+
+
+def _config() -> ProxyConfig:
+ return ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ memory_enabled=False,
+ ccr_inject_tool=True,
+ ccr_handle_responses=True,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+
+
+class _ContinuationClient:
+ """Stands in for ``http_client``; the CCR continuation posts through it."""
+
+ def __init__(self, reply: httpx.Response) -> None:
+ self.reply = reply
+ self.calls: list[tuple[dict[str, str], dict[str, Any]]] = []
+
+ async def post(self, url, *, content=None, headers=None, timeout=None): # noqa: ANN001
+ self.calls.append((dict(headers or {}), json.loads(content)))
+ return self.reply
+
+ async def aclose(self) -> None:
+ return None
+
+
+def _accept(headers: dict[str, str]) -> list[str]:
+ return [v for k, v in headers.items() if k.lower() == "accept"]
+
+
+def test_the_buffered_turn_and_its_ccr_continuation_stream_upstream() -> None:
+ store = get_compression_store()
+ marker = store.store(
+ original=json.dumps({"secret": "retrieved answer"}),
+ compressed="{}",
+ original_item_count=1,
+ )
+ retrieve_turn = [
+ _message_start(),
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {
+ "type": "thinking",
+ "thinking": "",
+ "signature": "",
+ "estimated_tokens": None,
+ },
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "thinking_delta", "thinking": "I need it."},
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "signature_delta", "signature": "sig"},
+ },
+ {"type": "content_block_stop", "index": 0},
+ {
+ "type": "content_block_start",
+ "index": 1,
+ "content_block": {
+ "type": "tool_use",
+ "id": "toolu_ccr",
+ "name": "headroom_retrieve",
+ "input": {},
+ },
+ },
+ {
+ "type": "content_block_delta",
+ "index": 1,
+ "delta": {"type": "input_json_delta", "partial_json": json.dumps({"hash": marker})},
+ },
+ {"type": "content_block_stop", "index": 1},
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": "tool_use", "stop_sequence": None},
+ "usage": {"output_tokens": 9},
+ },
+ {"type": "message_stop"},
+ ]
+ sent: list[tuple[dict[str, str], dict[str, Any]]] = []
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ sent.append((dict(headers), json.loads(json.dumps(body))))
+ return _stream_reply(*retrieve_turn)
+
+ proxy._retry_request = _fake_retry # type: ignore[assignment]
+ continuation = _ContinuationClient(_stream_reply(*_text_turn("the answer")))
+ proxy.http_client = continuation
+
+ resp = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-opus-5",
+ "max_tokens": 64,
+ "stream": True,
+ "tools": [create_ccr_tool_definition("anthropic")],
+ "messages": [{"role": "user", "content": f"use <<ccr:{marker}>>"}],
+ },
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert "the answer" in resp.text
+ assert "headroom_retrieve" not in resp.text
+ [(headers, body)] = sent
+ assert body["stream"] is True
+ assert _accept(headers) == ["text/event-stream"]
+ [(cont_headers, cont_body)] = continuation.calls
+ assert cont_body["stream"] is True
+ assert _accept(cont_headers) == ["text/event-stream"]
+ # The folded turn is resent as history: no stream-only member may ride along.
+ assert cont_body["messages"][-2]["content"][0] == {
+ "type": "thinking",
+ "thinking": "I need it.",
+ "signature": "sig",
+ }
+
+
+def test_a_client_stream_false_request_is_left_alone() -> None:
+ marker = get_compression_store().store(
+ original=json.dumps({"earlier": "tool output"}),
+ compressed="{}",
+ original_item_count=1,
+ )
+ sent: list[tuple[dict[str, str], dict[str, Any], dict[str, Any]]] = []
+ app = create_app(_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ sent.append((dict(headers), json.loads(json.dumps(body)), kwargs))
+ return httpx.Response(200, json=fold_anthropic_stream(_sse(*_text_turn("ok"))))
+
+ proxy._retry_request = _fake_retry # type: ignore[assignment]
+ resp = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-opus-5",
+ "max_tokens": 64,
+ "stream": False,
+ "tools": [create_ccr_tool_definition("anthropic")],
+ "messages": [{"role": "user", "content": f"use <<ccr:{marker}>>"}],
+ },
+ )
+
+ assert resp.status_code == 200, resp.text
+ [(headers, body, kwargs)] = sent
+ assert body["stream"] is False
+ assert _accept(headers) == ["application/json"]
+ assert kwargs["original_body_bytes"] is not None
diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.py
index 8130d0145..82a2fbe52 100644
--- a/tests/test_ccr_buffered_stream_signed_thinking.py
+++ b/tests/test_ccr_buffered_stream_signed_thinking.py
@@ -175,7 +175,8 @@ def test_signed_thinking_history_skips_the_buffered_ccr_path(
assert calls["stream_body"]["stream"] is True
else:
assert "buffered_body" in calls, "expected the buffered CCR path"
- assert calls["buffered_body"]["stream"] is False
+ # The buffered request itself streams; the proxy folds the reply.
+ assert calls["buffered_body"]["stream"] is True
@pytest.mark.parametrize("upstream_delay", [0.0, 1.2], ids=["prompt", "past-keepalive"])
@@ -278,7 +279,8 @@ def test_buffering_is_gated_on_a_redeemable_marker(
assert resp.status_code == 200, resp.text
if expect_buffered:
assert "buffered_body" in calls, "a redeemable marker must still buffer"
- assert calls["buffered_body"]["stream"] is False
+ # The buffered request itself streams; the proxy folds the reply.
+ assert calls["buffered_body"]["stream"] is True
else:
assert "stream_body" in calls, "nothing to retrieve — the client must keep streaming"
assert "buffered_body" not in calls
diff --git a/tests/test_no_ccr_disables_response_handling.py b/tests/test_no_ccr_disables_response_handling.py
index 5a1947b8a..2f71fcab6 100644
--- a/tests/test_no_ccr_disables_response_handling.py
+++ b/tests/test_no_ccr_disables_response_handling.py
@@ -156,9 +156,10 @@ def _upstream_stream_field(*, ccr_handle_responses: bool) -> object:
def test_client_offered_tool_still_buffers_when_handling_is_on(_store) -> None: # noqa: ANN001
"""The behaviour being switched off — pinned so the switch is meaningful."""
- assert _upstream_stream_field(ccr_handle_responses=True) is False
+ # Only the buffered path sends through `_retry_request`, and its request streams.
+ assert _upstream_stream_field(ccr_handle_responses=True) is True
def test_client_offered_tool_does_not_buffer_under_no_ccr(_store) -> None: # noqa: ANN001
"""With the switch honoured, the turn keeps streaming despite the tool."""
- assert _upstream_stream_field(ccr_handle_responses=False) is not False
+ assert _upstream_stream_field(ccr_handle_responses=False) is None
diff --git a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py
index 2bf8110e5..274c4ba92 100644
--- a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py
+++ b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py
@@ -141,7 +141,7 @@ def test_streaming_headroom_retrieve_is_intercepted_and_returned_as_sse() -> Non
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
initial_bodies.append(json.loads(json.dumps(body)))
assert stream is False
- assert body["stream"] is False
+ assert body["stream"] is True
return httpx.Response(200, json=initial_response)
proxy._retry_request = _fake_retry # type: ignore[assignment]
@@ -167,10 +167,10 @@ def test_streaming_headroom_retrieve_is_intercepted_and_returned_as_sse() -> Non
assert "text/event-stream" in resp.headers["content-type"]
assert "retrieved answer is now available" in resp.text
assert "headroom_retrieve" not in resp.text
- assert initial_bodies and initial_bodies[0]["stream"] is False
+ assert initial_bodies and initial_bodies[0]["stream"] is True
assert len(continuation_client.post_calls) == 1
continuation_body = json.loads(continuation_client.post_calls[0]["content"].decode())
- assert continuation_body["stream"] is False
+ assert continuation_body["stream"] is True
continuation_headers = {
key.lower(): value for key, value in continuation_client.post_calls[0]["headers"].items()
}
@@ -232,7 +232,7 @@ def test_streaming_with_headroom_retrieve_available_but_unused_returns_sse() ->
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
initial_bodies.append(json.loads(json.dumps(body)))
assert stream is False
- assert body["stream"] is False
+ assert body["stream"] is True
return httpx.Response(200, json=text_response)
proxy._retry_request = _fake_retry # type: ignore[assignment]
@@ -253,7 +253,7 @@ def test_streaming_with_headroom_retrieve_available_but_unused_returns_sse() ->
assert "text/event-stream" in resp.headers["content-type"]
assert "plain answer" in resp.text
assert "headroom_retrieve" not in resp.text
- assert initial_bodies and initial_bodies[0]["stream"] is False
+ assert initial_bodies and initial_bodies[0]["stream"] is True
assert continuation_client.post_calls == []
proxy._stream_response.assert_not_awaited()
@@ -311,7 +311,7 @@ def test_mixed_ccr_and_client_tool_serves_retrieval_then_streams_reissued_tool()
proxy.http_client = continuation_client
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
- assert body["stream"] is False
+ assert body["stream"] is True
return httpx.Response(200, json=initial_response)
proxy._retry_request = _fake_retry # type: ignore[assignment]
@@ -393,7 +393,7 @@ def test_mixed_turn_streams_the_continuation_or_the_turn_on_failure(status_code:
proxy.http_client = continuation_client
async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
- assert body["stream"] is False
+ assert body["stream"] is True
return httpx.Response(200, json=initial_response)
proxy._retry_request = _fake_retry # type: ignore[assignment]
diff --git a/tests/test_proxy_response_cache_replay.py b/tests/test_proxy_response_cache_replay.py
index 40b6b249b..6ba184dd2 100644
--- a/tests/test_proxy_response_cache_replay.py
+++ b/tests/test_proxy_response_cache_replay.py
@@ -295,7 +295,8 @@ def test_buffered_ccr_turn_does_not_write_the_response_cache():
assert resp.status_code == 200, resp.text
# The conversion really happened — otherwise this test proves nothing.
- assert forwarded_bodies and forwarded_bodies[0]["stream"] is False
+ # Buffered: the request streams upstream and the proxy folds the reply.
+ assert forwarded_bodies and forwarded_bodies[0]["stream"] is True
# ...and nothing was written to the response cache.
proxy.cache.set.assert_not_awaited()
reset_compression_store()