Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): stream the buffered CCR upstream request so idle-killing networks cannot cut it (#4206)
gglucassca6ba3486c09
diff --git a/docs/content/docs/security-model.mdx b/docs/content/docs/security-model.mdxindex dcf93eff6..1ab994d81 100644--- a/docs/content/docs/security-model.mdx+++ b/docs/content/docs/security-model.mdx@@ -284,6 +284,7 @@ accept arbitrary environment variables — writes are restricted to a hardcoded| `HEADROOM_INTERCEPT_READ_MIN_CHARS` | ast-grep read-interception threshold || `HEADROOM_VERIFY_TRUNCATION_ON_DISK` | Verify ast-grep Read truncation against disk when no client banner is found. Always returns `UNKNOWN` on Windows, where `dir_fd` / `O_NOFOLLOW` support is unavailable. || `HEADROOM_VERIFY_TRUNCATION_MAX_BYTES` | Max on-disk file size (bytes) the disk-verify fallback will read |+| `HEADROOM_BUFFERED_UPSTREAM_STREAM` | Send buffered CCR requests (and their continuations) upstream as `stream: true` and fold the reply (default on; `0` sends `stream: false`) |Keys outside this list are ignored, and `/admin/upstream` is read-only — there is nowrite route on the admin surface, specifically so a caller cannot redirectdiff --git a/headroom/proxy/anthropic_wire.py b/headroom/proxy/anthropic_wire.pyindex df7b62048..c4066ec70 100644--- a/headroom/proxy/anthropic_wire.py+++ b/headroom/proxy/anthropic_wire.py@@ -59,6 +59,13 @@ _ACCUMULATED_FIELDS = {"input_json_delta": "_partial_json","thinking_delta": "thinking",}+# What a non-streaming Message carries for block types whose stream form adds+# placeholders: a thinking ``content_block_start`` carries+# ``estimated_tokens: null``, which the Message schema lacks.+_MESSAGE_BLOCK_FIELDS: dict[str, frozenset[str]] = {+ "thinking": frozenset({"type", "thinking", "signature"}),+ "redacted_thinking": frozenset({"type", "data"}),+}_KNOWN_MESSAGE_FIELDS = frozenset({"id",@@ -421,7 +428,11 @@ def _response_from_events(frames: list[_SSEFrame]) -> _Reconstruction:response[key] = copy.deepcopy(delta[key])extensions.message_delta_delta.update(_extensions(delta, _MESSAGE_DELTA_FIELDS))if isinstance(usage, dict):- response.setdefault("usage", {}).update(copy.deepcopy(usage))+ # Merged the way the Anthropic SDK's MessageStream does: a null+ # count here never overwrites the one message_start reported.+ response.setdefault("usage", {}).update(+ {key: copy.deepcopy(value) for key, value in usage.items() if value is not None}+ )for key in usage:if key not in extensions.message_delta_usage_keys:extensions.message_delta_usage_keys.append(key)@@ -682,3 +693,51 @@ def render_anthropic_sse_response(response: dict[str, Any]) -> list[bytes]:"""return _render_known_response(response)+++def fold_anthropic_stream(raw_sse_bytes: bytes) -> dict[str, Any] | None:+ """Fold a whole Anthropic event stream into the body ``stream: false`` returns.++ Blocks are accumulated as the Anthropic SDK's ``MessageStream`` does (see+ ``_response_from_events``), then trimmed to the non-streaming Message+ shape: a null member leaves every block, and thinking blocks keep only+ their schema. The proxy's own continuations resend these blocks as+ history, and Anthropic rejects a block carrying a stream-only member with+ 400 "Extra inputs are not permitted".++ Args:+ raw_sse_bytes: The complete Anthropic SSE reply.++ Returns:+ The Message; the stream's ``error`` event when it carried one (the+ same shape as an Anthropic HTTP error body); or None when the stream+ ended before ``message_stop`` or with a block still open.++ Raises:+ ValueError: The stream is complete but holds a known event this+ module cannot fold, such as a delta type it does not know.+ """++ envelope = AnthropicSSEEnvelope.parse(raw_sse_bytes)+ if envelope.saw_error:+ for raw in _split_frames(raw_sse_bytes):+ event_type, payload = _json_payload(raw)+ if event_type == "error" and payload is not None:+ return payload+ if not envelope.is_complete():+ return None+ if envelope.saw_unreconstructable_frame:+ raise ValueError("the event stream holds a frame that cannot be folded")+ message = envelope.message+ content = []+ for block in message["content"]:+ fields = _MESSAGE_BLOCK_FIELDS.get(block.get("type"))+ content.append(+ {+ key: value+ for key, value in block.items()+ if value is not None and (fields is None or key in fields)+ }+ )+ message["content"] = content+ return messagediff --git a/headroom/proxy/buffered_ccr_response.py b/headroom/proxy/buffered_ccr_response.pyindex cd0bc0b9a..95daf130d 100644--- a/headroom/proxy/buffered_ccr_response.py+++ b/headroom/proxy/buffered_ccr_response.py@@ -76,18 +76,21 @@ class BufferedCCRErrorFormat:heartbeat: bytes+# The wire types Anthropic documents; clients switch their retry behaviour+# on these, so a 429 must not arrive labelled `api_error`.+ANTHROPIC_ERROR_TYPES: dict[int, str] = {+ 400: "invalid_request_error",+ 401: "authentication_error",+ 403: "permission_error",+ 404: "not_found_error",+ 413: "request_too_large",+ 429: "rate_limit_error",+ 529: "overloaded_error",+}++def _anthropic_error_type(status: int) -> str:- # The wire types Anthropic documents; clients switch their retry behaviour- # on these, so a 429 must not arrive labelled `api_error`.- return {- 400: "invalid_request_error",- 401: "authentication_error",- 403: "permission_error",- 404: "not_found_error",- 413: "request_too_large",- 429: "rate_limit_error",- 529: "overloaded_error",- }.get(status, "api_error")+ return ANTHROPIC_ERROR_TYPES.get(status, "api_error")def _openai_error_type(status: int) -> str:diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 742372ec7..3344e9fef 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -12,9 +12,9 @@ import loggingimport osimport timeimport uuid-from collections.abc import Callable+from collections.abc import Awaitable, Callablefrom datetime import datetime-from functools import lru_cache+from functools import lru_cache, partialfrom pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import urlsplit@@ -40,6 +40,7 @@ from headroom.pipeline import PipelineStage, summarize_routing_markersfrom headroom.proxy import public_errorsfrom headroom.proxy.anthropic_wire import (build_anthropic_upstream_url,+ fold_anthropic_stream,is_safeguard_capable_request,preserve_opaque_response_fields,strip_safeguard_payload,@@ -51,6 +52,7 @@ from headroom.proxy.auth_mode import ()from headroom.proxy.buffered_ccr_response import (ANTHROPIC_ERROR_FORMAT,+ ANTHROPIC_ERROR_TYPES,DEFAULT_BUFFERED_CCR_GRACE_SECONDS,buffered_ccr_asgi_call,)@@ -402,6 +404,87 @@ class AnthropicHandlerMixin:)return httpx.Response(200, json=parsed, headers=headers)+ async def _send_buffered_as_stream(+ self,+ send: Callable[[dict[str, str], dict[str, Any]], Awaitable[httpx.Response]],+ headers: dict[str, str],+ body: dict[str, Any],+ *,+ url: str,+ request_id: str,+ ) -> httpx.Response:+ """Send a non-streaming request Headroom built as a stream, and fold the reply.++ The buffered CCR path flips a streaming turn to ``stream: false`` so the+ whole reply is in hand before server-side retrieval, and the CCR, memory+ and turn-hook continuations are built from that body. Upstream then+ sends nothing until the generation is complete, and a network that+ drops HTTP connections idle for ~60s (TLS-inspecting antivirus, VPN,+ corporate proxy; TCP keepalive does not reset an HTTP idle timer) cuts+ every long turn: each retry fails at ~60s and the client gets a 502 it+ resends forever. Asked for as a stream, upstream sends deltas and pings+ for the whole generation; folded, the reply is the same Message JSON the+ caller already reads.++ ``send(headers, body)`` makes one upstream call. A reply that is not a+ 200 event stream (an HTTP error, or an upstream that answered JSON+ anyway) is returned unchanged. A connection dropped mid-stream raises+ inside ``send``, as it does for a ``stream: false`` reply; a stream that+ ends early gets the 502 of ``_adapt_event_stream_to_json``; an in-band+ ``error`` event comes back with its HTTP status. A complete stream this+ proxy cannot fold is answered by sending the original ``stream: false``+ body instead, so it costs one more generation rather than a 502 the+ client would resend forever.++ First-party Anthropic only: a gateway's event stream is not something+ the fold has been checked against. Kill switch:+ ``HEADROOM_BUFFERED_UPSTREAM_STREAM=0``, a runtime-env knob, so+ ``headroom wrap`` also pushes it to a reused proxy.+ """+ from headroom.providers.claude.runtime import is_custom_anthropic_base_url+ from headroom.proxy import runtime_env++ if (+ body.get("stream")+ or is_custom_anthropic_base_url(url)+ or runtime_env.getenv("HEADROOM_BUFFERED_UPSTREAM_STREAM", "1").strip().lower()+ in ("0", "false", "no", "off")+ ):+ return await send(headers, body)++ stream_headers = {k: v for k, v in headers.items() if k.lower() != "accept"}+ stream_headers["accept"] = "text/event-stream"+ response = await send(stream_headers, {**body, "stream": True})+ if response.status_code != 200 or not _looks_like_sse_response(response):+ return response+ try:+ folded = fold_anthropic_stream(response.content)+ except ValueError as exc:+ logger.warning(+ f"[{request_id}] Could not fold the upstream event stream ({exc}); "+ "sending the request again as stream:false"+ )+ return await send(headers, body)+ if folded is None:+ # Ended early: the strict 502 any truncated reply here gets (#3266).+ return self._adapt_event_stream_to_json(response, request_id)++ status_code = 200+ if folded.get("type") == "error":+ error_type = (folded.get("error") or {}).get("type")+ status_code = {name: code for code, name in ANTHROPIC_ERROR_TYPES.items()}.get(+ error_type, 500+ )+ logger.warning(+ f"[{request_id}] Upstream event stream carried an in-band "+ f"{error_type!r} error; answering {status_code}"+ )+ return httpx.Response(+ status_code,+ json=folded,+ headers=sanitize_forwarded_response_headers(response.headers, "content-type"),+ )+async def _count_tokens_offloaded(self, model, messages): # noqa: ANN001, ANN201from headroom.proxy.token_counting import count_tokens_offloaded@@ -4212,11 +4295,10 @@ class AnthropicHandlerMixin:async def _buffered_ccr_operation():async with stage_timer.measure("upstream_connect"):- response = await self._retry_request(+ send_upstream = partial(+ self._retry_request,"POST",url,- headers,- body,original_body_bytes=original_body_bytes,body_mutated=body_mutation_tracker.mutated,mutation_reasons=body_mutation_tracker.reasons,@@ -4225,6 +4307,14 @@ class AnthropicHandlerMixin:path_for_log="/v1/messages",timeout=self._anthropic_buffered_request_timeout(),)+ # Only the CCR flip goes out as a stream: a client's+ # own stream:false request keeps its bytes.+ if buffered_stream_ccr:+ response = await self._send_buffered_as_stream(+ send_upstream, headers, body, url=url, request_id=request_id+ )+ else:+ response = await send_upstream(headers, body)self.pipeline_extensions.emit(PipelineStage.POST_SEND,operation="proxy.request",@@ -4520,13 +4610,6 @@ class AnthropicHandlerMixin:)from headroom.proxy.helpers import log_outbound_request- ccr_outbound_bytes, ccr_outbound_source = (- prepare_outbound_body_bytes(- body=continuation_body,- original_body_bytes=None,- body_mutated=True,- )- )# A continuation is a non-streaming call, so it# needs a matching Accept for the same reason the# buffered flip above does (#3078).@@ -4537,23 +4620,43 @@ class AnthropicHandlerMixin:}ccr_outbound_headers["content-type"] = "application/json"ccr_outbound_headers["accept"] = "application/json"- log_outbound_request(- forwarder="anthropic_ccr_continuation",- method="POST",- path=url,- body_bytes_count=len(ccr_outbound_bytes),- body_mutated=True,- mutation_reasons=["ccr_continuation"],- request_id=request_id,- source=ccr_outbound_source,- )- try:- cont_response = await self.http_client.post(++ async def _post_continuation(+ send_headers: dict[str, str], send_body: dict[str, Any]+ ) -> httpx.Response:+ assert self.http_client is not None+ ccr_outbound_bytes, ccr_outbound_source = (+ prepare_outbound_body_bytes(+ body=send_body,+ original_body_bytes=None,+ body_mutated=True,+ )+ )+ log_outbound_request(+ forwarder="anthropic_ccr_continuation",+ method="POST",+ path=url,+ body_bytes_count=len(ccr_outbound_bytes),+ body_mutated=True,+ mutation_reasons=["ccr_continuation"],+ request_id=request_id,+ source=ccr_outbound_source,+ )+ return await self.http_client.post(url,content=ccr_outbound_bytes,- headers=ccr_outbound_headers,+ headers=send_headers,timeout=self._anthropic_buffered_request_timeout(),)++ try:+ cont_response = await self._send_buffered_as_stream(+ _post_continuation,+ ccr_outbound_headers,+ continuation_body,+ url=url,+ request_id=request_id,+ )logger.info(f"CCR: Got response status={cont_response.status_code}, "f"content-encoding={cont_response.headers.get('content-encoding')}"@@ -4758,12 +4861,17 @@ class AnthropicHandlerMixin:if tools:continuation_body["tools"] = tools- cont_response = await self._retry_request(- "POST",- url,+ cont_response = await self._send_buffered_as_stream(+ partial(+ self._retry_request,+ "POST",+ url,+ timeout=self._anthropic_buffered_request_timeout(),+ ),headers,continuation_body,- timeout=self._anthropic_buffered_request_timeout(),+ url=url,+ request_id=request_id,)# Update response with continuation@@ -4800,12 +4908,17 @@ class AnthropicHandlerMixin:hook_messages: list[dict[str, Any]],) -> dict[str, Any]:continuation_body = {**body, "messages": hook_messages}- continuation_response = await self._retry_request(- "POST",- url,+ continuation_response = await self._send_buffered_as_stream(+ partial(+ self._retry_request,+ "POST",+ url,+ timeout=self._anthropic_buffered_request_timeout(),+ ),headers,continuation_body,- timeout=self._anthropic_buffered_request_timeout(),+ url=url,+ request_id=request_id,)continuation_json = continuation_response.json()_hook_usage.record(continuation_json)diff --git a/headroom/proxy/runtime_env.py b/headroom/proxy/runtime_env.pyindex a3415042e..08110776a 100644--- a/headroom/proxy/runtime_env.py+++ b/headroom/proxy/runtime_env.py@@ -84,6 +84,12 @@ RUNTIME_ENV_KNOBS: tuple[Knob, ...] = ("int","Max on-disk file size (bytes) the disk-verify fallback will read.",),+ Knob(+ "HEADROOM_BUFFERED_UPSTREAM_STREAM",+ "bool",+ "Send buffered CCR requests upstream as stream:true and fold the reply "+ "(default on; 0 sends stream:false).",+ ),)_KNOBS_BY_ENV: dict[str, Knob] = {k.env: k for k in RUNTIME_ENV_KNOBS}diff --git a/tests/test_anthropic_buffered_ccr_usage.py b/tests/test_anthropic_buffered_ccr_usage.pyindex 46569cce4..034b48479 100644--- a/tests/test_anthropic_buffered_ccr_usage.py+++ b/tests/test_anthropic_buffered_ccr_usage.py@@ -1,9 +1,9 @@"""A buffered-CCR turn is a billed call and its usage must reach accounting.When a ``stream:true`` request carries the ``headroom_retrieve`` tool, the-Anthropic handler rewrites it to ``stream:false`` upstream so it can resolve-retrievals server-side, then re-synthesizes SSE for the client. That buffered-response carries the same ``usage`` block any non-stream reply does, so the+Anthropic handler buffers the whole reply (upstream streams it, the proxy folds+the stream into one Message) so it can resolve retrievals server-side, then+re-synthesizes SSE for the client. That buffered response carries the same ``usage`` block any non-stream reply does, so theprovider's cache-read / cache-write / output counts must land on the outcome.If they don't, every cached token on the dominant Claude Code path is invisible:@@ -15,6 +15,7 @@ compression savings on the buffered turns still count in full.from __future__ import annotations+import jsonfrom typing import Anyimport httpx@@ -25,6 +26,7 @@ pytest.importorskip("fastapi")from fastapi.testclient import TestClient # noqa: E402+from headroom.proxy.anthropic_wire import render_anthropic_sse_response # noqa: E402from headroom.proxy.loopback_guard import require_loopback # noqa: E402from headroom.proxy.server import ProxyConfig, create_app # noqa: E402@@ -139,15 +141,19 @@ def test_buffered_ccr_turn_records_provider_usage(monkeypatch, ccr_marker: str)"""A stream:true + headroom_retrieve turn must book the same usage block."""app, outcomes = _app_and_outcomes(monkeypatch)route = respx.post("https://api.anthropic.com/v1/messages").mock(- return_value=httpx.Response(200, json=_RESPONSE)+ return_value=httpx.Response(+ 200,+ content=b"".join(render_anthropic_sse_response(_RESPONSE)),+ headers={"content-type": "text/event-stream"},+ ))r = _post(app, stream=True, tools=[_RETRIEVE_TOOL], marker=ccr_marker)assert r.status_code == 200- # The handler must have buffered it: upstream saw stream:false.+ # The handler buffered it: upstream was asked for the stream it then folded.sent = route.calls.last.request- assert b'"stream": false' in sent.content or b'"stream":false' in sent.content+ assert json.loads(sent.content)["stream"] is Trueassert outcomes, "an outcome must be recorded"o = outcomes[-1]diff --git a/tests/test_buffered_ccr_accept_header.py b/tests/test_buffered_ccr_accept_header.pyindex 3a663cb62..822cbb7be 100644--- a/tests/test_buffered_ccr_accept_header.py+++ b/tests/test_buffered_ccr_accept_header.py@@ -34,6 +34,17 @@ from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402from headroom.proxy.server import ProxyConfig, create_app # noqa: E402[email protected](autouse=True)+def _stream_false_leg(monkeypatch: pytest.MonkeyPatch) -> None:+ """Pin the ``stream: false`` request #3078 is about.++ A first-party buffered turn now asks upstream for a stream and folds it+ (``test_buffered_ccr_upstream_stream.py``). This request is still what a+ custom upstream such as Copilot gets, and what the kill switch restores.+ """+ monkeypatch.setenv("HEADROOM_BUFFERED_UPSTREAM_STREAM", "0")++@pytest.fixture(autouse=True)def _store():reset_compression_store()diff --git a/tests/test_buffered_ccr_upstream_stream.py b/tests/test_buffered_ccr_upstream_stream.pynew file mode 100644index 000000000..acb3c7822--- /dev/null+++ b/tests/test_buffered_ccr_upstream_stream.py@@ -0,0 +1,540 @@+"""The buffered CCR request streams from upstream, and the proxy folds the reply.++The buffered CCR path flips a streaming turn to ``stream: false`` so the whole+reply is in hand before server-side retrieval. Sent that way, upstream sends+nothing until the generation is complete, and a network that drops HTTP+connections idle for ~60s (TLS-inspecting antivirus, VPN, corporate proxy)+cut every long turn: three attempts failed at ~60s each and the client got a+502 it resent forever. The request now goes out as a stream (as do the CCR,+memory and turn-hook continuations built from it) and the event stream is+folded back into the Message the buffered path reads.+"""++from __future__ import annotations++import inspect+import json+from typing import Any++import pytest++fastapi = pytest.importorskip("fastapi")+httpx = pytest.importorskip("httpx")++from fastapi.testclient import TestClient # noqa: E402++from headroom.cache.backends import InMemoryBackend # noqa: E402+from headroom.cache.compression_store import ( # noqa: E402+ get_compression_store,+ reset_compression_store,+)+from headroom.ccr.tool_injection import create_ccr_tool_definition # noqa: E402+from headroom.proxy.anthropic_wire import fold_anthropic_stream # noqa: E402+from headroom.proxy.server import HeadroomProxy, ProxyConfig, create_app # noqa: E402++URL = "https://api.anthropic.com/v1/messages"+++def _sse(*events: dict[str, Any]) -> bytes:+ return b"".join(+ f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events+ )+++def _message_start(model: str = "claude-opus-5") -> dict[str, Any]:+ return {+ "type": "message_start",+ "message": {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": model,+ "content": [],+ "stop_reason": None,+ "stop_sequence": None,+ "usage": {+ "input_tokens": 25,+ "cache_creation_input_tokens": 0,+ "cache_read_input_tokens": 100,+ "output_tokens": 1,+ },+ },+ }+++def _text_turn(text: str) -> list[dict[str, Any]]:+ return [+ _message_start(),+ {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},+ {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},+ {"type": "content_block_stop", "index": 0},+ {+ "type": "message_delta",+ "delta": {"stop_reason": "end_turn", "stop_sequence": None},+ "usage": {"output_tokens": 7},+ },+ {"type": "message_stop"},+ ]+++# A turn exercising every delta the fold accumulates, with the members only a+# stream carries: a thinking block's ``estimated_tokens`` placeholder, a delta's+# progress count, interleaved pings, and a null usage count in message_delta.+EVERY_DELTA = [+ _message_start(),+ {+ "type": "content_block_start",+ "index": 0,+ "content_block": {+ "type": "thinking",+ "thinking": "",+ "signature": "",+ "estimated_tokens": None,+ },+ },+ {"type": "ping"},+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "thinking_delta", "thinking": "Let me ", "estimated_tokens": 3},+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "thinking_delta", "thinking": "check."},+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "signature_delta", "signature": "sig-1"},+ },+ {"type": "content_block_stop", "index": 0},+ {+ "type": "content_block_start",+ "index": 1,+ "content_block": {"type": "redacted_thinking", "data": "opaque"},+ },+ {"type": "content_block_stop", "index": 1},+ {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}},+ {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "Paris"}},+ {+ "type": "content_block_delta",+ "index": 2,+ "delta": {+ "type": "citations_delta",+ "citation": {+ "type": "char_location",+ "cited_text": "Paris",+ "document_index": 0,+ "document_title": "Atlas",+ "start_char_index": 0,+ "end_char_index": 5,+ },+ },+ },+ {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": " it is."}},+ {"type": "content_block_stop", "index": 2},+ {+ "type": "content_block_start",+ "index": 3,+ "content_block": {"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {}},+ },+ {+ "type": "content_block_delta",+ "index": 3,+ "delta": {"type": "input_json_delta", "partial_json": '{"file_pa'},+ },+ {+ "type": "content_block_delta",+ "index": 3,+ "delta": {"type": "input_json_delta", "partial_json": 'th": "a.py"}'},+ },+ {"type": "content_block_stop", "index": 3},+ {"type": "ping"},+ {+ "type": "message_delta",+ "delta": {"stop_reason": "tool_use", "stop_sequence": None},+ "usage": {"output_tokens": 42, "input_tokens": None},+ },+ {"type": "message_stop"},+]+++# --------------------------------------------------------------------------- #+# The fold+# --------------------------------------------------------------------------- #+def test_fold_matches_the_anthropic_sdk_accumulator() -> None:+ streaming = pytest.importorskip("anthropic.lib.streaming._messages")+ accumulate = streaming.accumulate_event+ # anthropic 1.x keeps each tool block's partial JSON in a buffer the caller+ # owns (``json_bufs``); 0.x kept it on the block. Same accumulation either way.+ buffers: dict[str, Any] = (+ {"json_bufs": {}} if "json_bufs" in inspect.signature(accumulate).parameters else {}+ )+ snapshot = None+ for event in EVERY_DELTA:+ snapshot = accumulate(event=event, current_snapshot=snapshot, **buffers)+ assert snapshot is not None++ folded = fold_anthropic_stream(_sse(*EVERY_DELTA))++ assert folded is not None+ # The SDK keeps nulls (``stop_sequence``, the thinking placeholder); the+ # non-streaming Message drops the placeholder, so compare without them.+ assert {k: v for k, v in folded.items() if v is not None} == snapshot.model_dump(+ exclude_none=True+ )+ assert folded["usage"]["input_tokens"] == 25, "a null delta count must not erase it"+++def test_fold_keeps_stream_only_members_out_of_blocks() -> None:+ """The proxy resends folded blocks; Anthropic 400s on any extra member."""+ folded = fold_anthropic_stream(_sse(*EVERY_DELTA))++ assert folded is not None+ assert folded["content"][0] == {+ "type": "thinking",+ "thinking": "Let me check.",+ "signature": "sig-1",+ }+ assert folded["content"][1] == {"type": "redacted_thinking", "data": "opaque"}+ assert "estimated_tokens" not in json.dumps(folded)+++def test_fold_reports_a_truncated_stream() -> None:+ assert fold_anthropic_stream(_sse(*EVERY_DELTA[:-2])) is None+ assert fold_anthropic_stream(_sse(*EVERY_DELTA[:5])) is None # a block left open+++def test_fold_returns_an_in_band_error_event() -> None:+ error = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}++ assert fold_anthropic_stream(_sse(*EVERY_DELTA[:7], error)) == error+++def test_fold_refuses_a_delta_it_does_not_know() -> None:+ unknown = {"type": "content_block_delta", "index": 2, "delta": {"type": "future_delta"}}++ with pytest.raises(ValueError):+ fold_anthropic_stream(_sse(*EVERY_DELTA[:10], unknown, *EVERY_DELTA[10:]))+++# --------------------------------------------------------------------------- #+# The send helper+# --------------------------------------------------------------------------- #+def proxy() -> HeadroomProxy:+ return HeadroomProxy(ProxyConfig(optimize=False, cache_enabled=False, image_optimize=False))+++class _Upstream:+ """Answers a stream:true request with ``stream_reply``, anything else with JSON."""++ def __init__(self, stream_reply: httpx.Response) -> None:+ self.stream_reply = stream_reply+ self.calls: list[tuple[dict[str, str], dict[str, Any]]] = []++ async def __call__(self, headers: dict[str, str], body: dict[str, Any]) -> httpx.Response:+ self.calls.append((dict(headers), json.loads(json.dumps(body))))+ if body.get("stream"):+ return self.stream_reply+ return httpx.Response(200, json={"type": "message", "content": [], "via": "json"})+++def _stream_reply(*events: dict[str, Any]) -> httpx.Response:+ return httpx.Response(200, content=_sse(*events), headers={"content-type": "text/event-stream"})+++async def _send(+ proxy: HeadroomProxy, upstream: _Upstream, *, url: str = URL, body: dict | None = None+) -> httpx.Response:+ return await proxy._send_buffered_as_stream(+ upstream,+ {"Accept": "application/json", "x-api-key": "k"},+ body if body is not None else {"model": "m", "stream": False, "messages": []},+ url=url,+ request_id="t",+ )+++async def test_a_buffered_request_streams_and_folds(proxy) -> None:+ upstream = _Upstream(_stream_reply(*_text_turn("hello")))+ body = {"model": "m", "stream": False, "messages": []}++ response = await _send(proxy, upstream, body=body)++ assert response.status_code == 200+ assert response.json()["content"] == [{"type": "text", "text": "hello"}]+ assert len(upstream.calls) == 1+ headers, sent = upstream.calls[0]+ assert sent["stream"] is True+ assert [v for k, v in headers.items() if k.lower() == "accept"] == ["text/event-stream"]+ assert body["stream"] is False, "the caller's body is not mutated"+++ ("env", "url", "stream"),+ [+ ("0", URL, False), # kill switch+ ("1", "https://gateway.example/v1/messages", False), # not first-party+ ("1", URL, True), # already a stream+ ],+)+async def test_requests_it_should_not_touch_go_out_unchanged(+ proxy, monkeypatch, env: str, url: str, stream: bool+) -> None:+ monkeypatch.setenv("HEADROOM_BUFFERED_UPSTREAM_STREAM", env)+ upstream = _Upstream(_stream_reply(*_text_turn("hello")))+ body = {"model": "m", "stream": stream, "messages": []}++ await _send(proxy, upstream, url=url, body=body)++ assert upstream.calls == [({"Accept": "application/json", "x-api-key": "k"}, body)]+++async def test_a_propagated_kill_switch_keeps_the_request_non_streaming(proxy) -> None:+ # `headroom wrap` pushes this knob to a reused proxy, so the rollback works+ # without the proxy's own environment or a restart.+ from headroom.proxy import runtime_env++ assert runtime_env.explicit_env({"HEADROOM_BUFFERED_UPSTREAM_STREAM": "0"}) == {+ "HEADROOM_BUFFERED_UPSTREAM_STREAM": "0"+ }+ upstream = _Upstream(_stream_reply(*_text_turn("hello")))+ body = {"model": "m", "stream": False, "messages": []}+ runtime_env.set_overrides({"HEADROOM_BUFFERED_UPSTREAM_STREAM": "0"})+ try:+ response = await _send(proxy, upstream, body=body)+ finally:+ runtime_env.clear_overrides()++ assert upstream.calls == [({"Accept": "application/json", "x-api-key": "k"}, body)]+ assert response.json()["via"] == "json"+++ "reply",+ [+ httpx.Response(529, json={"type": "error", "error": {"type": "overloaded_error"}}),+ httpx.Response(200, json={"type": "message", "content": []}), # answered JSON anyway+ ],+)+async def test_a_reply_that_is_not_an_event_stream_is_returned_as_is(proxy, reply) -> None:+ upstream = _Upstream(reply)++ assert await _send(proxy, upstream) is reply+++async def test_an_in_band_error_comes_back_with_its_status(proxy) -> None:+ error = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}+ upstream = _Upstream(_stream_reply(_message_start(), error))++ response = await _send(proxy, upstream)++ assert response.status_code == 529+ assert response.json() == error+++async def test_a_truncated_stream_is_a_502(proxy) -> None:+ upstream = _Upstream(_stream_reply(*_text_turn("hello")[:-2]))++ response = await _send(proxy, upstream)++ assert response.status_code == 502+ assert response.json()["error"]["type"] == "upstream_protocol_error"+ assert len(upstream.calls) == 1+++async def test_a_stream_it_cannot_fold_is_sent_again_as_stream_false(proxy) -> None:+ unknown = {"type": "content_block_delta", "index": 0, "delta": {"type": "future_delta"}}+ events = _text_turn("hello")+ upstream = _Upstream(_stream_reply(*events[:3], unknown, *events[3:]))++ response = await _send(proxy, upstream)++ assert response.json()["via"] == "json"+ assert [sent["stream"] for _, sent in upstream.calls] == [True, False]+ assert upstream.calls[1][0]["Accept"] == "application/json"+++# --------------------------------------------------------------------------- #+# Through the handler+# --------------------------------------------------------------------------- #[email protected](autouse=True)+def _store():+ reset_compression_store()+ get_compression_store(backend=InMemoryBackend())+ try:+ yield+ finally:+ reset_compression_store()+++def _config() -> ProxyConfig:+ return ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ memory_enabled=False,+ ccr_inject_tool=True,+ ccr_handle_responses=True,+ ccr_context_tracking=False,+ image_optimize=False,+ )+++class _ContinuationClient:+ """Stands in for ``http_client``; the CCR continuation posts through it."""++ def __init__(self, reply: httpx.Response) -> None:+ self.reply = reply+ self.calls: list[tuple[dict[str, str], dict[str, Any]]] = []++ async def post(self, url, *, content=None, headers=None, timeout=None): # noqa: ANN001+ self.calls.append((dict(headers or {}), json.loads(content)))+ return self.reply++ async def aclose(self) -> None:+ return None+++def _accept(headers: dict[str, str]) -> list[str]:+ return [v for k, v in headers.items() if k.lower() == "accept"]+++def test_the_buffered_turn_and_its_ccr_continuation_stream_upstream() -> None:+ store = get_compression_store()+ marker = store.store(+ original=json.dumps({"secret": "retrieved answer"}),+ compressed="{}",+ original_item_count=1,+ )+ retrieve_turn = [+ _message_start(),+ {+ "type": "content_block_start",+ "index": 0,+ "content_block": {+ "type": "thinking",+ "thinking": "",+ "signature": "",+ "estimated_tokens": None,+ },+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "thinking_delta", "thinking": "I need it."},+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "signature_delta", "signature": "sig"},+ },+ {"type": "content_block_stop", "index": 0},+ {+ "type": "content_block_start",+ "index": 1,+ "content_block": {+ "type": "tool_use",+ "id": "toolu_ccr",+ "name": "headroom_retrieve",+ "input": {},+ },+ },+ {+ "type": "content_block_delta",+ "index": 1,+ "delta": {"type": "input_json_delta", "partial_json": json.dumps({"hash": marker})},+ },+ {"type": "content_block_stop", "index": 1},+ {+ "type": "message_delta",+ "delta": {"stop_reason": "tool_use", "stop_sequence": None},+ "usage": {"output_tokens": 9},+ },+ {"type": "message_stop"},+ ]+ sent: list[tuple[dict[str, str], dict[str, Any]]] = []++ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ sent.append((dict(headers), json.loads(json.dumps(body))))+ return _stream_reply(*retrieve_turn)++ proxy._retry_request = _fake_retry # type: ignore[assignment]+ continuation = _ContinuationClient(_stream_reply(*_text_turn("the answer")))+ proxy.http_client = continuation++ resp = client.post(+ "/v1/messages",+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-opus-5",+ "max_tokens": 64,+ "stream": True,+ "tools": [create_ccr_tool_definition("anthropic")],+ "messages": [{"role": "user", "content": f"use <<ccr:{marker}>>"}],+ },+ )++ assert resp.status_code == 200, resp.text+ assert "the answer" in resp.text+ assert "headroom_retrieve" not in resp.text+ [(headers, body)] = sent+ assert body["stream"] is True+ assert _accept(headers) == ["text/event-stream"]+ [(cont_headers, cont_body)] = continuation.calls+ assert cont_body["stream"] is True+ assert _accept(cont_headers) == ["text/event-stream"]+ # The folded turn is resent as history: no stream-only member may ride along.+ assert cont_body["messages"][-2]["content"][0] == {+ "type": "thinking",+ "thinking": "I need it.",+ "signature": "sig",+ }+++def test_a_client_stream_false_request_is_left_alone() -> None:+ marker = get_compression_store().store(+ original=json.dumps({"earlier": "tool output"}),+ compressed="{}",+ original_item_count=1,+ )+ sent: list[tuple[dict[str, str], dict[str, Any], dict[str, Any]]] = []+ app = create_app(_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ sent.append((dict(headers), json.loads(json.dumps(body)), kwargs))+ return httpx.Response(200, json=fold_anthropic_stream(_sse(*_text_turn("ok"))))++ proxy._retry_request = _fake_retry # type: ignore[assignment]+ resp = client.post(+ "/v1/messages",+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-opus-5",+ "max_tokens": 64,+ "stream": False,+ "tools": [create_ccr_tool_definition("anthropic")],+ "messages": [{"role": "user", "content": f"use <<ccr:{marker}>>"}],+ },+ )++ assert resp.status_code == 200, resp.text+ [(headers, body, kwargs)] = sent+ assert body["stream"] is False+ assert _accept(headers) == ["application/json"]+ assert kwargs["original_body_bytes"] is not Nonediff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.pyindex 8130d0145..82a2fbe52 100644--- a/tests/test_ccr_buffered_stream_signed_thinking.py+++ b/tests/test_ccr_buffered_stream_signed_thinking.py@@ -175,7 +175,8 @@ def test_signed_thinking_history_skips_the_buffered_ccr_path(assert calls["stream_body"]["stream"] is Trueelse:assert "buffered_body" in calls, "expected the buffered CCR path"- assert calls["buffered_body"]["stream"] is False+ # The buffered request itself streams; the proxy folds the reply.+ assert calls["buffered_body"]["stream"] is True@pytest.mark.parametrize("upstream_delay", [0.0, 1.2], ids=["prompt", "past-keepalive"])@@ -278,7 +279,8 @@ def test_buffering_is_gated_on_a_redeemable_marker(assert resp.status_code == 200, resp.textif expect_buffered:assert "buffered_body" in calls, "a redeemable marker must still buffer"- assert calls["buffered_body"]["stream"] is False+ # The buffered request itself streams; the proxy folds the reply.+ assert calls["buffered_body"]["stream"] is Trueelse:assert "stream_body" in calls, "nothing to retrieve — the client must keep streaming"assert "buffered_body" not in callsdiff --git a/tests/test_no_ccr_disables_response_handling.py b/tests/test_no_ccr_disables_response_handling.pyindex 5a1947b8a..2f71fcab6 100644--- a/tests/test_no_ccr_disables_response_handling.py+++ b/tests/test_no_ccr_disables_response_handling.py@@ -156,9 +156,10 @@ def _upstream_stream_field(*, ccr_handle_responses: bool) -> object:def test_client_offered_tool_still_buffers_when_handling_is_on(_store) -> None: # noqa: ANN001"""The behaviour being switched off — pinned so the switch is meaningful."""- assert _upstream_stream_field(ccr_handle_responses=True) is False+ # Only the buffered path sends through `_retry_request`, and its request streams.+ assert _upstream_stream_field(ccr_handle_responses=True) is Truedef test_client_offered_tool_does_not_buffer_under_no_ccr(_store) -> None: # noqa: ANN001"""With the switch honoured, the turn keeps streaming despite the tool."""- assert _upstream_stream_field(ccr_handle_responses=False) is not False+ assert _upstream_stream_field(ccr_handle_responses=False) is Nonediff --git a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.pyindex 2bf8110e5..274c4ba92 100644--- a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py+++ b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py@@ -141,7 +141,7 @@ def test_streaming_headroom_retrieve_is_intercepted_and_returned_as_sse() -> Nonasync def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001initial_bodies.append(json.loads(json.dumps(body)))assert stream is False- assert body["stream"] is False+ assert body["stream"] is Truereturn httpx.Response(200, json=initial_response)proxy._retry_request = _fake_retry # type: ignore[assignment]@@ -167,10 +167,10 @@ def test_streaming_headroom_retrieve_is_intercepted_and_returned_as_sse() -> Nonassert "text/event-stream" in resp.headers["content-type"]assert "retrieved answer is now available" in resp.textassert "headroom_retrieve" not in resp.text- assert initial_bodies and initial_bodies[0]["stream"] is False+ assert initial_bodies and initial_bodies[0]["stream"] is Trueassert len(continuation_client.post_calls) == 1continuation_body = json.loads(continuation_client.post_calls[0]["content"].decode())- assert continuation_body["stream"] is False+ assert continuation_body["stream"] is Truecontinuation_headers = {key.lower(): value for key, value in continuation_client.post_calls[0]["headers"].items()}@@ -232,7 +232,7 @@ def test_streaming_with_headroom_retrieve_available_but_unused_returns_sse() ->async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001initial_bodies.append(json.loads(json.dumps(body)))assert stream is False- assert body["stream"] is False+ assert body["stream"] is Truereturn httpx.Response(200, json=text_response)proxy._retry_request = _fake_retry # type: ignore[assignment]@@ -253,7 +253,7 @@ def test_streaming_with_headroom_retrieve_available_but_unused_returns_sse() ->assert "text/event-stream" in resp.headers["content-type"]assert "plain answer" in resp.textassert "headroom_retrieve" not in resp.text- assert initial_bodies and initial_bodies[0]["stream"] is False+ assert initial_bodies and initial_bodies[0]["stream"] is Trueassert continuation_client.post_calls == []proxy._stream_response.assert_not_awaited()@@ -311,7 +311,7 @@ def test_mixed_ccr_and_client_tool_serves_retrieval_then_streams_reissued_tool()proxy.http_client = continuation_clientasync def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001- assert body["stream"] is False+ assert body["stream"] is Truereturn httpx.Response(200, json=initial_response)proxy._retry_request = _fake_retry # type: ignore[assignment]@@ -393,7 +393,7 @@ def test_mixed_turn_streams_the_continuation_or_the_turn_on_failure(status_code:proxy.http_client = continuation_clientasync def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001- assert body["stream"] is False+ assert body["stream"] is Truereturn httpx.Response(200, json=initial_response)proxy._retry_request = _fake_retry # type: ignore[assignment]diff --git a/tests/test_proxy_response_cache_replay.py b/tests/test_proxy_response_cache_replay.pyindex 40b6b249b..6ba184dd2 100644--- a/tests/test_proxy_response_cache_replay.py+++ b/tests/test_proxy_response_cache_replay.py@@ -295,7 +295,8 @@ def test_buffered_ccr_turn_does_not_write_the_response_cache():assert resp.status_code == 200, resp.text# The conversion really happened — otherwise this test proves nothing.- assert forwarded_bodies and forwarded_bodies[0]["stream"] is False+ # Buffered: the request streams upstream and the proxy folds the reply.+ assert forwarded_bodies and forwarded_bodies[0]["stream"] is True# ...and nothing was written to the response cache.proxy.cache.set.assert_not_awaited()reset_compression_store()