Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(client): preserve Anthropic create stream flags (#4500)

Rudy Mizrahi Celekli3910caabd683
diff --git a/examples/anthropic_create_stream_local.py b/examples/anthropic_create_stream_local.py
new file mode 100644
index 000000000..e0eb1d99a
--- /dev/null
+++ b/examples/anthropic_create_stream_local.py
@@ -0,0 +1,217 @@
+"""Actual SDK + loopback HTTP proof; no remote provider or model calls."""
+
+import json
+import os
+import tempfile
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+
+def main():
+ scratch = tempfile.TemporaryDirectory(prefix="headroom-client-proof-")
+ for key in ["HEADROOM_WORKSPACE_DIR", "HEADROOM_CONFIG_DIR", "XDG_CACHE_HOME"]:
+ os.environ[key] = str(Path(scratch.name) / key.lower())
+ os.environ.update(HEADROOM_OFFLINE="1", HEADROOM_BEACON="off", DO_NOT_TRACK="1")
+
+ import httpx
+ from anthropic import Anthropic
+
+ from headroom import AnthropicProvider, HeadroomClient
+ from headroom.config import HeadroomConfig
+
+ requests = []
+ observed_styles = []
+
+ class Observer:
+ def on_pipeline_event(self, event):
+ if "api_style" in event.metadata:
+ observed_styles.append(event.metadata["api_style"])
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def do_POST(self):
+ data = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ requests.append(data)
+ if data.get("stream"):
+ events = [
+ (
+ "message_start",
+ {
+ "type": "message_start",
+ "message": {
+ "id": "msg_fixture",
+ "type": "message",
+ "role": "assistant",
+ "model": data["model"],
+ "content": [],
+ "stop_reason": None,
+ "stop_sequence": None,
+ "usage": {"input_tokens": 1, "output_tokens": 0},
+ },
+ },
+ ),
+ (
+ "content_block_start",
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {"type": "text", "text": ""},
+ },
+ ),
+ (
+ "content_block_delta",
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "text_delta", "text": "local fixture"},
+ },
+ ),
+ ("content_block_stop", {"type": "content_block_stop", "index": 0}),
+ (
+ "message_delta",
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+ "usage": {"output_tokens": 2},
+ },
+ ),
+ ("message_stop", {"type": "message_stop"}),
+ ]
+ payload = "".join(
+ f"event: {name}\ndata: {json.dumps(event)}\n\n" for name, event in events
+ ).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream")
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+ return
+ payload = json.dumps(
+ {
+ "id": "msg_fixture",
+ "type": "message",
+ "role": "assistant",
+ "model": data["model"],
+ "content": [{"type": "text", "text": "local fixture"}],
+ "stop_reason": "end_turn",
+ "stop_sequence": None,
+ "usage": {"input_tokens": 1, "output_tokens": 2},
+ }
+ ).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ responses = []
+ http = httpx.Client(event_hooks={"response": [responses.append]})
+ original = Anthropic(
+ http_client=http,
+ api_key="local-fixture",
+ base_url=f"http://127.0.0.1:{server.server_port}",
+ max_retries=0,
+ )
+ wrapped = HeadroomClient(
+ original,
+ AnthropicProvider(warn=False),
+ store_url=f"sqlite:///{scratch.name}/metrics.db",
+ enable_cache_optimizer=False,
+ config=HeadroomConfig(
+ store_url=f"sqlite:///{scratch.name}/metrics.db",
+ discover_pipeline_extensions=False,
+ pipeline_extensions=[Observer()],
+ ),
+ )
+ kwargs = {
+ "model": "claude-fixture",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 8,
+ "stream": False,
+ }
+ try:
+ native = original.messages.create(**kwargs)
+ print(
+ json.dumps(
+ {"case": "native", "text": native.content[0].text, "requests": len(requests)}
+ )
+ )
+ try:
+ result = wrapped.messages.create(**kwargs)
+ print(
+ json.dumps(
+ {"case": "wrapped", "text": result.content[0].text, "requests": len(requests)}
+ )
+ )
+ kwargs["stream"] = True
+ native_stream = original.messages.create(**kwargs)
+ native_types = [event.type for event in native_stream]
+ native_stream.close()
+ wrapped_stream = wrapped.messages.create(**kwargs)
+ wrapped_types = [event.type for event in wrapped_stream]
+ wrapped_stream.close()
+ assert native_types == wrapped_types
+ print(
+ json.dumps(
+ {
+ "case": "create_stream",
+ "native_types": native_types,
+ "wrapped_types": wrapped_types,
+ "native_closed": native_stream.response.is_closed,
+ "requests": len(requests),
+ }
+ )
+ )
+ del kwargs["stream"]
+ with wrapped.messages.stream(**kwargs) as manager:
+ text = "".join(manager.text_stream)
+ assert text == "local fixture"
+ assert manager.response.is_closed
+ assert all(response.is_closed for response in responses)
+ assert observed_styles and set(observed_styles) == {"anthropic"}
+ print(
+ json.dumps(
+ {
+ "case": "resources",
+ "http_responses": len(responses),
+ "all_closed": all(response.is_closed for response in responses),
+ }
+ )
+ )
+ print(
+ json.dumps(
+ {
+ "case": "stream_manager",
+ "text": text,
+ "response_closed": manager.response.is_closed,
+ "requests": len(requests),
+ }
+ )
+ )
+ except Exception as exc:
+ print(
+ json.dumps(
+ {
+ "case": "wrapped",
+ "error": type(exc).__name__,
+ "message": str(exc),
+ "requests": len(requests),
+ }
+ )
+ )
+ raise
+ finally:
+ wrapped.close()
+ original.close()
+ server.shutdown()
+ server.server_close()
+ scratch.cleanup()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/headroom/client.py b/headroom/client.py
index 81b58cc63..91088efac 100644
--- a/headroom/client.py
+++ b/headroom/client.py
@@ -202,6 +202,7 @@ class Messages:
model: str,
messages: list[dict[str, Any]],
max_tokens: int = 1024,
+ stream: bool = False,
# Headroom-specific parameters
headroom_mode: str | None = None,
headroom_cache_prefix_tokens: int | None = None,
@@ -218,6 +219,7 @@ class Messages:
model: Model name.
messages: List of messages.
max_tokens: Maximum tokens in response.
+ stream: Return an iterator of raw message events when True.
headroom_mode: Override default mode ("audit" | "optimize").
headroom_cache_prefix_tokens: Target cache-aligned prefix size.
headroom_output_buffer_tokens: Reserve tokens for output.
@@ -231,13 +233,13 @@ class Messages:
return self._client._create(
model=model,
messages=messages,
- stream=False,
+ stream=stream,
headroom_mode=headroom_mode,
headroom_cache_prefix_tokens=headroom_cache_prefix_tokens,
headroom_output_buffer_tokens=headroom_output_buffer_tokens,
headroom_keep_turns=headroom_keep_turns,
headroom_tool_profiles=headroom_tool_profiles,
- api_style="anthropic",
+ api_style="anthropic_create",
max_tokens=max_tokens,
**kwargs,
)
@@ -474,6 +476,9 @@ class HeadroomClient:
**kwargs: Any,
) -> Any:
"""Internal implementation of create."""
+ transport_style = api_style
+ if api_style == "anthropic_create":
+ api_style = "anthropic"
request_id = generate_request_id()
timestamp = datetime.now(timezone.utc).replace(tzinfo=None)
mode = HeadroomMode(headroom_mode) if headroom_mode else self._default_mode
@@ -675,7 +680,7 @@ class HeadroomClient:
# Call underlying client based on API style
try:
response = call_client_transport(
- api_style,
+ transport_style,
self,
model=model,
messages=optimized_messages,
diff --git a/headroom/providers/registry.py b/headroom/providers/registry.py
index 96c19d025..10a6374c5 100644
--- a/headroom/providers/registry.py
+++ b/headroom/providers/registry.py
@@ -373,6 +373,34 @@ def _call_openai_transport(
return response
+def _call_anthropic_create_transport(
+ client: Any,
+ *,
+ model: str,
+ messages: list[dict[str, Any]],
+ stream: bool,
+ metrics: Any,
+ **kwargs: Any,
+) -> Any:
+ if stream:
+ response = client._original.messages.create(
+ model=model,
+ messages=messages,
+ stream=True,
+ **kwargs,
+ )
+ return client._wrap_stream(response, metrics)
+
+ return _call_anthropic_transport(
+ client,
+ model=model,
+ messages=messages,
+ stream=False,
+ metrics=metrics,
+ **kwargs,
+ )
+
+
def _call_anthropic_transport(
client: Any,
*,
@@ -409,6 +437,7 @@ def _call_anthropic_transport(
_ClientTransport = Callable[..., Any]
_CLIENT_TRANSPORTS: dict[str, _ClientTransport] = {
"anthropic": _call_anthropic_transport,
+ "anthropic_create": _call_anthropic_create_transport,
"openai": _call_openai_transport,
}
diff --git a/tests/test_anthropic_client_create_stream.py b/tests/test_anthropic_client_create_stream.py
new file mode 100644
index 000000000..0fe5418bb
--- /dev/null
+++ b/tests/test_anthropic_client_create_stream.py
@@ -0,0 +1,67 @@
+"""The SDK wrapper preserves Anthropic create/stream entrypoint contracts."""
+
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from headroom.client import Messages
+from headroom.providers.registry import call_client_transport
+
+
[email protected]("stream", [False, True])
+def test_messages_create_consumes_explicit_stream_flag(stream):
+ client = SimpleNamespace(_create=Mock(return_value="response"))
+ assert Messages(client).create(model="claude", messages=[], stream=stream) == "response"
+ assert client._create.call_args.kwargs["stream"] is stream
+ assert client._create.call_args.kwargs["api_style"] == "anthropic_create"
+
+
+def test_create_stream_uses_native_create_iterator_not_stream_manager():
+ events = iter([{"type": "message_start"}, {"type": "message_stop"}])
+ original = SimpleNamespace(
+ messages=SimpleNamespace(create=Mock(return_value=events), stream=Mock())
+ )
+ storage = SimpleNamespace(save=Mock())
+ client = SimpleNamespace(
+ _original=original,
+ _storage=storage,
+ _wrap_stream=Mock(side_effect=lambda value, metrics: value),
+ )
+ metrics = SimpleNamespace(tokens_output=0, cached_tokens=0)
+
+ result = call_client_transport(
+ "anthropic_create",
+ client,
+ model="claude",
+ messages=[],
+ stream=True,
+ metrics=metrics,
+ max_tokens=8,
+ )
+
+ assert list(result) == [{"type": "message_start"}, {"type": "message_stop"}]
+ original.messages.create.assert_called_once_with(
+ model="claude", messages=[], stream=True, max_tokens=8
+ )
+ original.messages.stream.assert_not_called()
+ client._wrap_stream.assert_called_once_with(events, metrics)
+
+
+def test_messages_stream_keeps_native_context_manager():
+ manager = object()
+ original = SimpleNamespace(
+ messages=SimpleNamespace(create=Mock(), stream=Mock(return_value=manager))
+ )
+ storage = SimpleNamespace(save=Mock())
+ client = SimpleNamespace(_original=original, _storage=storage)
+ metrics = SimpleNamespace(tokens_output=0, cached_tokens=0)
+
+ result = call_client_transport(
+ "anthropic", client, model="claude", messages=[], stream=True, metrics=metrics, max_tokens=8
+ )
+
+ assert result is manager
+ original.messages.stream.assert_called_once_with(model="claude", messages=[], max_tokens=8)
+ original.messages.create.assert_not_called()
+ storage.save.assert_called_once_with(metrics)