Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): only queue mid-turn messages for opt-in clients with explicit session header (#1951)
wzy-delc365c7ff81dd
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 9ee4878a1..d02d09afc 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -2558,11 +2558,16 @@ class AnthropicHandlerMixin:metadata={"path": pipeline_path, "stream": True},)await _finalize_pre_upstream()+ explicit_session_header = request.headers.get("x-headroom-session-id")session_key = self._get_session_key(body,- session_header=request.headers.get("x-headroom-session-id"),+ session_header=explicit_session_header,)- if session_key in self._active_streams:+ # Only opt-in (header-bearing) callers participate in+ # mid-turn steering; see StreamingMixin._should_queue_mid_turn+ # for why the coarse md5 fallback must not queue concurrent+ # independent streams (it wrongly 202s a streaming caller).+ if self._should_queue_mid_turn(session_key, explicit_session_header):from fastapi.responses import JSONResponsequeued = self._queue_mid_turn_message(session_key, body)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 6b291836a..9da56dbb0 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -95,6 +95,27 @@ class StreamingMixin:self._mid_turn_queues[session_key].put_nowait(body)return {"status": 202, "event": "headroom_queued"}+ def _should_queue_mid_turn(self, session_key: str, explicit_session_header: str | None) -> bool:+ """Return True only when a follow-up should be queued as a mid-turn message.++ Mid-turn steering is a private Headroom protocol: a queued message is+ only ever drained back to the client via the custom+ ``headroom_pending_messages`` SSE event, which a standard Anthropic SDK+ does not understand. We therefore only engage it for clients that+ *opt in* by sending an explicit ``x-headroom-session-id`` header.++ Without that header the session identity falls back to+ ``md5(model + system[:500])`` (see ``_get_session_key`` /+ ``prefix_tracker.compute_session_id``). That fallback is intentionally+ coarse and cannot distinguish genuinely concurrent, independent streams+ that happen to share a model + system prompt (e.g. a main conversation+ plus its background/parallel requests). Queuing those as mid-turn+ messages wrongly returns a 202 + JSON body to a caller that issued a+ *streaming* request, whose stream parser then sees an empty (non-SSE)+ stream and fails. So only opt-in (header-bearing) callers get queued.+ """+ return bool(explicit_session_header) and session_key in self._active_streams+def _cleanup_mid_turn_stream(self, session_key: str, *, drain_pending_messages: bool = False) -> list[dict]:diff --git a/tests/test_mid_turn_steering.py b/tests/test_mid_turn_steering.pyindex fae01a5c9..1776190a6 100644--- a/tests/test_mid_turn_steering.py+++ b/tests/test_mid_turn_steering.py@@ -42,6 +42,34 @@ class TestMidTurnSteering:assert session_key not in mixin._active_streamsassert session_key not in mixin._mid_turn_queues+ def test_should_queue_only_with_explicit_session_header(self):+ """Regression: mid-turn queuing must require an explicit session header.++ Without ``x-headroom-session-id`` the session key is a coarse+ ``md5(model + system[:500])`` shared by concurrent independent streams+ (e.g. a main conversation plus background/parallel requests). Queuing+ those wrongly returns a 202 to a streaming caller, whose SDK stream+ parser then fails on an empty (non-SSE) stream. Only opt-in callers+ that send the header may be queued.+ """+ from headroom.proxy.handlers.streaming import StreamingMixin++ mixin = StreamingMixin()+ session_key = "shared-md5-key"+ # An earlier stream on this (fallback) session key is in flight.+ mixin._active_streams.add(session_key)+ try:+ # No explicit header (header-less concurrent stream): must NOT queue,+ # even though the key collides in _active_streams.+ assert mixin._should_queue_mid_turn(session_key, None) is False+ assert mixin._should_queue_mid_turn(session_key, "") is False+ # Explicit header present: opt-in client, queuing is allowed.+ assert mixin._should_queue_mid_turn(session_key, session_key) is True+ # Explicit header but no active stream: nothing to queue behind.+ assert mixin._should_queue_mid_turn("other-key", "other-key") is False+ finally:+ mixin._active_streams.discard(session_key)+def _create_mock_proxy(self):proxy = object.__new__(HeadroomProxy)proxy.http_client = MagicMock(spec=httpx.AsyncClient)