Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): only queue mid-turn messages for opt-in clients with explicit session header (#1951)

wzy-delc365c7ff81dd
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 9ee4878a1..d02d09afc 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -2558,11 +2558,16 @@ class AnthropicHandlerMixin:
metadata={"path": pipeline_path, "stream": True},
)
await _finalize_pre_upstream()
+ explicit_session_header = request.headers.get("x-headroom-session-id")
session_key = self._get_session_key(
body,
- session_header=request.headers.get("x-headroom-session-id"),
+ session_header=explicit_session_header,
)
- if session_key in self._active_streams:
+ # Only opt-in (header-bearing) callers participate in
+ # mid-turn steering; see StreamingMixin._should_queue_mid_turn
+ # for why the coarse md5 fallback must not queue concurrent
+ # independent streams (it wrongly 202s a streaming caller).
+ if self._should_queue_mid_turn(session_key, explicit_session_header):
from fastapi.responses import JSONResponse
queued = self._queue_mid_turn_message(session_key, body)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 6b291836a..9da56dbb0 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -95,6 +95,27 @@ class StreamingMixin:
self._mid_turn_queues[session_key].put_nowait(body)
return {"status": 202, "event": "headroom_queued"}
+ def _should_queue_mid_turn(self, session_key: str, explicit_session_header: str | None) -> bool:
+ """Return True only when a follow-up should be queued as a mid-turn message.
+
+ Mid-turn steering is a private Headroom protocol: a queued message is
+ only ever drained back to the client via the custom
+ ``headroom_pending_messages`` SSE event, which a standard Anthropic SDK
+ does not understand. We therefore only engage it for clients that
+ *opt in* by sending an explicit ``x-headroom-session-id`` header.
+
+ Without that header the session identity falls back to
+ ``md5(model + system[:500])`` (see ``_get_session_key`` /
+ ``prefix_tracker.compute_session_id``). That fallback is intentionally
+ coarse and cannot distinguish genuinely concurrent, independent streams
+ that happen to share a model + system prompt (e.g. a main conversation
+ plus its background/parallel requests). Queuing those as mid-turn
+ messages wrongly returns a 202 + JSON body to a caller that issued a
+ *streaming* request, whose stream parser then sees an empty (non-SSE)
+ stream and fails. So only opt-in (header-bearing) callers get queued.
+ """
+ return bool(explicit_session_header) and session_key in self._active_streams
+
def _cleanup_mid_turn_stream(
self, session_key: str, *, drain_pending_messages: bool = False
) -> list[dict]:
diff --git a/tests/test_mid_turn_steering.py b/tests/test_mid_turn_steering.py
index fae01a5c9..1776190a6 100644
--- a/tests/test_mid_turn_steering.py
+++ b/tests/test_mid_turn_steering.py
@@ -42,6 +42,34 @@ class TestMidTurnSteering:
assert session_key not in mixin._active_streams
assert session_key not in mixin._mid_turn_queues
+ def test_should_queue_only_with_explicit_session_header(self):
+ """Regression: mid-turn queuing must require an explicit session header.
+
+ Without ``x-headroom-session-id`` the session key is a coarse
+ ``md5(model + system[:500])`` shared by concurrent independent streams
+ (e.g. a main conversation plus background/parallel requests). Queuing
+ those wrongly returns a 202 to a streaming caller, whose SDK stream
+ parser then fails on an empty (non-SSE) stream. Only opt-in callers
+ that send the header may be queued.
+ """
+ from headroom.proxy.handlers.streaming import StreamingMixin
+
+ mixin = StreamingMixin()
+ session_key = "shared-md5-key"
+ # An earlier stream on this (fallback) session key is in flight.
+ mixin._active_streams.add(session_key)
+ try:
+ # No explicit header (header-less concurrent stream): must NOT queue,
+ # even though the key collides in _active_streams.
+ assert mixin._should_queue_mid_turn(session_key, None) is False
+ assert mixin._should_queue_mid_turn(session_key, "") is False
+ # Explicit header present: opt-in client, queuing is allowed.
+ assert mixin._should_queue_mid_turn(session_key, session_key) is True
+ # Explicit header but no active stream: nothing to queue behind.
+ assert mixin._should_queue_mid_turn("other-key", "other-key") is False
+ finally:
+ mixin._active_streams.discard(session_key)
+
def _create_mock_proxy(self):
proxy = object.__new__(HeadroomProxy)
proxy.http_client = MagicMock(spec=httpx.AsyncClient)