Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(litellm): preserve Anthropic thinking blocks; fix OpenAI-in reasoning passthrough (#3586)

Tejas Chopra9f32800b86ad
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 0297474d4..70e9f4d49 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -35,6 +35,14 @@ _OPENAI_STANDARD_PARAMS = (
"response_format",
"seed",
"n",
+ # Reasoning / token dials. These MUST be forwarded as top-level litellm
+ # kwargs, not swept into `extra_body`: litellm maps them per provider
+ # (e.g. reasoning_effort -> Anthropic thinking config), whereas anything in
+ # `extra_body` is shipped to the provider verbatim and a non-OpenAI target
+ # like Claude 400s on the unknown field. Without this, an OpenAI-in
+ # reasoning request routed cross-protocol to a Claude model fails.
+ "reasoning_effort",
+ "max_completion_tokens",
)
_OPENAI_CONSUMED_BODY_KEYS = frozenset(
@@ -580,6 +588,96 @@ def _parse_tool_arguments(arguments: Any) -> Any:
return arguments
+def _is_anthropic_family_model(litellm_model: str) -> bool:
+ """True when the resolved litellm target speaks the Anthropic Messages
+ dialect: the Anthropic API, Bedrock-Claude, Vertex-Claude, Azure-Claude.
+
+ Only these accept — and on a tool-use continuation *require* — the prior
+ assistant turn's signed ``thinking`` blocks echoed back verbatim. Any other
+ target (OpenAI, DeepSeek, ...) must instead have thinking stripped, because
+ litellm forwards a stray ``thinking_blocks``/``reasoning_content`` field to
+ them unchanged and they reject the unknown key. Keyed on the model id, not
+ ``self.provider``, so a Bedrock/Vertex backend serving a Claude profile is
+ recognised while the same backend serving a non-Claude model is not.
+
+ Known limitation: a substring test on the resolved id. An OPAQUE Bedrock
+ application-inference-profile ARN naming neither "claude" nor "anthropic"
+ reads as non-family, so thinking is stripped even though the target is
+ Claude; operators using such an ARN should map it to an id carrying the
+ model name. Conversely a non-Claude model whose id happens to contain
+ "claude" would be treated as family.
+ """
+ m = (litellm_model or "").lower()
+ return "claude" in m or "anthropic" in m
+
+
+def _thinking_block_to_dict(block: Any) -> dict[str, Any]:
+ """Normalise a litellm thinking block (dict or pydantic) to a plain dict."""
+ if isinstance(block, dict):
+ return block
+ if hasattr(block, "model_dump"):
+ try:
+ dumped = block.model_dump(exclude_none=True)
+ if isinstance(dumped, dict):
+ return dumped
+ except Exception: # noqa: BLE001 - fall through to attribute scrape
+ pass
+ return {
+ k: getattr(block, k)
+ for k in ("type", "thinking", "signature", "data")
+ if getattr(block, k, None) is not None
+ }
+
+
+def _extract_thinking_content_blocks(message: Any) -> list[dict[str, Any]]:
+ """Anthropic-shaped ``thinking``/``redacted_thinking`` content blocks to
+ prepend to a rebuilt Anthropic response, taken from litellm's
+ ``message.thinking_blocks``.
+
+ Only blocks that can be legally replayed are emitted: a ``thinking`` block
+ is kept only if it carries a real ``signature`` (Anthropic validates it
+ cryptographically and 400s an unsigned block on the next turn), and a
+ ``redacted_thinking`` block only if it carries its opaque ``data``. Bare,
+ signatureless reasoning text is dropped rather than turned into a poison
+ block the client cannot send back.
+ """
+ raw = getattr(message, "thinking_blocks", None) or []
+ out: list[dict[str, Any]] = []
+ for block in raw:
+ bd = _thinking_block_to_dict(block)
+ btype = bd.get("type")
+ if btype == "redacted_thinking":
+ if bd.get("data"):
+ out.append({"type": "redacted_thinking", "data": bd["data"]})
+ elif btype in (None, "thinking"):
+ signature = bd.get("signature")
+ if signature:
+ out.append(
+ {
+ "type": "thinking",
+ "thinking": bd.get("thinking") or "",
+ "signature": signature,
+ }
+ )
+ return out
+
+
+def _reasoning_from_stream_delta(delta: Any) -> tuple[str, str | None]:
+ """Incremental reasoning text and (if present) signature from a litellm
+ streaming delta. Prefers ``reasoning_content`` for the text and reads the
+ signature off ``thinking_blocks`` (where litellm places it, usually on the
+ final reasoning chunk)."""
+ text = getattr(delta, "reasoning_content", None) or ""
+ signature: str | None = None
+ for block in getattr(delta, "thinking_blocks", None) or []:
+ bd = _thinking_block_to_dict(block)
+ if not text and bd.get("thinking"):
+ text = bd["thinking"]
+ if bd.get("signature"):
+ signature = bd["signature"]
+ return text, signature
+
+
class LiteLLMBackend(Backend):
"""Backend using LiteLLM for multi-provider support.
@@ -746,7 +844,11 @@ class LiteLLMBackend(Backend):
return True
return "claude" in model.lower() or model in self._model_map
- def _convert_messages_for_litellm(self, messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ def _convert_messages_for_litellm(
+ self,
+ messages: list[dict[str, Any]],
+ preserve_thinking: bool = False,
+ ) -> list[dict[str, Any]]:
"""Convert Anthropic message format to LiteLLM/OpenAI format.
Anthropic and OpenAI have different representations for tool calls:
@@ -755,6 +857,15 @@ class LiteLLMBackend(Backend):
This method converts Anthropic-style messages to OpenAI-style so LiteLLM
can send them to any provider.
+
+ When ``preserve_thinking`` is set (Anthropic-family target only), an
+ assistant turn's ``thinking``/``redacted_thinking`` blocks are carried
+ through on the outgoing message's ``thinking_blocks`` field. litellm's
+ own Anthropic and Bedrock request transforms read that field and forward
+ the blocks — signature and position intact — which Anthropic *requires*
+ on a tool-use continuation. Left unset (cross-vendor target), the blocks
+ are dropped: litellm would otherwise ship the unknown field to a
+ non-Anthropic provider, which rejects it.
"""
converted = []
for msg in messages:
@@ -772,6 +883,7 @@ class LiteLLMBackend(Backend):
text_parts = []
tool_use_blocks = []
tool_result_blocks = []
+ thinking_blocks: list[dict[str, Any]] = []
for block in content:
if not isinstance(block, dict):
@@ -783,6 +895,15 @@ class LiteLLMBackend(Backend):
tool_use_blocks.append(block)
elif block_type == "tool_result":
tool_result_blocks.append(block)
+ elif block_type == "redacted_thinking":
+ if block.get("data"):
+ thinking_blocks.append(block)
+ elif block_type == "thinking":
+ # Only a signed block is legal to replay: Anthropic 400s
+ # an unsigned thinking block, and litellm would drop it
+ # anyway. Symmetric with the response-side guard.
+ if block.get("signature"):
+ thinking_blocks.append(block)
# tool_result blocks → OpenAI "tool" role messages
if tool_result_blocks:
@@ -831,14 +952,23 @@ class LiteLLMBackend(Backend):
}
for tu in tool_use_blocks
]
+ # Anthropic 400s a tool continuation whose assistant turn
+ # carried thinking but lost it here. litellm's Anthropic /
+ # Bedrock transforms read thinking_blocks off the message and
+ # re-emit them, signature and lead position intact.
+ if preserve_thinking and thinking_blocks:
+ assistant_msg["thinking_blocks"] = thinking_blocks
converted.append(assistant_msg)
continue
- # Simple text only
- if text_parts:
- converted.append({"role": role, "content": "\n".join(text_parts)})
- else:
- converted.append({"role": role, "content": ""})
+ # Simple text only (or a thinking-only assistant turn)
+ simple_msg: dict[str, Any] = {
+ "role": role,
+ "content": "\n".join(text_parts) if text_parts else "",
+ }
+ if preserve_thinking and thinking_blocks and role == "assistant":
+ simple_msg["thinking_blocks"] = thinking_blocks
+ converted.append(simple_msg)
return converted
@@ -905,8 +1035,12 @@ class LiteLLMBackend(Backend):
choice = litellm_response.choices[0]
message = choice.message
- # Build Anthropic content blocks
- content = []
+ # Build Anthropic content blocks. Signed thinking must LEAD the content
+ # (Anthropic rejects a thinking block that does not come first) and
+ # round-trip verbatim so the client can replay it on the next tool turn.
+ # litellm surfaces upstream thinking on message.thinking_blocks with the
+ # signature intact; unsigned reasoning is dropped, not emitted.
+ content: list[dict[str, Any]] = list(_extract_thinking_content_blocks(message))
if message.content:
content.append({"type": "text", "text": message.content})
@@ -953,10 +1087,13 @@ class LiteLLMBackend(Backend):
"""Send message via LiteLLM."""
original_model = body.get("model", "claude-3-5-sonnet-20241022")
litellm_model = self.map_model_id(original_model)
+ preserve_thinking = _is_anthropic_family_model(litellm_model)
try:
# Convert messages
- messages = self._convert_messages_for_litellm(body.get("messages", []))
+ messages = self._convert_messages_for_litellm(
+ body.get("messages", []), preserve_thinking=preserve_thinking
+ )
# Build kwargs for litellm
kwargs: dict[str, Any] = {
@@ -973,6 +1110,12 @@ class LiteLLMBackend(Backend):
kwargs["top_p"] = body["top_p"]
if "stop_sequences" in body:
kwargs["stop"] = body["stop_sequences"]
+ # Forward the extended-thinking config to Anthropic-family targets.
+ # Required for consistency with preserved history: Anthropic errors
+ # if an assistant message carries thinking blocks while thinking is
+ # disabled for the turn. Never sent cross-vendor.
+ if preserve_thinking and "thinking" in body:
+ kwargs["thinking"] = body["thinking"]
# Tools (convert Anthropic format to OpenAI format)
if "tools" in body:
@@ -1073,9 +1216,12 @@ class LiteLLMBackend(Backend):
"""
original_model = body.get("model", "claude-3-5-sonnet-20241022")
litellm_model = self.map_model_id(original_model)
+ preserve_thinking = _is_anthropic_family_model(litellm_model)
try:
- messages = self._convert_messages_for_litellm(body.get("messages", []))
+ messages = self._convert_messages_for_litellm(
+ body.get("messages", []), preserve_thinking=preserve_thinking
+ )
kwargs: dict[str, Any] = {
"model": litellm_model,
@@ -1091,6 +1237,10 @@ class LiteLLMBackend(Backend):
kwargs["top_p"] = body["top_p"]
if "stop_sequences" in body:
kwargs["stop"] = body["stop_sequences"]
+ # Forward extended-thinking config to Anthropic-family targets only
+ # (see send_message for why). Never sent cross-vendor.
+ if preserve_thinking and "thinking" in body:
+ kwargs["thinking"] = body["thinking"]
if "tools" in body:
tools_in = body["tools"]
# Bedrock Converse API hard-rejects tool names over 64 chars.
@@ -1175,6 +1325,99 @@ class LiteLLMBackend(Backend):
final_cache_read_tokens = 0
final_cache_write_tokens = 0
+ # Extended thinking is BUFFERED, not streamed live. A thinking block
+ # is only legal to replay if it leads the turn and carries a real
+ # signature; litellm delivers the signature on a later chunk, so
+ # emitting live would risk (a) an unsigned block if the stream ends
+ # first, and (b) a block reopened after text/tool_use if a signature
+ # arrives late. Instead we accumulate here and flush a single signed
+ # block at the transition to the first non-thinking content (or at
+ # end of stream), dropping it entirely if no signature ever arrived —
+ # the same guard the non-streaming path applies.
+ #
+ # KNOWN LIMITATION (litellm-routed streaming): litellm normalises the
+ # native Anthropic SSE into a flattened OpenAI shape — reasoning text
+ # concatenated on `reasoning_content`, blocks accumulated on
+ # `thinking_blocks` — which loses per-block boundaries. So this
+ # reconstructs the common case (one signed thinking block) faithfully
+ # but does NOT reconstruct `redacted_thinking` blocks, nor multiple
+ # distinct thinking blocks with independent signatures, on the stream.
+ # Anthropic validates signatures BY POSITION, so guessing an order
+ # would risk a corrupt (400/poison) block — strictly worse than a
+ # clean drop, which on this litellm path merely makes litellm disable
+ # thinking for the next turn (it detects the missing blocks), not a
+ # crash. Full streaming fidelity for those cases requires routing
+ # Anthropic-family targets through litellm.anthropic_messages (the
+ # native passthrough), which preserves the native SSE untouched.
+ thinking_text = ""
+ thinking_signature: str | None = None
+ thinking_seen = False
+ thinking_flushed = False
+ content_started = False # a text/tool_use block has opened
+
+ def _flush_thinking() -> list[StreamEvent]:
+ nonlocal current_block_index, active_block_type, output_tokens
+ nonlocal thinking_flushed
+ thinking_flushed = True
+ if not thinking_signature or content_started:
+ # Drop when unsigned (Anthropic 400s it on replay) or when
+ # content already started (a thinking block MUST lead — a
+ # late one cannot be legally placed). Matches the
+ # non-streaming guard; a genuine Anthropic stream always
+ # sends thinking, signed, before any text/tool_use anyway.
+ return []
+ events: list[StreamEvent] = []
+ if active_block_type is not None:
+ events.append(
+ StreamEvent(
+ event_type="content_block_stop",
+ data={"type": "content_block_stop", "index": current_block_index},
+ )
+ )
+ current_block_index += 1
+ idx = current_block_index
+ events.append(
+ StreamEvent(
+ event_type="content_block_start",
+ data={
+ "type": "content_block_start",
+ "index": idx,
+ "content_block": {"type": "thinking", "thinking": ""},
+ },
+ )
+ )
+ if thinking_text:
+ events.append(
+ StreamEvent(
+ event_type="content_block_delta",
+ data={
+ "type": "content_block_delta",
+ "index": idx,
+ "delta": {"type": "thinking_delta", "thinking": thinking_text},
+ },
+ )
+ )
+ output_tokens += 1
+ events.append(
+ StreamEvent(
+ event_type="content_block_delta",
+ data={
+ "type": "content_block_delta",
+ "index": idx,
+ "delta": {"type": "signature_delta", "signature": thinking_signature},
+ },
+ )
+ )
+ events.append(
+ StreamEvent(
+ event_type="content_block_stop",
+ data={"type": "content_block_stop", "index": idx},
+ )
+ )
+ # Thinking block fully closed; the next content opens fresh.
+ active_block_type = None
+ return events
+
async for chunk in response:
if hasattr(chunk, "usage") and chunk.usage:
cu = chunk.usage
@@ -1198,6 +1441,28 @@ class LiteLLMBackend(Backend):
elif choice.finish_reason == "length":
stop_reason = "max_tokens"
+ # Accumulate extended-thinking (buffered — see _flush_thinking).
+ # Only for Anthropic-family targets, and only until flushed;
+ # a late reasoning delta after content has started is ignored
+ # rather than emitted as an illegal non-leading thinking block.
+ if preserve_thinking and not thinking_flushed:
+ reasoning_piece, signature_piece = _reasoning_from_stream_delta(delta)
+ if reasoning_piece:
+ thinking_text += reasoning_piece
+ thinking_seen = True
+ if signature_piece:
+ thinking_signature = signature_piece
+ thinking_seen = True
+
+ # First non-thinking content: flush the buffered thinking block
+ # (signed) so it leads, before opening any text/tool_use block.
+ has_content = bool(getattr(delta, "tool_calls", None)) or bool(
+ getattr(delta, "content", None)
+ )
+ if thinking_seen and not thinking_flushed and has_content:
+ for ev in _flush_thinking():
+ yield ev
+
# Handle tool_calls in the delta
if hasattr(delta, "tool_calls") and delta.tool_calls:
for tc in delta.tool_calls:
@@ -1216,6 +1481,7 @@ class LiteLLMBackend(Backend):
current_block_index += 1
tool_block_map[idx] = current_block_index
active_block_type = "tool_use"
+ content_started = True
tool_id = tc.id or f"toolu_{uuid.uuid4().hex[:24]}"
tool_name = tc.function.name if tc.function and tc.function.name else ""
yield StreamEvent(
@@ -1263,6 +1529,7 @@ class LiteLLMBackend(Backend):
# Open a new text block
current_block_index += 1
active_block_type = "text"
+ content_started = True
yield StreamEvent(
event_type="content_block_start",
data={
@@ -1282,6 +1549,12 @@ class LiteLLMBackend(Backend):
)
output_tokens += 1
+ # Thinking-only turn: no non-thinking content ever arrived to trigger
+ # the mid-loop flush, so flush it now (still signature-guarded).
+ if thinking_seen and not thinking_flushed:
+ for ev in _flush_thinking():
+ yield ev
+
# Close the last open block
if active_block_type is not None:
yield StreamEvent(
@@ -1457,6 +1730,28 @@ class LiteLLMBackend(Backend):
"message": {
"role": c.message.role,
"content": c.message.content,
+ # Carry reasoning through for the OpenAI-in ->
+ # Anthropic-out direction (e.g. Codex routed to a
+ # Claude model): the model's thinking would otherwise
+ # be dropped from the OpenAI-shape response. Keep
+ # both dialects — reasoning_content (text) and the
+ # signed thinking_blocks — so a caller can display or
+ # replay it.
+ **(
+ {"reasoning_content": c.message.reasoning_content}
+ if getattr(c.message, "reasoning_content", None)
+ else {}
+ ),
+ **(
+ {
+ "thinking_blocks": [
+ _thinking_block_to_dict(b)
+ for b in c.message.thinking_blocks
+ ]
+ }
+ if getattr(c.message, "thinking_blocks", None)
+ else {}
+ ),
**(
{
"tool_calls": [
diff --git a/tests/test_litellm_openai_passthrough.py b/tests/test_litellm_openai_passthrough.py
index 23ee5ceb9..1604102ad 100644
--- a/tests/test_litellm_openai_passthrough.py
+++ b/tests/test_litellm_openai_passthrough.py
@@ -143,3 +143,49 @@ async def test_standard_params_still_forwarded() -> None:
assert kwargs["top_p"] == 0.9
assert kwargs["response_format"] == {"type": "json_object"}
assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}
+
+
+async def test_reasoning_params_forwarded_top_level_not_extra_body() -> None:
+ # reasoning_effort / max_completion_tokens are litellm-mapped params: they
+ # must go as top-level kwargs (litellm maps reasoning_effort -> Anthropic
+ # thinking), NOT swept into extra_body, which a Claude target 400s. Regression
+ # for the OpenAI-in -> Claude-out cross-protocol reasoning path.
+ backend = make_backend()
+
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = make_response()
+
+ await backend.send_openai_message(
+ request_body(reasoning_effort="low", max_completion_tokens=256),
+ {},
+ )
+
+ kwargs = mock_acomp.await_args.kwargs
+ assert kwargs["reasoning_effort"] == "low"
+ assert kwargs["max_completion_tokens"] == 256
+ # Must NOT leak into extra_body (which would reach a non-OpenAI provider raw).
+ assert "reasoning_effort" not in kwargs.get("extra_body", {})
+ assert "max_completion_tokens" not in kwargs.get("extra_body", {})
+ assert "extra_body" not in kwargs
+
+
+async def test_reasoning_params_forwarded_top_level_streaming() -> None:
+ backend = make_backend()
+ stream = FakeAsyncStream(
+ [SimpleNamespace(model_dump=lambda **kwargs: {"id": "c1", "choices": []})]
+ )
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = stream
+ _ = [
+ c
+ async for c in backend.stream_openai_message(
+ request_body(reasoning_effort="high"),
+ {},
+ )
+ ]
+
+ kwargs = mock_acomp.await_args.kwargs
+ assert kwargs["reasoning_effort"] == "high"
+ assert "reasoning_effort" not in kwargs.get("extra_body", {})
diff --git a/tests/test_litellm_thinking_fidelity.py b/tests/test_litellm_thinking_fidelity.py
new file mode 100644
index 000000000..035e981ea
--- /dev/null
+++ b/tests/test_litellm_thinking_fidelity.py
@@ -0,0 +1,438 @@
+"""LiteLLMBackend thinking-block fidelity.
+
+Regression suite for the bug where the Anthropic<->LiteLLM converter dropped
+extended-thinking blocks in both directions. Dropping them is not a quality
+loss but a hard Anthropic 400 on a tool-use continuation
+(`Expected thinking or redacted_thinking, but found tool_use`), and the
+signature is validated cryptographically, so blocks must round-trip verbatim.
+
+Contract pinned here:
+- REQUEST (Anthropic-family target): assistant thinking/redacted_thinking blocks
+ are carried on the outgoing message's `thinking_blocks` field, which litellm's
+ own Anthropic/Bedrock transforms forward with signature + lead position.
+- REQUEST (cross-vendor target): thinking is stripped, so litellm cannot ship an
+ unknown field to a non-Anthropic provider.
+- RESPONSE: a signed thinking block from litellm (message.thinking_blocks) is
+ rebuilt as a LEADING Anthropic thinking block; unsigned reasoning is dropped.
+- STREAMING: thinking is emitted as a leading thinking block with thinking_delta
+ + signature_delta before any text/tool_use.
+- OpenAI-in: reasoning survives into the rebuilt OpenAI response (Codex->Claude).
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from tests._dotenv import importorskip_no_env_leak
+
+importorskip_no_env_leak("litellm")
+
+from headroom.backends.litellm import ( # noqa: E402
+ LiteLLMBackend,
+ _extract_thinking_content_blocks,
+ _is_anthropic_family_model,
+)
+
+SIG = "ErUBCkYIBRgCKkB_signed_opaque_blob=="
+
+
+def _backend(provider: str = "anthropic") -> LiteLLMBackend:
+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):
+ return LiteLLMBackend(provider=provider)
+
+
+THINKING_TOOL_TURN = {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "The user wants the weather.", "signature": SIG},
+ {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}},
+ ],
+}
+
+THINKING_TEXT_TURN = {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "Simple greeting.", "signature": SIG},
+ {"type": "text", "text": "Hello!"},
+ ],
+}
+
+
+# --------------------------------------------------------------------------- #
+# helper unit
+# --------------------------------------------------------------------------- #
+ "model,expected",
+ [
+ ("anthropic/claude-opus-4-20250514", True),
+ ("bedrock/anthropic.claude-sonnet-4-20250514-v1:0", True),
+ ("vertex_ai/claude-3-5-sonnet", True),
+ ("claude-opus-5", True),
+ ("gpt-4o", False),
+ ("openai/gpt-5", False),
+ ("deepseek/deepseek-chat", False),
+ ("", False),
+ ],
+)
+def test_is_anthropic_family_model(model: str, expected: bool) -> None:
+ assert _is_anthropic_family_model(model) is expected
+
+
+# --------------------------------------------------------------------------- #
+# request-direction conversion
+# --------------------------------------------------------------------------- #
+def test_thinking_survives_conversion_for_anthropic_target() -> None:
+ backend = _backend()
+ converted = backend._convert_messages_for_litellm([THINKING_TOOL_TURN], preserve_thinking=True)
+ assistant = next(m for m in converted if m.get("tool_calls"))
+ assert "thinking_blocks" in assistant, "signed thinking dropped for Anthropic target"
+ tb = assistant["thinking_blocks"][0]
+ assert tb["type"] == "thinking"
+ assert tb["signature"] == SIG, "signature must round-trip byte-for-byte"
+
+
+def test_thinking_survives_on_text_only_assistant_turn() -> None:
+ backend = _backend()
+ converted = backend._convert_messages_for_litellm([THINKING_TEXT_TURN], preserve_thinking=True)
+ assistant = next(m for m in converted if m["role"] == "assistant")
+ assert assistant["thinking_blocks"][0]["signature"] == SIG
+ assert assistant["content"] == "Hello!"
+
+
+def test_thinking_stripped_for_cross_vendor_target() -> None:
+ backend = _backend(provider="openai")
+ converted = backend._convert_messages_for_litellm([THINKING_TOOL_TURN], preserve_thinking=False)
+ assistant = next(m for m in converted if m.get("tool_calls"))
+ assert "thinking_blocks" not in assistant, (
+ "thinking must NOT be shipped to a non-Anthropic provider"
+ )
+
+
+def test_redacted_thinking_survives_conversion() -> None:
+ backend = _backend()
+ turn = {
+ "role": "assistant",
+ "content": [
+ {"type": "redacted_thinking", "data": "ENCRYPTED_BLOB=="},
+ {"type": "tool_use", "id": "t1", "name": "f", "input": {}},
+ ],
+ }
+ converted = backend._convert_messages_for_litellm([turn], preserve_thinking=True)
+ assistant = next(m for m in converted if m.get("tool_calls"))
+ assert assistant["thinking_blocks"][0] == {
+ "type": "redacted_thinking",
+ "data": "ENCRYPTED_BLOB==",
+ }
+
+
+# --------------------------------------------------------------------------- #
+# response-direction reconstruction (pure helper)
+# --------------------------------------------------------------------------- #
+def test_extract_thinking_keeps_signed_block_first() -> None:
+ message = SimpleNamespace(
+ content="Answer.",
+ thinking_blocks=[{"type": "thinking", "thinking": "reasoning", "signature": SIG}],
+ )
+ blocks = _extract_thinking_content_blocks(message)
+ assert blocks == [{"type": "thinking", "thinking": "reasoning", "signature": SIG}]
+
+
+def test_extract_thinking_drops_unsigned_reasoning() -> None:
+ # An unsigned block would be rejected by Anthropic on replay; never emit it.
+ message = SimpleNamespace(
+ content="Answer.",
+ thinking_blocks=[{"type": "thinking", "thinking": "reasoning", "signature": ""}],
+ reasoning_content="reasoning",
+ )
+ assert _extract_thinking_content_blocks(message) == []
+
+
+def test_extract_thinking_keeps_redacted() -> None:
+ message = SimpleNamespace(thinking_blocks=[{"type": "redacted_thinking", "data": "BLOB"}])
+ assert _extract_thinking_content_blocks(message) == [
+ {"type": "redacted_thinking", "data": "BLOB"}
+ ]
+
+
+# --------------------------------------------------------------------------- #
+# send_message end-to-end (mocked acompletion)
+# --------------------------------------------------------------------------- #
+def _mock_response(*, content="Done.", thinking_blocks=None, reasoning=None, tool_calls=None):
+ message = SimpleNamespace(
+ role="assistant",
+ content=content,
+ tool_calls=tool_calls,
+ thinking_blocks=thinking_blocks,
+ reasoning_content=reasoning,
+ )
+ return SimpleNamespace(
+ id="resp_1",
+ created=1,
+ choices=[SimpleNamespace(index=0, finish_reason="stop", message=message)],
+ usage=SimpleNamespace(prompt_tokens=5, completion_tokens=3, total_tokens=8),
+ )
+
+
+async def test_send_message_forwards_thinking_and_reconstructs_response() -> None:
+ with (
+ patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp,
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ mock_acomp.return_value = _mock_response(
+ thinking_blocks=[{"type": "thinking", "thinking": "resp reasoning", "signature": SIG}]
+ )
+ backend = LiteLLMBackend(provider="anthropic")
+ body = {
+ "model": "claude-opus-4-20250514",
+ "messages": [THINKING_TOOL_TURN, {"role": "user", "content": "and tomorrow?"}],
+ "thinking": {"type": "enabled", "budget_tokens": 2000},
+ }
+ result = await backend.send_message(body, {})
+
+ # request: thinking preserved into outgoing history + thinking param forwarded
+ sent = mock_acomp.call_args.kwargs
+ hist_assistant = next(m for m in sent["messages"] if m.get("tool_calls"))
+ assert hist_assistant["thinking_blocks"][0]["signature"] == SIG
+ assert sent["thinking"] == {"type": "enabled", "budget_tokens": 2000}
+
+ # response: signed thinking rebuilt as the LEADING content block
+ content = result.body["content"]
+ assert content[0]["type"] == "thinking"
+ assert content[0]["signature"] == SIG
+ assert content[1]["type"] == "text"
+
+
+async def test_send_message_strips_thinking_for_cross_vendor() -> None:
+ with (
+ patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp,
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ mock_acomp.return_value = _mock_response()
+ backend = LiteLLMBackend(provider="openai")
+ body = {
+ "model": "gpt-4o",
+ "messages": [THINKING_TOOL_TURN, {"role": "user", "content": "hi"}],
+ "thinking": {"type": "enabled", "budget_tokens": 2000},
+ }
+ await backend.send_message(body, {})
+
+ sent = mock_acomp.call_args.kwargs
+ hist_assistant = next(m for m in sent["messages"] if m.get("tool_calls"))
+ assert "thinking_blocks" not in hist_assistant
+ assert "thinking" not in sent, "thinking config must not be forwarded cross-vendor"
+
+
+# --------------------------------------------------------------------------- #
+# streaming response reconstruction
+# --------------------------------------------------------------------------- #
+class _FakeAsyncStream:
+ def __init__(self, chunks):
+ self._chunks = chunks
+
+ def __aiter__(self):
+ self._it = iter(self._chunks)
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._it)
+ except StopIteration:
+ raise StopAsyncIteration from None
+
+
+def _delta(**kw):
+ base = {"reasoning_content": None, "thinking_blocks": None, "content": None, "tool_calls": None}
+ base.update(kw)
+ return SimpleNamespace(**base)
+
+
+def _chunk(delta, finish_reason=None, usage=None):
+ ns = SimpleNamespace(choices=[SimpleNamespace(finish_reason=finish_reason, delta=delta)])
+ if usage is not None:
+ ns.usage = usage
+ return ns
+
+
+async def test_streaming_emits_leading_thinking_block_with_signature() -> None:
+ chunks = [
+ _chunk(_delta(reasoning_content="Let me ")),
+ _chunk(_delta(reasoning_content="think.")),
+ _chunk(_delta(thinking_blocks=[{"type": "thinking", "signature": SIG}])),
+ _chunk(_delta(content="The answer.")),
+ _chunk(
+ _delta(),
+ finish_reason="stop",
+ usage=SimpleNamespace(
+ prompt_tokens=10, cache_read_input_tokens=0, cache_creation_input_tokens=0
+ ),
+ ),
+ ]
+ with (
+ patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp,
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ mock_acomp.return_value = _FakeAsyncStream(chunks)
+ backend = LiteLLMBackend(provider="anthropic")
+ body = {"model": "claude-opus-4-20250514", "messages": [{"role": "user", "content": "hi"}]}
+ events = [ev.data async for ev in backend.stream_message(body, {})]
+
+ types = [
+ (
+ e.get("type"),
+ e.get("delta", {}).get("type")
+ if e.get("type") == "content_block_delta"
+ else e.get("content_block", {}).get("type")
+ if e.get("type") == "content_block_start"
+ else None,
+ )
+ for e in events
+ ]
+ # thinking block opens first, carries thinking_delta then signature_delta,
+ # closes, then the text block opens.
+ assert ("content_block_start", "thinking") in types
+ assert ("content_block_delta", "thinking_delta") in types
+ assert ("content_block_delta", "signature_delta") in types
+ assert ("content_block_start", "text") in types
+ # ordering: the thinking block_start precedes the text block_start
+ start_types = [t for t in types if t[0] == "content_block_start"]
+ assert start_types[0] == ("content_block_start", "thinking")
+ assert start_types[1] == ("content_block_start", "text")
+ # the signed thinking text was accumulated
+ thinking_text = "".join(
+ e["delta"]["thinking"]
+ for e in events
+ if e.get("type") == "content_block_delta" and e["delta"].get("type") == "thinking_delta"
+ )
+ assert thinking_text == "Let me think."
+ sig = [
+ e["delta"]["signature"]
+ for e in events
+ if e.get("type") == "content_block_delta" and e["delta"].get("type") == "signature_delta"
+ ]
+ assert sig == [SIG]
+
+
+# --------------------------------------------------------------------------- #
+# OpenAI-in reasoning passthrough (Codex -> Claude direction)
+# --------------------------------------------------------------------------- #
+async def test_send_openai_message_carries_reasoning_through() -> None:
+ with (
+ patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp,
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ mock_acomp.return_value = _mock_response(
+ reasoning="claude reasoning",
+ thinking_blocks=[
+ {"type": "thinking", "thinking": "claude reasoning", "signature": SIG}
+ ],
+ )
+ backend = LiteLLMBackend(provider="anthropic")
+ body = {"model": "claude-opus-4-20250514", "messages": [{"role": "user", "content": "hi"}]}
+ result = await backend.send_openai_message(body, {})
+
+ msg = result.body["choices"][0]["message"]
+ assert msg["reasoning_content"] == "claude reasoning"
+ assert msg["thinking_blocks"][0]["signature"] == SIG
+
+
+# --------------------------------------------------------------------------- #
+# streaming edge cases (adversarial-review regressions)
+# --------------------------------------------------------------------------- #
+def _block_starts(events):
+ return [e["content_block"]["type"] for e in events if e.get("type") == "content_block_start"]
+
+
+def _delta_types(events):
+ return [e["delta"]["type"] for e in events if e.get("type") == "content_block_delta"]
+
+
+async def _run_stream(chunks):
+ with (
+ patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp,
+ patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}),
+ ):
+ mock_acomp.return_value = _FakeAsyncStream(chunks)
+ backend = LiteLLMBackend(provider="anthropic")
+ body = {"model": "claude-opus-4-20250514", "messages": [{"role": "user", "content": "hi"}]}
+ return [ev.data async for ev in backend.stream_message(body, {})]
+
+
+async def test_streaming_drops_unsigned_thinking() -> None:
+ # Reasoning streams but a signature never arrives (e.g. truncation): the
+ # thinking block must be dropped, never emitted unsigned (rule 3).
+ events = await _run_stream(
+ [
+ _chunk(_delta(reasoning_content="Let me ")),
+ _chunk(_delta(reasoning_content="think.")),
+ _chunk(_delta(content="answer")),
+ _chunk(_delta(), finish_reason="stop"),
+ ]
+ )
+ assert "thinking" not in _block_starts(events), "unsigned thinking must not be emitted"
+ assert "thinking_delta" not in _delta_types(events)
+ assert "signature_delta" not in _delta_types(events)
+ assert _block_starts(events) == ["text"]
+
+
+async def test_streaming_late_signature_after_text_is_dropped_not_reordered() -> None:
+ # Reasoning + signature arrive AFTER text already started. A thinking block
+ # can only lead, so it must be dropped — never reopened after text (rule 2),
+ # and no stray signature_delta emitted against a closed/foreign block.
+ events = await _run_stream(
+ [
+ _chunk(_delta(content="Hello")),
+ _chunk(_delta(reasoning_content="late thought")),
+ _chunk(_delta(thinking_blocks=[{"type": "thinking", "signature": SIG}])),
+ _chunk(_delta(content=" world")),
+ _chunk(_delta(), finish_reason="stop"),
+ ]
+ )
+ assert _block_starts(events) == ["text"], "no thinking block may follow text"
+ assert "thinking_delta" not in _delta_types(events)
+ assert "signature_delta" not in _delta_types(events)
+
+
+async def test_streaming_thinking_only_turn_flushes_signed() -> None:
+ # Signed reasoning with no subsequent content: still emitted as a leading
+ # (and only) thinking block at end of stream.
+ events = await _run_stream(
+ [
+ _chunk(_delta(reasoning_content="just thinking")),
+ _chunk(_delta(thinking_blocks=[{"type": "thinking", "signature": SIG}])),
+ _chunk(_delta(), finish_reason="stop"),
+ ]
+ )
+ assert _block_starts(events) == ["thinking"]
+ assert "signature_delta" in _delta_types(events)
+ sig = [
+ e["delta"]["signature"]
+ for e in events
+ if e.get("type") == "content_block_delta" and e["delta"].get("type") == "signature_delta"
+ ]
+ assert sig == [SIG]
+
+
+def test_request_drops_unsigned_thinking_block() -> None:
+ # Symmetric with the response path: an unsigned thinking block in history is
+ # NOT forwarded (litellm would ship it and Anthropic would 400 the turn).
+ backend = _backend()
+ turn = {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "unsigned"}, # no signature
+ {"type": "tool_use", "id": "t1", "name": "f", "input": {}},
+ ],
+ }
+ converted = backend._convert_messages_for_litellm([turn], preserve_thinking=True)
+ assistant = next(m for m in converted if m.get("tool_calls"))
+ assert "thinking_blocks" not in assistant, "unsigned thinking must not be forwarded"