Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(backends): don't crash the OpenAI->Anthropic converter on empty choices (#2484)
Abhay Singh43a7b578a137
diff --git a/headroom/backends/anyllm.py b/headroom/backends/anyllm.pyindex e5d3d8ee7..468d20450 100644--- a/headroom/backends/anyllm.py+++ b/headroom/backends/anyllm.py@@ -143,6 +143,30 @@ class AnyLLMBackend(Backend):"""Convert any-llm/OpenAI response to Anthropic format."""msg_id = f"msg_{uuid.uuid4().hex[:24]}"+ # A non-streaming upstream response can be HTTP 200 with an empty+ # ``choices`` list (e.g. Azure OpenAI content filtering, or any+ # OpenAI-compatible gateway on a usage-only / filtered turn). The+ # streaming sibling already skips this (`if ... and chunk.choices`);+ # indexing ``choices[0]`` here would instead raise IndexError and 500+ # the request. Return a valid empty assistant turn.+ if not getattr(response, "choices", None):+ usage = {"input_tokens": 0, "output_tokens": 0}+ if getattr(response, "usage", None):+ usage = {+ "input_tokens": getattr(response.usage, "prompt_tokens", 0) or 0,+ "output_tokens": getattr(response.usage, "completion_tokens", 0) or 0,+ }+ return {+ "id": msg_id,+ "type": "message",+ "role": "assistant",+ "content": [],+ "model": original_model,+ "stop_reason": "end_turn",+ "stop_sequence": None,+ "usage": usage,+ }+choice = response.choices[0]message = choice.messagediff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 08e972908..8430789ce 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -793,6 +793,24 @@ class LiteLLMBackend(Backend):"""Convert LiteLLM/OpenAI response to Anthropic format."""msg_id = f"msg_{uuid.uuid4().hex[:24]}"+ # A non-streaming upstream response can be HTTP 200 with an empty+ # ``choices`` list (e.g. Azure OpenAI content filtering, or any+ # OpenAI-compatible gateway on a usage-only / filtered turn). The+ # streaming sibling already `continue`s past this (`if not+ # chunk.choices`); indexing ``choices[0]`` here would instead raise+ # IndexError and 500 the request. Return a valid empty assistant turn.+ if not getattr(litellm_response, "choices", None):+ return {+ "id": msg_id,+ "type": "message",+ "role": "assistant",+ "content": [],+ "model": original_model,+ "stop_reason": "end_turn",+ "stop_sequence": None,+ "usage": _anthropic_usage_from_litellm(getattr(litellm_response, "usage", None)),+ }+# Extract content from OpenAI formatchoice = litellm_response.choices[0]message = choice.messagediff --git a/tests/test_backend_anyllm.py b/tests/test_backend_anyllm.pyindex ec152f0d4..3d56fcb0a 100644--- a/tests/test_backend_anyllm.py+++ b/tests/test_backend_anyllm.py@@ -242,6 +242,26 @@ def test_to_anthropic_response_maps_tool_calls_and_usage(monkeypatch: pytest.Monassert converted["content"][2]["input"] == {"query": "python"}+def test_to_anthropic_response_empty_choices_returns_empty_turn(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # A content-filtered / usage-only upstream response can be 200 with an empty+ # choices list (e.g. Azure OpenAI content filtering). Indexing choices[0]+ # would raise IndexError; the converter must return a valid empty turn, the+ # way the streaming path already skips empty-choice chunks.+ backend, _instance = make_backend(monkeypatch)+ response = make_response(usage=SimpleNamespace(prompt_tokens=9, completion_tokens=0))++ converted = backend._to_anthropic_response(response, "claude-sonnet")++ assert converted["type"] == "message"+ assert converted["role"] == "assistant"+ assert converted["model"] == "claude-sonnet"+ assert converted["content"] == []+ assert converted["stop_reason"] == "end_turn"+ assert converted["usage"] == {"input_tokens": 9, "output_tokens": 0}++@pytest.mark.asyncioasync def test_send_message_builds_anthropic_response(monkeypatch: pytest.MonkeyPatch) -> None:backend, instance = make_backend(monkeypatch)diff --git a/tests/test_litellm_nonstream_cache_usage.py b/tests/test_litellm_nonstream_cache_usage.pyindex 1943669b6..968a4b7cd 100644--- a/tests/test_litellm_nonstream_cache_usage.py+++ b/tests/test_litellm_nonstream_cache_usage.py@@ -84,3 +84,27 @@ def test_output_tokens_none_coerced_to_zero() -> None:)assert usage["output_tokens"] == 0assert isinstance(usage["output_tokens"], int)+++def test_to_anthropic_response_empty_choices_returns_empty_turn() -> None:+ # A content-filtered / usage-only upstream response can be HTTP 200 with an+ # empty choices list (e.g. Azure OpenAI content filtering). Indexing+ # choices[0] would raise IndexError and 500 the request; the converter must+ # return a valid empty assistant turn, the way the streaming path already+ # `continue`s on an empty-choice chunk. _to_anthropic_response uses no+ # instance state, so exercise it on a bare instance.+ backend = object.__new__(litellm_backend.LiteLLMBackend)+ response = SimpleNamespace(+ choices=[],+ usage=SimpleNamespace(prompt_tokens=42, completion_tokens=0),+ )++ converted = backend._to_anthropic_response(response, "claude-sonnet")++ assert converted["type"] == "message"+ assert converted["role"] == "assistant"+ assert converted["model"] == "claude-sonnet"+ assert converted["content"] == []+ assert converted["stop_reason"] == "end_turn"+ assert converted["usage"]["input_tokens"] == 42+ assert converted["usage"]["output_tokens"] == 0