Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(litellm): forward chat_template_kwargs and other vendor top-level fields to OpenAI-compatible backends via extra_body (#2128) (#2163)

Rod Boevfb683e18d4cb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac6bb98c8..9d75700f5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
### Fixed
+- **litellm:** vendor-specific top-level fields on `/v1/chat/completions`, including vLLM's `chat_template_kwargs` for per-request Qwen3 thinking-mode toggles, now reach OpenAI-compatible backends through LiteLLM `extra_body` instead of being dropped by the standard-parameter allowlist ([#2128](https://github.com/headroomlabs-ai/headroom/issues/2128)).
- **cache aligner:** hash the actual frozen Claude Code prefix instead of only system-message text, so `stable_prefix_hash` / `prefix_changed` now surface prompt-cache churn when a cached tool-result block changes without any system-prompt edit ([#2085](https://github.com/headroomlabs-ai/headroom/issues/2085)).
- **proxy image compression:** run native image compression in a spawned subprocess so an OpenCV or KleidiCV crash now fails open to the original image payload instead of taking down the whole proxy process ([#2107](https://github.com/headroomlabs-ai/headroom/issues/2107)).
- **proxy/windows:** support Windows selector-event-loop startup on uvicorn versions older than 0.36. Newer uvicorn accepts `loop="asyncio:SelectorEventLoop"` as a custom loop-factory import path, but older versions treat it as an unknown built-in loop name and raise `KeyError`. Windows now sets `WindowsSelectorEventLoopPolicy` for those older versions instead of passing an unsupported `loop` value ([#1650](https://github.com/headroomlabs-ai/headroom/issues/1650), [#1621](https://github.com/headroomlabs-ai/headroom/issues/1621)).
diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py
index 8954f2412..526a8d76e 100644
--- a/headroom/backends/litellm.py
+++ b/headroom/backends/litellm.py
@@ -25,6 +25,28 @@ from .base import Backend, BackendResponse, StreamEvent
logger = logging.getLogger(__name__)
+_OPENAI_STANDARD_PARAMS = (
+ "max_tokens",
+ "temperature",
+ "top_p",
+ "stop",
+ "tools",
+ "tool_choice",
+ "response_format",
+ "seed",
+ "n",
+)
+
+_OPENAI_CONSUMED_BODY_KEYS = frozenset(
+ {
+ "model",
+ "messages",
+ "stream",
+ "stream_options",
+ *_OPENAI_STANDARD_PARAMS,
+ }
+)
+
# litellm calls `dotenv.load_dotenv()` during its own import, which loads
# the project `.env` into `os.environ`. We don't want that side effect —
# importing a backend module should not silently leak API keys into the
@@ -139,6 +161,17 @@ def _build_bedrock_fallback_map(region: str) -> dict[str, str]:
return {name: f"bedrock/{prefix}.{model_id}" for name, model_id in _CLAUDE_MODELS}
+def _build_openai_extra_body(body: dict[str, Any]) -> dict[str, Any]:
+ """Return unconsumed top-level OpenAI request fields for vendor passthrough."""
+ return {
+ key: value
+ for key, value in body.items()
+ if key not in _OPENAI_CONSUMED_BODY_KEYS
+ and not key.startswith("x-headroom-")
+ and not key.startswith("x_headroom_")
+ }
+
+
def _fetch_bedrock_inference_profiles(
region: str | None, profile_name: str | None = None
) -> dict[str, str]:
@@ -1177,20 +1210,14 @@ class LiteLLMBackend(Backend):
}
# Pass through OpenAI parameters
- for param in [
- "max_tokens",
- "temperature",
- "top_p",
- "stop",
- "tools",
- "tool_choice",
- "response_format",
- "seed",
- "n",
- ]:
+ for param in _OPENAI_STANDARD_PARAMS:
if param in body:
kwargs[param] = body[param]
+ extra_body = _build_openai_extra_body(body)
+ if extra_body:
+ kwargs["extra_body"] = extra_body
+
# Provider-specific region config
if self.region:
if self.provider == "bedrock":
@@ -1352,23 +1379,17 @@ class LiteLLMBackend(Backend):
"stream": True,
}
- for param in [
- "max_tokens",
- "temperature",
- "top_p",
- "stop",
- "tools",
- "tool_choice",
- "response_format",
- "seed",
- "n",
- ]:
+ for param in _OPENAI_STANDARD_PARAMS:
if param in body:
kwargs[param] = body[param]
if "stream_options" in body:
kwargs["stream_options"] = body["stream_options"]
+ extra_body = _build_openai_extra_body(body)
+ if extra_body:
+ kwargs["extra_body"] = extra_body
+
# Provider-specific region config
if self.region:
if self.provider == "bedrock":
diff --git a/tests/test_litellm_openai_passthrough.py b/tests/test_litellm_openai_passthrough.py
new file mode 100644
index 000000000..23ee5ceb9
--- /dev/null
+++ b/tests/test_litellm_openai_passthrough.py
@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from tests._dotenv import importorskip_no_env_leak
+
+importorskip_no_env_leak("litellm")
+
+from headroom.backends.litellm import LiteLLMBackend # noqa: E402
+
+
+class FakeAsyncStream:
+ def __init__(self, items) -> None: # noqa: ANN001
+ self._items = list(items)
+
+ def __aiter__(self):
+ self._iter = iter(self._items)
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._iter)
+ except StopIteration as exc:
+ raise StopAsyncIteration from exc
+
+
+def make_backend() -> LiteLLMBackend:
+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):
+ return LiteLLMBackend(provider="openrouter")
+
+
+def make_response() -> SimpleNamespace:
+ return SimpleNamespace(
+ id="resp_123",
+ created=123456,
+ choices=[
+ SimpleNamespace(
+ index=0,
+ finish_reason="stop",
+ message=SimpleNamespace(role="assistant", content="ok", tool_calls=None),
+ )
+ ],
+ usage=SimpleNamespace(prompt_tokens=2, completion_tokens=3, total_tokens=5),
+ )
+
+
+def request_body(**overrides):
+ body = {
+ "model": "qwen3",
+ "messages": [{"role": "user", "content": "hello"}],
+ "max_tokens": 32,
+ }
+ body.update(overrides)
+ return body
+
+
+async def test_chat_template_kwargs_forwarded_buffered() -> None:
+ backend = make_backend()
+
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = make_response()
+
+ await backend.send_openai_message(
+ request_body(chat_template_kwargs={"enable_thinking": False}),
+ {},
+ )
+
+ kwargs = mock_acomp.await_args.kwargs
+ assert kwargs["max_tokens"] == 32
+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}
+
+
+async def test_chat_template_kwargs_forwarded_streaming() -> None:
+ backend = make_backend()
+
+ stream = FakeAsyncStream(
+ [
+ SimpleNamespace(model_dump=lambda **kwargs: {"id": "chunk1", "choices": []}),
+ ]
+ )
+
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = stream
+
+ chunks = [
+ chunk
+ async for chunk in backend.stream_openai_message(
+ request_body(
+ chat_template_kwargs={"enable_thinking": False},
+ stream_options={"include_usage": True},
+ ),
+ {},
+ )
+ ]
+
+ kwargs = mock_acomp.await_args.kwargs
+ assert kwargs["stream"] is True
+ assert kwargs["stream_options"] == {"include_usage": True}
+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}
+ assert chunks[-1] == "data: [DONE]\n\n"
+
+
+async def test_standard_only_body_has_no_extra_body() -> None:
+ backend = make_backend()
+
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = make_response()
+
+ await backend.send_openai_message(
+ request_body(temperature=0.1, top_p=0.9),
+ {},
+ )
+
+ kwargs = mock_acomp.await_args.kwargs
+ assert "extra_body" not in kwargs
+
+
+async def test_standard_params_still_forwarded() -> None:
+ backend = make_backend()
+
+ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:
+ mock_acomp.return_value = make_response()
+
+ await backend.send_openai_message(
+ request_body(
+ temperature=0.1,
+ top_p=0.9,
+ response_format={"type": "json_object"},
+ chat_template_kwargs={"enable_thinking": False},
+ ),
+ {},
+ )
+
+ kwargs = mock_acomp.await_args.kwargs
+ assert kwargs["temperature"] == 0.1
+ assert kwargs["top_p"] == 0.9
+ assert kwargs["response_format"] == {"type": "json_object"}
+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}