Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(litellm): forward chat_template_kwargs and other vendor top-level fields to OpenAI-compatible backends via extra_body (#2128) (#2163)
Rod Boevfb683e18d4cb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ac6bb98c8..9d75700f5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased### Fixed+- **litellm:** vendor-specific top-level fields on `/v1/chat/completions`, including vLLM's `chat_template_kwargs` for per-request Qwen3 thinking-mode toggles, now reach OpenAI-compatible backends through LiteLLM `extra_body` instead of being dropped by the standard-parameter allowlist ([#2128](https://github.com/headroomlabs-ai/headroom/issues/2128)).- **cache aligner:** hash the actual frozen Claude Code prefix instead of only system-message text, so `stable_prefix_hash` / `prefix_changed` now surface prompt-cache churn when a cached tool-result block changes without any system-prompt edit ([#2085](https://github.com/headroomlabs-ai/headroom/issues/2085)).- **proxy image compression:** run native image compression in a spawned subprocess so an OpenCV or KleidiCV crash now fails open to the original image payload instead of taking down the whole proxy process ([#2107](https://github.com/headroomlabs-ai/headroom/issues/2107)).- **proxy/windows:** support Windows selector-event-loop startup on uvicorn versions older than 0.36. Newer uvicorn accepts `loop="asyncio:SelectorEventLoop"` as a custom loop-factory import path, but older versions treat it as an unknown built-in loop name and raise `KeyError`. Windows now sets `WindowsSelectorEventLoopPolicy` for those older versions instead of passing an unsupported `loop` value ([#1650](https://github.com/headroomlabs-ai/headroom/issues/1650), [#1621](https://github.com/headroomlabs-ai/headroom/issues/1621)).diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.pyindex 8954f2412..526a8d76e 100644--- a/headroom/backends/litellm.py+++ b/headroom/backends/litellm.py@@ -25,6 +25,28 @@ from .base import Backend, BackendResponse, StreamEventlogger = logging.getLogger(__name__)+_OPENAI_STANDARD_PARAMS = (+ "max_tokens",+ "temperature",+ "top_p",+ "stop",+ "tools",+ "tool_choice",+ "response_format",+ "seed",+ "n",+)++_OPENAI_CONSUMED_BODY_KEYS = frozenset(+ {+ "model",+ "messages",+ "stream",+ "stream_options",+ *_OPENAI_STANDARD_PARAMS,+ }+)+# litellm calls `dotenv.load_dotenv()` during its own import, which loads# the project `.env` into `os.environ`. We don't want that side effect —# importing a backend module should not silently leak API keys into the@@ -139,6 +161,17 @@ def _build_bedrock_fallback_map(region: str) -> dict[str, str]:return {name: f"bedrock/{prefix}.{model_id}" for name, model_id in _CLAUDE_MODELS}+def _build_openai_extra_body(body: dict[str, Any]) -> dict[str, Any]:+ """Return unconsumed top-level OpenAI request fields for vendor passthrough."""+ return {+ key: value+ for key, value in body.items()+ if key not in _OPENAI_CONSUMED_BODY_KEYS+ and not key.startswith("x-headroom-")+ and not key.startswith("x_headroom_")+ }++def _fetch_bedrock_inference_profiles(region: str | None, profile_name: str | None = None) -> dict[str, str]:@@ -1177,20 +1210,14 @@ class LiteLLMBackend(Backend):}# Pass through OpenAI parameters- for param in [- "max_tokens",- "temperature",- "top_p",- "stop",- "tools",- "tool_choice",- "response_format",- "seed",- "n",- ]:+ for param in _OPENAI_STANDARD_PARAMS:if param in body:kwargs[param] = body[param]+ extra_body = _build_openai_extra_body(body)+ if extra_body:+ kwargs["extra_body"] = extra_body+# Provider-specific region configif self.region:if self.provider == "bedrock":@@ -1352,23 +1379,17 @@ class LiteLLMBackend(Backend):"stream": True,}- for param in [- "max_tokens",- "temperature",- "top_p",- "stop",- "tools",- "tool_choice",- "response_format",- "seed",- "n",- ]:+ for param in _OPENAI_STANDARD_PARAMS:if param in body:kwargs[param] = body[param]if "stream_options" in body:kwargs["stream_options"] = body["stream_options"]+ extra_body = _build_openai_extra_body(body)+ if extra_body:+ kwargs["extra_body"] = extra_body+# Provider-specific region configif self.region:if self.provider == "bedrock":diff --git a/tests/test_litellm_openai_passthrough.py b/tests/test_litellm_openai_passthrough.pynew file mode 100644index 000000000..23ee5ceb9--- /dev/null+++ b/tests/test_litellm_openai_passthrough.py@@ -0,0 +1,145 @@+from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import AsyncMock, patch++import pytest++from tests._dotenv import importorskip_no_env_leak++importorskip_no_env_leak("litellm")++from headroom.backends.litellm import LiteLLMBackend # noqa: E402+++class FakeAsyncStream:+ def __init__(self, items) -> None: # noqa: ANN001+ self._items = list(items)++ def __aiter__(self):+ self._iter = iter(self._items)+ return self++ async def __anext__(self):+ try:+ return next(self._iter)+ except StopIteration as exc:+ raise StopAsyncIteration from exc+++def make_backend() -> LiteLLMBackend:+ with patch("headroom.backends.litellm._fetch_bedrock_inference_profiles", return_value={}):+ return LiteLLMBackend(provider="openrouter")+++def make_response() -> SimpleNamespace:+ return SimpleNamespace(+ id="resp_123",+ created=123456,+ choices=[+ SimpleNamespace(+ index=0,+ finish_reason="stop",+ message=SimpleNamespace(role="assistant", content="ok", tool_calls=None),+ )+ ],+ usage=SimpleNamespace(prompt_tokens=2, completion_tokens=3, total_tokens=5),+ )+++def request_body(**overrides):+ body = {+ "model": "qwen3",+ "messages": [{"role": "user", "content": "hello"}],+ "max_tokens": 32,+ }+ body.update(overrides)+ return body+++async def test_chat_template_kwargs_forwarded_buffered() -> None:+ backend = make_backend()++ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = make_response()++ await backend.send_openai_message(+ request_body(chat_template_kwargs={"enable_thinking": False}),+ {},+ )++ kwargs = mock_acomp.await_args.kwargs+ assert kwargs["max_tokens"] == 32+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}+++async def test_chat_template_kwargs_forwarded_streaming() -> None:+ backend = make_backend()++ stream = FakeAsyncStream(+ [+ SimpleNamespace(model_dump=lambda **kwargs: {"id": "chunk1", "choices": []}),+ ]+ )++ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = stream++ chunks = [+ chunk+ async for chunk in backend.stream_openai_message(+ request_body(+ chat_template_kwargs={"enable_thinking": False},+ stream_options={"include_usage": True},+ ),+ {},+ )+ ]++ kwargs = mock_acomp.await_args.kwargs+ assert kwargs["stream"] is True+ assert kwargs["stream_options"] == {"include_usage": True}+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}+ assert chunks[-1] == "data: [DONE]\n\n"+++async def test_standard_only_body_has_no_extra_body() -> None:+ backend = make_backend()++ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = make_response()++ await backend.send_openai_message(+ request_body(temperature=0.1, top_p=0.9),+ {},+ )++ kwargs = mock_acomp.await_args.kwargs+ assert "extra_body" not in kwargs+++async def test_standard_params_still_forwarded() -> None:+ backend = make_backend()++ with patch("headroom.backends.litellm.acompletion", new_callable=AsyncMock) as mock_acomp:+ mock_acomp.return_value = make_response()++ await backend.send_openai_message(+ request_body(+ temperature=0.1,+ top_p=0.9,+ response_format={"type": "json_object"},+ chat_template_kwargs={"enable_thinking": False},+ ),+ {},+ )++ kwargs = mock_acomp.await_args.kwargs+ assert kwargs["temperature"] == 0.1+ assert kwargs["top_p"] == 0.9+ assert kwargs["response_format"] == {"type": "json_object"}+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}