Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): keep mid-conversation system sections for third-party models (#4591)

R-Okauchi5ad236379853
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 5c21a4786..4fb3f85b5 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -66,6 +66,7 @@ from headroom.proxy.handlers._failure_logging import log_hook_failure, log_reque
from headroom.proxy.helpers import (
extract_tags,
invalid_request_body_message,
+ is_third_party_model,
relocate_system_messages_to_top_level,
sanitize_forwarded_response_headers,
)
@@ -3883,6 +3884,8 @@ class AnthropicHandlerMixin:
# compression). Anthropic rejects that with a 400 ("messages.0: use
# the top-level 'system' parameter ..."), so relocate it back to the
# top-level ``system`` parameter as the last step before forwarding.
+ # A non-Claude model on a third-party upstream (configured target or
+ # per-request base URL) keeps the client's valid sections in place.
relocated_messages, relocated_system, system_relocated = (
relocate_system_messages_to_top_level(
body["messages"],
@@ -3896,6 +3899,12 @@ class AnthropicHandlerMixin:
)
else None
),
+ third_party_model=is_third_party_model(
+ model,
+ third_party_upstream=bool(
+ upstream_base_url or _third_party_anthropic_upstream
+ ),
+ ),
)
)
if system_relocated:
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index f138c5024..6ffd4cf16 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -1085,10 +1085,24 @@ def _system_message_to_blocks(message: dict[str, Any]) -> list[Any]:
return []
+def is_third_party_model(model: str | None, *, third_party_upstream: bool) -> bool:
+ """Whether a request is for a non-Claude model on an upstream Headroom does not own.
+
+ Headroom has no wire contract for such a model (a vLLM or GLM endpoint
+ speaking the Anthropic Messages format), and the client already sends its
+ mid-conversation system sections there directly. A Claude-named model
+ behind a gateway may still reach a provider without that feature, so it is
+ not one.
+ """
+ return third_party_upstream and "claude" not in str(model or "").lower()
+
+
def relocate_system_messages_to_top_level(
messages: list[dict[str, Any]],
system: Any,
model: str | None = None,
+ *,
+ third_party_model: bool = False,
) -> tuple[list[dict[str, Any]], Any, bool]:
"""Relocate only system messages invalid for the selected Anthropic model.
@@ -1098,6 +1112,12 @@ def relocate_system_messages_to_top_level(
cached prefix. The initial/invalid forms are still moved to the top-level
field as the issue-765 last-line wire-contract guard.
+ ``third_party_model`` (see :func:`is_third_party_model`) treats the model as
+ supported: the client's sections stay where it put them. Hoisting them
+ there would append to the top-level ``system`` whenever a turn adds a
+ section, so the upstream's prefix cache would miss the whole conversation
+ on every such turn.
+
The relocated content is appended after any existing top-level ``system``
so wire order (system prompt, then conversation) is preserved. Only
text-shaped content moves: non-text blocks (images, documents) stay in a
@@ -1111,7 +1131,7 @@ def relocate_system_messages_to_top_level(
the common path is untouched.
"""
model_id = str(model or "").lower()
- supports_mid_conversation = any(
+ supports_mid_conversation = third_party_model or any(
family in model_id
for family in (
"claude-fable-5",
diff --git a/tests/test_anthropic_system_relocation_third_party.py b/tests/test_anthropic_system_relocation_third_party.py
new file mode 100644
index 000000000..39e18fe49
--- /dev/null
+++ b/tests/test_anthropic_system_relocation_third_party.py
@@ -0,0 +1,129 @@
+"""The #765 relocation guard on third-party upstreams, through the real handler.
+
+A non-Claude model behind a configured Anthropic target or a per-request
+``x-headroom-base-url`` keeps its valid mid-conversation system sections where
+the client put them. A Claude-named model there, and any model on Anthropic
+itself, keeps the existing relocation.
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+import httpx
+import pytest
+from fastapi.testclient import TestClient
+
+from headroom.proxy.server import HeadroomProxy, ProxyConfig, create_app
+
+THIRD_PARTY_TARGET = "https://vllm.example"
+OVERRIDE = "https://override.example"
+SYSTEM = [{"type": "text", "text": "You are a coding agent."}]
+MESSAGES = [
+ {"role": "user", "content": "Run the tests."},
+ {
+ "role": "system",
+ "content": "<system-reminder>tests/test_app.py was opened</system-reminder>",
+ },
+ {"role": "assistant", "content": "Running them now."},
+ {"role": "user", "content": "And fix what fails."},
+]
+
+
[email protected](autouse=True)
+def _isolate_upstream(monkeypatch: pytest.MonkeyPatch) -> None:
+ # create_app() writes the resolved target onto the class; put it back after.
+ monkeypatch.setattr(HeadroomProxy, "ANTHROPIC_API_URL", HeadroomProxy.ANTHROPIC_API_URL)
+ monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", "override.example")
+
+
+def _forward(model: str, *, api_url: str | None = None, base_url: str | None = None) -> dict:
+ """Send one request through the handler; return the URL and body it forwarded."""
+ app = create_app(
+ ProxyConfig(
+ optimize=True,
+ mode="cache",
+ disable_kompress=True,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ anthropic_api_url=api_url,
+ )
+ )
+ forwarded: dict[str, Any] = {}
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+
+ async def _upstream(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN003, ANN202
+ forwarded.update(url=url, body=copy.deepcopy(body))
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_x",
+ "type": "message",
+ "role": "assistant",
+ "model": model,
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 10, "output_tokens": 2},
+ },
+ )
+
+ proxy._retry_request = _upstream
+ headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
+ if base_url:
+ headers["x-headroom-base-url"] = base_url
+ response = client.post(
+ "/v1/messages",
+ headers=headers,
+ json={
+ "model": model,
+ "max_tokens": 16,
+ "system": SYSTEM,
+ "messages": copy.deepcopy(MESSAGES),
+ },
+ )
+ assert response.status_code == 200, response.text[:300]
+ return forwarded
+
+
+ ("upstream", "target"),
+ [
+ ({"api_url": THIRD_PARTY_TARGET}, THIRD_PARTY_TARGET),
+ ({"base_url": OVERRIDE}, OVERRIDE),
+ ],
+)
+def test_third_party_model_keeps_mid_conversation_system_sections(
+ upstream: dict[str, str], target: str
+) -> None:
+ forwarded = _forward("qwen3.8-flash-next", **upstream)
+
+ assert forwarded["url"].startswith(target)
+ # Forwarded where the client put it; the top-level system is untouched, so
+ # it stays the same from turn to turn and the upstream's prefix cache holds.
+ assert forwarded["body"]["messages"] == MESSAGES
+ assert forwarded["body"]["system"] == SYSTEM
+
+
+ ("model", "upstream"),
+ [
+ # A Claude-named model behind a gateway keeps the conservative hoist.
+ ("claude-sonnet-4-6", {"api_url": THIRD_PARTY_TARGET}),
+ ("claude-sonnet-4-6", {"base_url": OVERRIDE}),
+ # Anthropic itself keeps the model allow-list.
+ ("qwen3.8-flash-next", {}),
+ ],
+)
+def test_other_requests_still_relocate_mid_conversation_system_sections(
+ model: str, upstream: dict[str, str]
+) -> None:
+ forwarded = _forward(model, **upstream)
+
+ assert [m["role"] for m in forwarded["body"]["messages"]] == ["user", "assistant", "user"]
+ assert forwarded["body"]["system"] == [
+ *SYSTEM,
+ {"type": "text", "text": MESSAGES[1]["content"]},
+ ]
diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.py
index 40a9312df..1396bf025 100644
--- a/tests/test_proxy_handler_helpers.py
+++ b/tests/test_proxy_handler_helpers.py
@@ -20,6 +20,7 @@ from headroom.proxy.handlers.openai import (
)
from headroom.proxy.helpers import (
_headroom_bypass_enabled,
+ is_third_party_model,
relocate_system_messages_to_top_level,
)
from headroom.proxy.outcome import emit_request_outcome
@@ -415,6 +416,71 @@ def test_relocate_system_messages_moves_valid_shape_for_unsupported_model(model:
assert system == [{"type": "text", "text": "mid-turn instruction"}]
+ ("model", "third_party_upstream", "expected"),
+ [
+ ("qwen3.8-flash-next", True, True),
+ ("glm-5.3", True, True),
+ # Claude-named models behind a gateway keep the conservative hoist.
+ ("claude-opus-5", True, False),
+ ("anthropic/claude-sonnet-4-6", True, False),
+ ("global.anthropic.claude-sonnet-5-v1:0", True, False),
+ # Anthropic itself (or Vertex / a backend) keeps the model allow-list.
+ ("qwen3.8-flash-next", False, False),
+ ],
+)
+def test_is_third_party_model(model: str, third_party_upstream: bool, expected: bool) -> None:
+ assert is_third_party_model(model, third_party_upstream=third_party_upstream) is expected
+
+
+def test_relocate_system_messages_keeps_valid_sections_for_third_party_model() -> None:
+ # #3552 kept non-text blocks in place for third-party upstreams, but text
+ # sections were still hoisted for every model there. A client that appends a
+ # section most turns changed the top-level system prompt on each request, and
+ # the upstream's prefix cache missed the whole conversation every turn.
+ messages = [
+ {"role": "user", "content": "Run the tests."},
+ {"role": "system", "content": "The user added: update the changelog too."},
+ {"role": "assistant", "content": "I will do both."},
+ {"role": "user", "content": [{"type": "tool_result", "content": "ok"}]},
+ {"role": "system", "content": "<system-reminder>todo list changed</system-reminder>"},
+ ]
+
+ clean, system, changed = relocate_system_messages_to_top_level(
+ messages, "base", "qwen3.8-flash-next", third_party_model=True
+ )
+
+ assert changed is False
+ assert clean is messages
+ assert system == "base"
+
+
+ "messages",
+ [
+ # The #765 case: a system block left at messages[0] is invalid anywhere.
+ [
+ {"role": "system", "content": "You are a harness."},
+ {"role": "user", "content": "hi"},
+ ],
+ [
+ {"role": "assistant", "content": "answer"},
+ {"role": "system", "content": "You are a harness."},
+ ],
+ ],
+)
+def test_relocate_system_messages_still_moves_invalid_placement_for_third_party_model(
+ messages: list[dict],
+) -> None:
+ clean, system, changed = relocate_system_messages_to_top_level(
+ messages, None, "qwen3.8-flash-next", third_party_model=True
+ )
+
+ assert changed is True
+ assert all(message.get("role") != "system" for message in clean)
+ assert system == [{"type": "text", "text": "You are a harness."}]
+
+
def test_relocate_system_messages_keeps_image_blocks_out_of_top_level_system() -> None:
image_block = {
"type": "image",