Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): keep mid-conversation system sections for third-party models (#4591)
R-Okauchi5ad236379853
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 5c21a4786..4fb3f85b5 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -66,6 +66,7 @@ from headroom.proxy.handlers._failure_logging import log_hook_failure, log_requefrom headroom.proxy.helpers import (extract_tags,invalid_request_body_message,+ is_third_party_model,relocate_system_messages_to_top_level,sanitize_forwarded_response_headers,)@@ -3883,6 +3884,8 @@ class AnthropicHandlerMixin:# compression). Anthropic rejects that with a 400 ("messages.0: use# the top-level 'system' parameter ..."), so relocate it back to the# top-level ``system`` parameter as the last step before forwarding.+ # A non-Claude model on a third-party upstream (configured target or+ # per-request base URL) keeps the client's valid sections in place.relocated_messages, relocated_system, system_relocated = (relocate_system_messages_to_top_level(body["messages"],@@ -3896,6 +3899,12 @@ class AnthropicHandlerMixin:)else None),+ third_party_model=is_third_party_model(+ model,+ third_party_upstream=bool(+ upstream_base_url or _third_party_anthropic_upstream+ ),+ ),))if system_relocated:diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex f138c5024..6ffd4cf16 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -1085,10 +1085,24 @@ def _system_message_to_blocks(message: dict[str, Any]) -> list[Any]:return []+def is_third_party_model(model: str | None, *, third_party_upstream: bool) -> bool:+ """Whether a request is for a non-Claude model on an upstream Headroom does not own.++ Headroom has no wire contract for such a model (a vLLM or GLM endpoint+ speaking the Anthropic Messages format), and the client already sends its+ mid-conversation system sections there directly. A Claude-named model+ behind a gateway may still reach a provider without that feature, so it is+ not one.+ """+ return third_party_upstream and "claude" not in str(model or "").lower()++def relocate_system_messages_to_top_level(messages: list[dict[str, Any]],system: Any,model: str | None = None,+ *,+ third_party_model: bool = False,) -> tuple[list[dict[str, Any]], Any, bool]:"""Relocate only system messages invalid for the selected Anthropic model.@@ -1098,6 +1112,12 @@ def relocate_system_messages_to_top_level(cached prefix. The initial/invalid forms are still moved to the top-levelfield as the issue-765 last-line wire-contract guard.+ ``third_party_model`` (see :func:`is_third_party_model`) treats the model as+ supported: the client's sections stay where it put them. Hoisting them+ there would append to the top-level ``system`` whenever a turn adds a+ section, so the upstream's prefix cache would miss the whole conversation+ on every such turn.+The relocated content is appended after any existing top-level ``system``so wire order (system prompt, then conversation) is preserved. Onlytext-shaped content moves: non-text blocks (images, documents) stay in a@@ -1111,7 +1131,7 @@ def relocate_system_messages_to_top_level(the common path is untouched."""model_id = str(model or "").lower()- supports_mid_conversation = any(+ supports_mid_conversation = third_party_model or any(family in model_idfor family in ("claude-fable-5",diff --git a/tests/test_anthropic_system_relocation_third_party.py b/tests/test_anthropic_system_relocation_third_party.pynew file mode 100644index 000000000..39e18fe49--- /dev/null+++ b/tests/test_anthropic_system_relocation_third_party.py@@ -0,0 +1,129 @@+"""The #765 relocation guard on third-party upstreams, through the real handler.++A non-Claude model behind a configured Anthropic target or a per-request+``x-headroom-base-url`` keeps its valid mid-conversation system sections where+the client put them. A Claude-named model there, and any model on Anthropic+itself, keeps the existing relocation.+"""++from __future__ import annotations++import copy+from typing import Any++import httpx+import pytest+from fastapi.testclient import TestClient++from headroom.proxy.server import HeadroomProxy, ProxyConfig, create_app++THIRD_PARTY_TARGET = "https://vllm.example"+OVERRIDE = "https://override.example"+SYSTEM = [{"type": "text", "text": "You are a coding agent."}]+MESSAGES = [+ {"role": "user", "content": "Run the tests."},+ {+ "role": "system",+ "content": "<system-reminder>tests/test_app.py was opened</system-reminder>",+ },+ {"role": "assistant", "content": "Running them now."},+ {"role": "user", "content": "And fix what fails."},+]++[email protected](autouse=True)+def _isolate_upstream(monkeypatch: pytest.MonkeyPatch) -> None:+ # create_app() writes the resolved target onto the class; put it back after.+ monkeypatch.setattr(HeadroomProxy, "ANTHROPIC_API_URL", HeadroomProxy.ANTHROPIC_API_URL)+ monkeypatch.setenv("HEADROOM_ALLOWED_BASE_URLS", "override.example")+++def _forward(model: str, *, api_url: str | None = None, base_url: str | None = None) -> dict:+ """Send one request through the handler; return the URL and body it forwarded."""+ app = create_app(+ ProxyConfig(+ optimize=True,+ mode="cache",+ disable_kompress=True,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ anthropic_api_url=api_url,+ )+ )+ forwarded: dict[str, Any] = {}+ with TestClient(app) as client:+ proxy = client.app.state.proxy++ async def _upstream(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001, ANN003, ANN202+ forwarded.update(url=url, body=copy.deepcopy(body))+ return httpx.Response(+ 200,+ json={+ "id": "msg_x",+ "type": "message",+ "role": "assistant",+ "model": model,+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 10, "output_tokens": 2},+ },+ )++ proxy._retry_request = _upstream+ headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}+ if base_url:+ headers["x-headroom-base-url"] = base_url+ response = client.post(+ "/v1/messages",+ headers=headers,+ json={+ "model": model,+ "max_tokens": 16,+ "system": SYSTEM,+ "messages": copy.deepcopy(MESSAGES),+ },+ )+ assert response.status_code == 200, response.text[:300]+ return forwarded+++ ("upstream", "target"),+ [+ ({"api_url": THIRD_PARTY_TARGET}, THIRD_PARTY_TARGET),+ ({"base_url": OVERRIDE}, OVERRIDE),+ ],+)+def test_third_party_model_keeps_mid_conversation_system_sections(+ upstream: dict[str, str], target: str+) -> None:+ forwarded = _forward("qwen3.8-flash-next", **upstream)++ assert forwarded["url"].startswith(target)+ # Forwarded where the client put it; the top-level system is untouched, so+ # it stays the same from turn to turn and the upstream's prefix cache holds.+ assert forwarded["body"]["messages"] == MESSAGES+ assert forwarded["body"]["system"] == SYSTEM+++ ("model", "upstream"),+ [+ # A Claude-named model behind a gateway keeps the conservative hoist.+ ("claude-sonnet-4-6", {"api_url": THIRD_PARTY_TARGET}),+ ("claude-sonnet-4-6", {"base_url": OVERRIDE}),+ # Anthropic itself keeps the model allow-list.+ ("qwen3.8-flash-next", {}),+ ],+)+def test_other_requests_still_relocate_mid_conversation_system_sections(+ model: str, upstream: dict[str, str]+) -> None:+ forwarded = _forward(model, **upstream)++ assert [m["role"] for m in forwarded["body"]["messages"]] == ["user", "assistant", "user"]+ assert forwarded["body"]["system"] == [+ *SYSTEM,+ {"type": "text", "text": MESSAGES[1]["content"]},+ ]diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex 40a9312df..1396bf025 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -20,6 +20,7 @@ from headroom.proxy.handlers.openai import ()from headroom.proxy.helpers import (_headroom_bypass_enabled,+ is_third_party_model,relocate_system_messages_to_top_level,)from headroom.proxy.outcome import emit_request_outcome@@ -415,6 +416,71 @@ def test_relocate_system_messages_moves_valid_shape_for_unsupported_model(model:assert system == [{"type": "text", "text": "mid-turn instruction"}]+ ("model", "third_party_upstream", "expected"),+ [+ ("qwen3.8-flash-next", True, True),+ ("glm-5.3", True, True),+ # Claude-named models behind a gateway keep the conservative hoist.+ ("claude-opus-5", True, False),+ ("anthropic/claude-sonnet-4-6", True, False),+ ("global.anthropic.claude-sonnet-5-v1:0", True, False),+ # Anthropic itself (or Vertex / a backend) keeps the model allow-list.+ ("qwen3.8-flash-next", False, False),+ ],+)+def test_is_third_party_model(model: str, third_party_upstream: bool, expected: bool) -> None:+ assert is_third_party_model(model, third_party_upstream=third_party_upstream) is expected+++def test_relocate_system_messages_keeps_valid_sections_for_third_party_model() -> None:+ # #3552 kept non-text blocks in place for third-party upstreams, but text+ # sections were still hoisted for every model there. A client that appends a+ # section most turns changed the top-level system prompt on each request, and+ # the upstream's prefix cache missed the whole conversation every turn.+ messages = [+ {"role": "user", "content": "Run the tests."},+ {"role": "system", "content": "The user added: update the changelog too."},+ {"role": "assistant", "content": "I will do both."},+ {"role": "user", "content": [{"type": "tool_result", "content": "ok"}]},+ {"role": "system", "content": "<system-reminder>todo list changed</system-reminder>"},+ ]++ clean, system, changed = relocate_system_messages_to_top_level(+ messages, "base", "qwen3.8-flash-next", third_party_model=True+ )++ assert changed is False+ assert clean is messages+ assert system == "base"+++ "messages",+ [+ # The #765 case: a system block left at messages[0] is invalid anywhere.+ [+ {"role": "system", "content": "You are a harness."},+ {"role": "user", "content": "hi"},+ ],+ [+ {"role": "assistant", "content": "answer"},+ {"role": "system", "content": "You are a harness."},+ ],+ ],+)+def test_relocate_system_messages_still_moves_invalid_placement_for_third_party_model(+ messages: list[dict],+) -> None:+ clean, system, changed = relocate_system_messages_to_top_level(+ messages, None, "qwen3.8-flash-next", third_party_model=True+ )++ assert changed is True+ assert all(message.get("role") != "system" for message in clean)+ assert system == [{"type": "text", "text": "You are a harness."}]++def test_relocate_system_messages_keeps_image_blocks_out_of_top_level_system() -> None:image_block = {"type": "image",