Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

53.4 KB
Raw Blame
import json
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from itertools import chain
from typing import Final, cast

import httpx
import openai
import pytest
from integration._support import bedrock_runtime_peer
from integration._support import prompt_cache_breakpoint as pcb
from integration._support.client import Gateway, Scenario, object_value
from integration._support.openai_wire import answering_model_discovery, chat_reply
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire, wire_server
from integration.providers._cache_control_marks_support import (
    ANTHROPIC_MODEL,
    ASK,
    ASK_LABEL,
    BEDROCK_MODEL,
    CITIES,
    EPHEMERAL,
    POINTS,
    PROVIDER_KEY,
    SYSTEM,
    SYSTEM_LABEL,
    UNMAPPED_BEDROCK_ARN,
    anthropic_labels,
    anthropic_marks,
    anthropic_peer,
    bedrock_labels,
    bedrock_peer,
    call_id,
    client_marked,
    final_label,
    final_text,
    marker_of,
    new_marker,
    responses_body,
    sent_without_cache_points,
    tool_use_label,
)
from openai.types.responses import ResponseInputParam
from pydantic import JsonValue, TypeAdapter

from litellm.utils import get_prompt_cache_min_tokens

_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_HOUR: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"}
_CHECK: Final = "I will check."
_ASSISTANT_LABEL: Final = f"assistant:text:{_CHECK}"
_RESULT_LABEL: Final = f"user:tool_result:{call_id(CITIES[0])}"
_BEDROCK_SYSTEM_LABEL: Final = f"system:text:{SYSTEM}"
_BEDROCK_RESULT_LABEL: Final = f"user:toolResult:{call_id(CITIES[0])}"
_TOOL: Final[dict[str, JsonValue]] = {"type": "function", "name": "lookup_weather", "parameters": {"type": "object"}}
_GEMINI: Final = "gemini/gemini-3.8-flash"
_GEMINI_GENERATE_ONLY: Final = [("POST", "/models/gemini-3.8-flash:generateContent")]
_GEMINI_REPLY: Final = json.dumps(
    {
        "candidates": [{"content": {"role": "model", "parts": [{"text": "sunny"}]}, "finishReason": "STOP"}],
        "usageMetadata": {"promptTokenCount": 1600, "candidatesTokenCount": 1, "totalTokenCount": 1601},
    }
).encode()
_OPENAI_COMPATIBLE: Final = "openai/gpt-5.4"


@dataclass(frozen=True, slots=True)
class _Answer:
    status: int
    text: str
    call_id: str


@pytest.fixture(scope="module")
def spend() -> Iterator[pcb.SpendLogs]:
    with pcb.spend_logs() as logs:
        yield logs


def _item(role: str, text: str, mark: JsonValue = EPHEMERAL) -> dict[str, JsonValue]:
    return {"role": role, "content": text, "cache_control": mark}


def _responses(model: str, items: Sequence[JsonValue], **fields: JsonValue) -> dict[str, JsonValue]:
    return {"model": model, "max_output_tokens": 64, "input": list(items), **fields}


def _marked_system(model: str, marker: str, mark: JsonValue = EPHEMERAL, **fields: JsonValue) -> dict[str, JsonValue]:
    return _responses(model, [_item("system", SYSTEM, mark), {"role": "user", "content": final_text(marker)}], **fields)


def _role_items(role: str, marker: str) -> list[JsonValue]:
    match role:
        case "system" | "developer":
            return [_item(role, SYSTEM), {"role": "user", "content": final_text(marker)}]
        case "user":
            return [_item("user", ASK), {"role": "user", "content": final_text(marker)}]
        case _:
            return [
                {"role": "user", "content": ASK},
                _item("assistant", _CHECK),
                {"role": "user", "content": final_text(marker)},
            ]


def _tool_items(marker: str, *, output_mark: JsonValue = None, call_mark: JsonValue = None) -> list[JsonValue]:
    call: Final[dict[str, JsonValue]] = {
        "type": "function_call",
        "call_id": call_id(CITIES[0]),
        "name": "lookup_weather",
        "arguments": json.dumps({"city": CITIES[0]}),
    }
    output: Final[dict[str, JsonValue]] = {
        "type": "function_call_output",
        "call_id": call_id(CITIES[0]),
        "output": "sunny.",
    }
    return [
        {"role": "user", "content": ASK},
        {**call, "cache_control": call_mark} if call_mark is not None else call,
        {**output, "cache_control": output_mark} if output_mark is not None else output,
        {"role": "user", "content": final_text(marker)},
    ]


def _call_id_of(response: httpx.Response) -> str:
    return response.headers["x-litellm-call-id"] if "x-litellm-call-id" in response.headers else ""


def _send(gateway: Gateway, body: dict[str, JsonValue], *, key: str | None = None) -> _Answer:
    headers: Final = {"Authorization": f"Bearer {gateway.key if key is None else key}"}
    with gateway.client.stream("POST", "/v1/responses", json=body, headers=headers) as response:
        return _Answer(response.status_code, response.read().decode(), _call_id_of(response))


def _send_raw(gateway: Gateway, raw: bytes) -> _Answer:
    headers: Final = {"Authorization": f"Bearer {gateway.key}", "Content-Type": "application/json"}
    response: Final = gateway.client.post("/v1/responses", content=raw, headers=headers)
    return _Answer(response.status_code, response.text, _call_id_of(response))


def _assert_completed(answer: _Answer, *, stream: bool = False) -> None:
    assert answer.status == 200, answer.text
    assert "sunny" in answer.text, answer.text
    if stream:
        assert '"type":"response.completed"' in answer.text, answer.text
        return
    assert _JSON_OBJECT.validate_json(answer.text)["status"] == "completed", answer.text


def _only_request(wire: Wire) -> Request:
    received: Final = wire.drain()
    assert len(received) == 1, [request.target for request in received]
    return received[0]


def _anthropic_deployment(scenario: Scenario, wire: Wire, *, inject: bool = False) -> str:
    if inject:
        return scenario.model(
            model=f"anthropic/{ANTHROPIC_MODEL}",
            api_base=wire.url,
            api_key=PROVIDER_KEY,
            cache_control_injection_points=POINTS,
        )
    return scenario.model(model=f"anthropic/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY)


def _bedrock_peer(request: Request) -> Reply:
    if request.target.endswith("/converse-stream"):
        return bedrock_runtime_peer.respond(request)
    return bedrock_peer(request)


def _bedrock_deployment(scenario: Scenario, wire: Wire, route: str = "converse") -> str:
    return scenario.model(
        model=f"bedrock/{route}/{BEDROCK_MODEL}",
        api_key=PROVIDER_KEY,
        api_base=wire.url,
        aws_region_name="us-east-1",
        aws_bedrock_runtime_endpoint=wire.url,
    )


def _unmapped_arn_deployment(scenario: Scenario, wire: Wire, **fields: JsonValue) -> str:
    return scenario.model(
        model=f"bedrock/converse/{UNMAPPED_BEDROCK_ARN}",
        api_key=PROVIDER_KEY,
        api_base=wire.url,
        aws_region_name="us-east-1",
        aws_bedrock_runtime_endpoint=wire.url,
        **fields,
    )


def _blocks_of(body: dict[str, JsonValue]) -> Iterator[JsonValue]:
    system: Final = body.get("system")
    yield from system if isinstance(system, list) else []
    messages: Final = body.get("messages")
    for message in messages if isinstance(messages, list) else []:
        yield from _content_blocks(object_value(message).get("content"))


def _content_blocks(content: JsonValue) -> Iterator[JsonValue]:
    yield from content if isinstance(content, list) else []


def _bedrock_ttls(request: Request) -> list[JsonValue]:
    blocks: Final = list(_blocks_of(_JSON_OBJECT.validate_json(request.body)))
    return [
        object_value(block["cachePoint"]).get("ttl")
        for block in blocks
        if isinstance(block, dict) and "cachePoint" in block
    ]


@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream"))
@pytest.mark.parametrize(
    ("role", "expected"),
    (
        pytest.param("system", SYSTEM_LABEL, id="system"),
        pytest.param("developer", SYSTEM_LABEL, id="developer"),
        pytest.param("user", ASK_LABEL, id="user"),
        pytest.param("assistant", _ASSISTANT_LABEL, id="assistant"),
    ),
)
def test_marked_item_reaches_anthropic_as_a_cache_point(
    gateway: Gateway, spend: pcb.SpendLogs, role: str, expected: str, stream: bool
) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        answer: Final = _send(gateway, _responses(model, _role_items(role, marker), stream=stream))
        _assert_completed(answer, stream=stream)
        assert anthropic_labels(_only_request(wire)) == [expected]
        spend.landed(model, answer.call_id, None)


def test_list_content_message_mark_lands_on_the_last_block(gateway: Gateway) -> None:
    marker: Final = new_marker()
    content: Final[list[JsonValue]] = [{"type": "input_text", "text": "first"}, {"type": "input_text", "text": ASK}]
    items: Final[list[JsonValue]] = [
        {"role": "user", "content": content, "cache_control": EPHEMERAL},
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _responses(model, items)))
        assert anthropic_labels(_only_request(wire)) == [ASK_LABEL]


def test_block_mark_outranks_the_message_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    items: Final[list[JsonValue]] = [
        {
            "role": "user",
            "content": [{"type": "input_text", "text": ASK, "cache_control": _HOUR}],
            "cache_control": EPHEMERAL,
        },
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _responses(model, items)))
        marks: Final = anthropic_marks(_only_request(wire))
    assert [(mark.label, mark.ttl) for mark in marks] == [(ASK_LABEL, "1h")]


def test_function_call_output_mark_reaches_anthropic_as_a_tool_result_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = _responses(model, _tool_items(marker, output_mark=EPHEMERAL), tools=[_TOOL])
        _assert_completed(_send(gateway, body))
        assert anthropic_labels(_only_request(wire)) == [_RESULT_LABEL]


def test_function_call_marks_reach_anthropic_as_tool_use_marks(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, responses_body(model, marker)))
        assert anthropic_labels(_only_request(wire)) == client_marked()


@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream"))
@pytest.mark.parametrize(
    ("role", "expected"),
    (
        pytest.param("system", _BEDROCK_SYSTEM_LABEL, id="system"),
        pytest.param("user", f"user:text:{ASK}", id="user"),
        pytest.param("assistant", f"assistant:text:{_CHECK}", id="assistant"),
    ),
)
def test_marked_item_reaches_bedrock_converse_as_a_cache_point(
    gateway: Gateway, role: str, expected: str, stream: bool
) -> None:
    marker: Final = new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = _bedrock_deployment(scenario, wire)
        answer: Final = _send(gateway, _responses(model, _role_items(role, marker), stream=stream))
        assert answer.status == 200, answer.text
        assert f"marker-{marker}" in answer.text or "sunny" in answer.text, answer.text
        request: Final = _only_request(wire)
    assert request.target.endswith("/converse-stream" if stream else "/converse"), request.target
    assert bedrock_labels(request) == [expected]


def test_function_call_output_mark_reaches_bedrock_as_a_tool_result_cache_point(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = _bedrock_deployment(scenario, wire)
        body: Final = _responses(model, _tool_items(marker, output_mark=EPHEMERAL), tools=[_TOOL])
        answer: Final = _send(gateway, body)
        assert answer.status == 200, answer.text
        assert bedrock_labels(_only_request(wire)) == [_BEDROCK_RESULT_LABEL]


@pytest.mark.parametrize("drop_params", (False, True), ids=("as-sent", "drop-params"))
def test_nova_tool_marks_follow_drop_params(gateway: Gateway, drop_params: bool) -> None:
    marker: Final = new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model="bedrock/converse/us.amazon.nova-2-lite-v1:0",
            api_key=PROVIDER_KEY,
            api_base=wire.url,
            aws_region_name="us-east-1",
            aws_bedrock_runtime_endpoint=wire.url,
            drop_params=drop_params,
        )
        items: Final = _tool_items(marker, output_mark=EPHEMERAL, call_mark=EPHEMERAL)
        answer: Final = _send(gateway, _responses(model, items, tools=[_TOOL]))
        assert answer.status == 200, answer.text
        labels: Final = bedrock_labels(_only_request(wire))
    assert (_BEDROCK_RESULT_LABEL in labels) is not drop_params, labels
    assert (len(labels) == 2) is not drop_params, labels


def test_one_hour_mark_keeps_its_ttl_on_anthropic(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _marked_system(model, marker, _HOUR)))
        marks: Final = anthropic_marks(_only_request(wire))
    assert [(mark.label, mark.ttl) for mark in marks] == [(SYSTEM_LABEL, "1h")]


def test_one_hour_mark_keeps_its_ttl_on_bedrock(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = _bedrock_deployment(scenario, wire)
        answer: Final = _send(gateway, _marked_system(model, marker, _HOUR))
        assert answer.status == 200, answer.text
        request: Final = _only_request(wire)
    assert bedrock_labels(request) == [_BEDROCK_SYSTEM_LABEL]
    assert _bedrock_ttls(request) == ["1h"]


def _openai_client(gateway: Gateway) -> openai.OpenAI:
    return openai.OpenAI(
        base_url=f"{gateway.client.base_url}/v1",
        api_key=gateway.key,
        max_retries=0,
        http_client=httpx.Client(trust_env=False, timeout=60),
    )


def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI:
    return openai.AsyncOpenAI(
        base_url=f"{gateway.client.base_url}/v1",
        api_key=gateway.key,
        max_retries=0,
        http_client=httpx.AsyncClient(trust_env=False, timeout=60),
    )


def _sdk_input(marker: str) -> ResponseInputParam:
    return cast(ResponseInputParam, _role_items("system", marker))


def test_openai_sdk_sync_responses_carries_the_system_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        with _openai_client(gateway) as client:
            response: Final = client.responses.create(model=model, input=_sdk_input(marker), max_output_tokens=64)
        assert response.status == "completed", response.model_dump()
        assert response.output_text == "sunny", response.model_dump()
        assert anthropic_labels(_only_request(wire)) == [SYSTEM_LABEL]


async def test_openai_sdk_async_stream_responses_carries_the_system_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        async with _async_openai_client(gateway) as client:
            stream: Final = await client.responses.create(
                model=model, input=_sdk_input(marker), max_output_tokens=64, stream=True
            )
            events: Final = [event async for event in stream]
        assert [event.type for event in events][-1] == "response.completed", events
        assert anthropic_labels(_only_request(wire)) == [SYSTEM_LABEL]


def _gemini_peer(request: Request) -> Reply:
    if "cachedContents" in request.target and request.method == "GET":
        return Reply(body=b'{"cachedContents":[]}')
    if "cachedContents" in request.target:
        return Reply(
            body=json.dumps(
                {
                    "name": "cachedContents/bridge",
                    "model": "models/gemini-3.8-flash",
                    "expireTime": "2099-01-01T00:00:00Z",
                }
            ).encode()
        )
    return Reply(body=_GEMINI_REPLY)


@pytest.mark.parametrize(
    "fields",
    (
        pytest.param({}, id="bare"),
        pytest.param({"instructions": SYSTEM, "tools": [_TOOL]}, id="instructions-and-tools"),
    ),
)
def test_gemini_marked_items_never_start_a_context_cache(gateway: Gateway, fields: dict[str, JsonValue]) -> None:
    marker: Final = new_marker()
    long_system: Final = "lorem " * (2 * get_prompt_cache_min_tokens(_GEMINI))
    items: Final[list[JsonValue]] = [
        _item("system", long_system),
        _item("developer", SYSTEM),
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(_gemini_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=_GEMINI, api_base=wire.url, api_key=PROVIDER_KEY)
        _assert_completed(_send(gateway, _responses(model, items, **fields)))
        targets: Final = [(request.method, request.target.split("?")[0]) for request in wire.drain()]
    assert targets == _GEMINI_GENERATE_ONLY


def test_azure_ai_claude_marked_system_item_reaches_the_anthropic_route(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(model=f"azure_ai/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY)
        _assert_completed(_send(gateway, _marked_system(model, marker)))
        request: Final = _only_request(wire)
    assert request.target == "/anthropic/v1/messages", request.target
    assert anthropic_labels(request) == [SYSTEM_LABEL]


def test_bedrock_invoke_marked_system_item_reaches_the_invoke_body(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _bedrock_deployment(scenario, wire, route="invoke")
        _assert_completed(_send(gateway, _marked_system(model, marker)))
        request: Final = _only_request(wire)
    assert request.target.endswith("/invoke"), request.target
    assert anthropic_labels(request) == [SYSTEM_LABEL]


def _openai_message_marks(request: Request) -> list[str]:
    body: Final = _JSON_OBJECT.validate_json(request.body)
    messages: Final = body["messages"] if isinstance(body["messages"], list) else []
    return [
        f"cache_control@{object_value(message).get('role')}:message"
        for message in messages
        if "cache_control" in object_value(message)
    ]


def _openai_block_marks(request: Request) -> list[JsonValue]:
    return [object_value(block).get("cache_control") for block in _blocks_of(_JSON_OBJECT.validate_json(request.body))]


def test_databricks_claude_keeps_the_item_mark_like_its_chat_route(gateway: Gateway) -> None:
    marker: Final = new_marker()
    reply: Final = chat_reply(f"chatcmpl-{marker}", "databricks-claude-sonnet-4", "sunny", stream=False)
    with wire_server(answering_model_discovery(lambda request: reply)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model="databricks/databricks-claude-sonnet-4", api_base=wire.url, api_key=PROVIDER_KEY
        )
        _assert_completed(_send(gateway, _marked_system(model, marker)))
        posted: Final = [request for request in wire.drain() if request.method == "POST"]
    assert [request.target for request in posted] == ["/chat/completions"], [request.target for request in posted]
    assert _openai_block_marks(posted[0]) == [EPHEMERAL]


def test_openai_compatible_chat_bridge_leaves_the_item_mark_off(gateway: Gateway) -> None:
    marker: Final = new_marker()
    reply: Final = chat_reply(f"chatcmpl-{marker}", "gpt-5.4", "sunny", stream=False)
    with wire_server(answering_model_discovery(lambda request: reply)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=_OPENAI_COMPATIBLE, api_base=f"{wire.url}/v1", api_key=PROVIDER_KEY, use_chat_completions_api=True
        )
        _assert_completed(_send(gateway, _marked_system(model, marker)))
        posted: Final = [request for request in wire.drain() if request.method == "POST"]
    assert [request.target for request in posted] == ["/v1/chat/completions"], [request.target for request in posted]
    assert _openai_message_marks(posted[0]) == []


def test_injection_points_keep_the_caller_one_hour_system_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire, inject=True)
        _assert_completed(_send(gateway, _marked_system(model, marker, _HOUR)))
        marks: Final = anthropic_marks(_only_request(wire))
    assert [(mark.label, mark.ttl) for mark in marks] == [(SYSTEM_LABEL, "1h"), (final_label(marker), None)]


@pytest.mark.parametrize(
    "mark",
    (
        pytest.param({}, id="empty-object"),
        pytest.param(None, id="null"),
        pytest.param("", id="empty-string"),
        pytest.param(0, id="zero"),
        pytest.param(False, id="false"),
    ),
)
def test_falsy_marks_are_dropped(gateway: Gateway, mark: JsonValue) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _marked_system(model, marker, mark)))
        assert anthropic_labels(_only_request(wire)) == []


def test_missing_mark_is_dropped(gateway: Gateway) -> None:
    marker: Final = new_marker()
    items: Final[list[JsonValue]] = [
        {"role": "system", "content": SYSTEM},
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _responses(model, items)))
        assert anthropic_labels(_only_request(wire)) == []


def test_response_cache_serves_the_marked_request_once(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = _marked_system(model, marker)
        answers: Final = tuple(_send(gateway, body) for _ in range(2))
        received: Final = wire.drain()
    for answer in answers:
        _assert_completed(answer)
    assert [anthropic_labels(request) for request in received] == [[SYSTEM_LABEL]]


def _chat_with_message_mark(model: str, marker: str, mark: JsonValue = EPHEMERAL) -> dict[str, JsonValue]:
    return {
        "model": model,
        "max_tokens": 64,
        "messages": [
            {"role": "system", "content": SYSTEM, "cache_control": mark},
            {"role": "user", "content": final_text(marker)},
        ],
    }


def _messages_with_system_mark(model: str, marker: str) -> dict[str, JsonValue]:
    return {
        "model": model,
        "max_tokens": 64,
        "system": [{"type": "text", "text": SYSTEM, "cache_control": EPHEMERAL}],
        "messages": [{"role": "user", "content": final_text(marker)}],
    }


@pytest.mark.parametrize("path", ("/v1/chat/completions", "/v1/messages"))
def test_message_marks_on_the_other_endpoints_are_unchanged(gateway: Gateway, path: str) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = (
            _chat_with_message_mark(model, marker)
            if path == "/v1/chat/completions"
            else _messages_with_system_mark(model, marker)
        )
        response: Final = gateway.request("POST", path, body)
        assert response.status_code == 200, response.text
        assert anthropic_labels(_only_request(wire)) == [SYSTEM_LABEL]


def test_block_mark_on_responses_input_is_unchanged(gateway: Gateway) -> None:
    marker: Final = new_marker()
    items: Final[list[JsonValue]] = [
        {"role": "user", "content": [{"type": "input_text", "text": ASK, "cache_control": EPHEMERAL}]},
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _responses(model, items, instructions=SYSTEM)))
        assert anthropic_labels(_only_request(wire)) == [ASK_LABEL]


def _non_object_marks(request: Request) -> list[JsonValue]:
    blocks: Final = list(_blocks_of(_JSON_OBJECT.validate_json(request.body)))
    return [
        block["cache_control"]
        for block in blocks
        if isinstance(block, dict) and "cache_control" in block and not isinstance(block["cache_control"], dict)
    ]


def _strict_anthropic_peer(request: Request) -> Reply:
    odd: Final = _non_object_marks(request)
    if odd:
        return Reply(
            status=400,
            body=json.dumps(
                {
                    "type": "error",
                    "error": {
                        "type": "invalid_request_error",
                        "message": f"cache_control: Input should be a valid dictionary, got {json.dumps(odd[0])[:64]}",
                    },
                }
            ).encode(),
        )
    return anthropic_peer(request)


@pytest.mark.parametrize(
    "mark",
    (
        pytest.param("ephemeral", id="string"),
        pytest.param(1, id="int"),
        pytest.param(["ephemeral"], id="list"),
        pytest.param("x" * 5120, id="5kb-string"),
    ),
)
def test_non_object_marks_reach_anthropic_as_they_do_on_chat_completions(gateway: Gateway, mark: JsonValue) -> None:
    marker: Final = new_marker()
    with wire_server(_strict_anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        on_chat: Final = gateway.request("POST", "/v1/chat/completions", _chat_with_message_mark(model, marker, mark))
        on_responses: Final = _send(gateway, _marked_system(model, marker, mark))
        received: Final = wire.drain()
    assert (on_chat.status_code, on_responses.status) == (400, 400), (on_chat.text, on_responses.text)
    assert [_non_object_marks(request) for request in received] == [[mark], [mark]]


def test_string_mark_reaches_bedrock_as_it_does_on_chat_completions(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = _bedrock_deployment(scenario, wire)
        on_chat: Final = gateway.request(
            "POST", "/v1/chat/completions", _chat_with_message_mark(model, marker, "ephemeral")
        )
        assert on_chat.status_code == 200, on_chat.text
        on_responses: Final = _send(gateway, _marked_system(model, marker, "ephemeral"))
        assert on_responses.status == 200, on_responses.text
        received: Final = wire.drain()
    assert len(received) == 2, [request.target for request in received]
    assert bedrock_labels(received[1]) == bedrock_labels(received[0])


def test_unknown_mark_type_is_forwarded_like_a_block_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _marked_system(model, marker, {"type": "persistent"})))
        assert anthropic_labels(_only_request(wire)) == [SYSTEM_LABEL]


def test_duplicate_mark_keys_keep_the_last_one(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        raw: Final = (
            f'{{"model":"{model}","max_output_tokens":64,"input":['
            f'{{"role":"system","content":{json.dumps(SYSTEM)},"cache_control":{{"type":"ephemeral"}},'
            f'"cache_control":{{"type":"ephemeral","ttl":"1h"}}}},'
            f'{{"role":"user","content":{json.dumps(final_text(marker))}}}]}}'
        ).encode()
        _assert_completed(_send_raw(gateway, raw))
        marks: Final = anthropic_marks(_only_request(wire))
    assert [(mark.label, mark.ttl) for mark in marks] == [(SYSTEM_LABEL, "1h")]


def _many_marks(marker: str, *, assistant_marked: bool) -> list[JsonValue]:
    assistant: Final[dict[str, JsonValue]] = {"role": "assistant", "content": _CHECK}
    return [
        _item("system", SYSTEM),
        _item("user", ASK),
        {**assistant, "cache_control": EPHEMERAL} if assistant_marked else assistant,
        {
            "type": "function_call",
            "call_id": call_id(CITIES[0]),
            "name": "lookup_weather",
            "arguments": json.dumps({"city": CITIES[0]}),
        },
        {
            "type": "function_call_output",
            "call_id": call_id(CITIES[0]),
            "output": "sunny.",
            "cache_control": EPHEMERAL,
        },
        _item("user", final_text(marker)),
    ]


def test_four_marks_reach_anthropic(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = _responses(model, _many_marks(marker, assistant_marked=False), tools=[_TOOL])
        _assert_completed(_send(gateway, body))
        labels: Final = anthropic_labels(_only_request(wire))
    assert labels == [SYSTEM_LABEL, ASK_LABEL, _RESULT_LABEL, final_label(marker)]


def test_five_marks_are_refused_by_the_vendor(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = _responses(model, _many_marks(marker, assistant_marked=True), tools=[_TOOL])
        answer: Final = _send(gateway, body)
        labels: Final = anthropic_labels(_only_request(wire))
    assert answer.status == 400, answer.text
    assert "A maximum of 4 blocks with cache_control" in answer.text, answer.text
    assert len(labels) == 5, labels


def test_five_marks_keep_the_newest_four_under_drop_params(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=f"anthropic/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY, drop_params=True
        )
        body: Final = _responses(model, _many_marks(marker, assistant_marked=True), tools=[_TOOL])
        _assert_completed(_send(gateway, body))
        labels: Final = anthropic_labels(_only_request(wire))
    assert labels == [ASK_LABEL, _ASSISTANT_LABEL, _RESULT_LABEL, final_label(marker)]


def test_a_top_level_mark_takes_a_slot_under_drop_params(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=f"anthropic/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY, drop_params=True
        )
        body: Final = _responses(
            model, _many_marks(marker, assistant_marked=False), tools=[_TOOL], cache_control=EPHEMERAL
        )
        _assert_completed(_send(gateway, body))
        labels: Final = anthropic_labels(_only_request(wire))
    assert labels == [ASK_LABEL, _RESULT_LABEL, final_label(marker), "request"]


@pytest.mark.parametrize(
    "mark",
    (
        pytest.param("ephemeral", id="string"),
        pytest.param({"type": "forever"}, id="unknown-type"),
    ),
)
def test_malformed_mark_is_dropped_under_drop_params(gateway: Gateway, mark: JsonValue) -> None:
    marker: Final = new_marker()
    with wire_server(_strict_anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=f"anthropic/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY, drop_params=True
        )
        _assert_completed(_send(gateway, _marked_system(model, marker, mark)))
        assert anthropic_labels(_only_request(wire)) == []


def test_ttl_order_violation_is_the_vendors_verdict(gateway: Gateway) -> None:
    marker: Final = new_marker()
    items: Final[list[JsonValue]] = [
        _item("system", SYSTEM),
        _item("user", ASK, _HOUR),
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        answer: Final = _send(gateway, _responses(model, items))
        marks: Final = anthropic_marks(_only_request(wire))
    assert answer.status == 400, answer.text
    assert "ttl='1h' cache_control block must not come after" in answer.text, answer.text
    assert [(mark.label, mark.ttl) for mark in marks] == [(SYSTEM_LABEL, None), (ASK_LABEL, "1h")]


def test_unauthenticated_marked_request_never_reaches_the_provider(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        answer: Final = _send(gateway, _marked_system(model, marker), key=f"sk-not-a-key-{marker}")
        received: Final = wire.drain()
    assert answer.status == 401, answer.text
    assert "error" in _JSON_OBJECT.validate_json(answer.text), answer.text
    assert received == ()


def _overloaded_peer(request: Request) -> Reply:
    return Reply(
        status=503,
        body=json.dumps({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}).encode(),
    )


def test_provider_error_on_a_marked_request_reaches_the_caller(gateway: Gateway, spend: pcb.SpendLogs) -> None:
    marker: Final = new_marker()
    with wire_server(_overloaded_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        answer: Final = _send(gateway, _marked_system(model, marker))
        labels: Final = anthropic_labels(_only_request(wire))
        assert answer.status == 500, answer.text
        assert "overloaded_error" in answer.text, answer.text
        assert labels == [SYSTEM_LABEL]
        spend.landed(model, answer.call_id, None, status="failure")


_SAP_DISCOVERY: Final = ("/oauth/token", "/v2/lm/")
_SAP_COMPLETION: Final = "/sap-deployment/v2/completion"


def _only_post(wire: Wire) -> Request:
    posted: Final = [
        request
        for request in wire.drain()
        if request.method == "POST" and not request.target.startswith(_SAP_DISCOVERY)
    ]
    assert len(posted) == 1, [request.target for request in posted]
    return posted[0]


def _message_mark_labels(message: JsonValue) -> list[str]:
    fields: Final = object_value(message)
    content: Final = fields.get("content")
    blocks: Final = content if isinstance(content, list) else []
    block_labels: Final = [
        f"{fields.get('role')}:block" for block in blocks if object_value(block).get("cache_control") is not None
    ]
    return [f"{fields.get('role')}:message"] * int(fields.get("cache_control") is not None) + block_labels


def _openai_mark_labels(request: Request) -> list[str]:
    messages: Final = _JSON_OBJECT.validate_json(request.body)["messages"]
    return list(chain.from_iterable(map(_message_mark_labels, messages if isinstance(messages, list) else [])))


def _sap_mark_labels(request: Request) -> list[str]:
    config: Final = object_value(_JSON_OBJECT.validate_json(request.body)["config"])
    prompt: Final = object_value(object_value(object_value(config["modules"])["prompt_templating"])["prompt"])
    template: Final = prompt["template"]
    return list(chain.from_iterable(map(_message_mark_labels, template if isinstance(template, list) else [])))


def _route_mark_labels(request: Request) -> list[str]:
    if "/converse" in request.target:
        return bedrock_labels(request)
    if request.target.endswith("/chat/completions"):
        return _openai_mark_labels(request)
    if request.target == _SAP_COMPLETION:
        return _sap_mark_labels(request)
    return anthropic_labels(request)


def _sap_peer(request: Request) -> Reply:
    match request.target:
        case "/oauth/token":
            return Reply(body=json.dumps({"access_token": "scripted", "expires_in": 3600}).encode())
        case "/v2/lm/deployments":
            deployment: Final = {
                "scenarioId": "orchestration",
                "configurationId": "orchestration-config",
                "deploymentUrl": f"http://{request.headers['host']}/sap-deployment",
                "createdAt": "2026-01-01T00:00:00Z",
            }
            return Reply(body=json.dumps({"resources": [deployment]}).encode())
        case "/v2/lm/configurations/orchestration-config":
            return Reply(body=json.dumps({"executableId": "orchestration"}).encode())
        case _:
            completion: Final = chat_reply(f"chatcmpl-{marker_of(request)}", "scripted", "sunny", stream=False)
            return Reply(body=json.dumps({"final_result": json.loads(completion.body)}).encode())


def _route_peer(request: Request) -> Reply:
    if "/converse" in request.target:
        return bedrock_peer(request)
    if request.target.endswith("/chat/completions"):
        return chat_reply(f"chatcmpl-{marker_of(request)}", "scripted", "sunny", stream=False)
    if request.target.startswith((*_SAP_DISCOVERY, "/sap-deployment/")):
        return _sap_peer(request)
    return anthropic_peer(request)


def _route_deployment(gateway: Gateway, scenario: Scenario, wire: Wire, route: str) -> str:
    match route:
        case "vertex-claude":
            return scenario.model(
                model=f"vertex_ai/{ANTHROPIC_MODEL}",
                api_key=None,
                api_base=wire.url,
                vertex_project="scripted-project",
                vertex_location="us-east5",
                vertex_credentials=service_account_json("scripted-project", gateway.upstream_url.rstrip("/")),
            )
        case "bedrock-llama":
            return scenario.model(
                model="bedrock/converse/meta.llama3-3-70b-instruct-v1:0",
                api_key=PROVIDER_KEY,
                api_base=wire.url,
                aws_region_name="us-east-1",
                aws_bedrock_runtime_endpoint=wire.url,
            )
        case "sap-claude":
            service_key: Final = {
                "clientid": "scripted",
                "clientsecret": PROVIDER_KEY,
                "url": wire.url,
                "serviceurls": {"AI_API_URL": wire.url},
            }
            return scenario.model(
                model="sap/anthropic--claude-4.5-sonnet", api_base=wire.url, api_key=json.dumps(service_key)
            )
        case _ if route.startswith("openrouter/"):
            return scenario.model(model=route, api_base=wire.url, api_key=PROVIDER_KEY, use_chat_completions_api=True)
        case _:
            return scenario.model(model=route, api_base=wire.url, api_key=PROVIDER_KEY)


def _chat_with_tool_mark(model: str, marker: str) -> dict[str, JsonValue]:
    tool_call: Final[dict[str, JsonValue]] = {
        "id": call_id(CITIES[0]),
        "type": "function",
        "function": {"name": "lookup_weather", "arguments": json.dumps({"city": CITIES[0]})},
    }
    return {
        "model": model,
        "max_tokens": 64,
        "tools": [{"type": "function", "function": {"name": "lookup_weather", "parameters": {"type": "object"}}}],
        "messages": [
            {"role": "user", "content": ASK},
            {"role": "assistant", "content": None, "tool_calls": [tool_call]},
            {"role": "tool", "tool_call_id": call_id(CITIES[0]), "content": "sunny.", "cache_control": EPHEMERAL},
            {"role": "user", "content": final_text(marker)},
        ],
    }


def _shaped_requests(model: str, shape: str) -> tuple[dict[str, JsonValue], dict[str, JsonValue]]:
    match shape:
        case "tool":
            bridged: Final = _responses(model, _tool_items(new_marker(), output_mark=EPHEMERAL), tools=[_TOOL])
            return _chat_with_tool_mark(model, new_marker()), bridged
        case _:
            return _chat_with_message_mark(model, new_marker()), _marked_system(model, new_marker())


@pytest.mark.parametrize(
    ("route", "shape", "keeps"),
    (
        pytest.param("dashscope/qwen-plus", "system", True, id="dashscope"),
        pytest.param("minimax/MiniMax-M2", "system", True, id="minimax"),
        pytest.param("zai/glm-4.6", "system", True, id="zai"),
        pytest.param(f"openrouter/anthropic/{ANTHROPIC_MODEL}", "system", True, id="openrouter-claude"),
        pytest.param("openrouter/google/gemini-3.8-flash", "system", True, id="openrouter-gemini"),
        pytest.param("snowflake/claude-4-sonnet", "tool", True, id="snowflake-claude-tool-output"),
        pytest.param("vertex-claude", "system", True, id="vertex-claude"),
        pytest.param("sap-claude", "system", True, id="sap-claude"),
        pytest.param("anthropic/my-gateway-model", "system", True, id="anthropic-non-claude-name"),
        pytest.param("snowflake/claude-4-sonnet", "system", False, id="snowflake-claude-string-system"),
        pytest.param("openrouter/openai/gpt-5.4", "system", False, id="openrouter-gpt"),
        pytest.param("snowflake/llama3.1-70b", "system", False, id="snowflake-llama"),
        pytest.param("snowflake/llama-claude-distill", "system", False, id="snowflake-claude-inside-the-name"),
        pytest.param("databricks/databricks-meta-llama-3-3-70b-instruct", "tool", False, id="databricks-llama"),
        pytest.param("bedrock-llama", "system", False, id="bedrock-llama"),
    ),
)
def test_item_mark_reaches_each_route_the_way_a_chat_message_mark_does(
    gateway: Gateway, route: str, shape: str, keeps: bool
) -> None:
    with wire_server(answering_model_discovery(_route_peer)) as wire, gateway.scenario() as scenario:
        model: Final = _route_deployment(gateway, scenario, wire, route)
        on_chat_body, bridged_body = _shaped_requests(model, shape)
        on_chat: Final = gateway.request("POST", "/v1/chat/completions", on_chat_body)
        assert on_chat.status_code == 200, on_chat.text
        chat_request: Final = _only_post(wire)
        _assert_completed(_send(gateway, bridged_body))
        bridged_request: Final = _only_post(wire)
    bridged_labels: Final = _route_mark_labels(bridged_request)
    assert bridged_request.target == chat_request.target
    assert bridged_labels == (_route_mark_labels(chat_request) if keeps else [])
    assert bool(bridged_labels) is keeps, bridged_labels


@pytest.mark.parametrize(
    ("drop_params", "expected"),
    (
        pytest.param(False, [_BEDROCK_SYSTEM_LABEL], id="as-sent"),
        pytest.param(True, [], id="drop-params"),
    ),
)
def test_unmapped_bedrock_arn_keeps_the_item_mark_unless_drop_params(
    gateway: Gateway, drop_params: bool, expected: list[str]
) -> None:
    marker: Final = new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = _unmapped_arn_deployment(scenario, wire, drop_params=drop_params)
        answer: Final = _send(gateway, _marked_system(model, marker))
        assert answer.status == 200, answer.text
        assert bedrock_labels(_only_request(wire)) == expected


@pytest.mark.parametrize("request_drop_params", (True, "true"), ids=("bool", "string"))
def test_request_drop_params_trims_five_marks_to_the_newest_four(
    gateway: Gateway, request_drop_params: JsonValue
) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        items: Final = _many_marks(marker, assistant_marked=True)
        _assert_completed(_send(gateway, _responses(model, items, tools=[_TOOL], drop_params=request_drop_params)))
        labels: Final = anthropic_labels(_only_request(wire))
    assert labels == [ASK_LABEL, _ASSISTANT_LABEL, _RESULT_LABEL, final_label(marker)]


@pytest.mark.parametrize(
    ("deployment_fields", "request_drop_params"),
    (
        pytest.param({}, None, id="request-null"),
        pytest.param({}, "", id="request-empty-string"),
        pytest.param({"drop_params": True}, False, id="request-false-over-deployment-true"),
    ),
)
def test_request_without_drop_params_forwards_five_marks_as_sent(
    gateway: Gateway, deployment_fields: dict[str, JsonValue], request_drop_params: JsonValue
) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=f"anthropic/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY, **deployment_fields
        )
        items: Final = _many_marks(marker, assistant_marked=True)
        answer: Final = _send(gateway, _responses(model, items, tools=[_TOOL], drop_params=request_drop_params))
        labels: Final = anthropic_labels(_only_request(wire))
    assert answer.status == 400, answer.text
    assert "A maximum of 4 blocks with cache_control" in answer.text, answer.text
    assert len(labels) == 5, labels


def _marked_function_call_items(marker: str) -> list[JsonValue]:
    calls: Final[list[JsonValue]] = [
        {
            "type": "function_call",
            "call_id": call_id(city),
            "name": "lookup_weather",
            "arguments": json.dumps({"city": city}),
            "cache_control": EPHEMERAL,
        }
        for city in CITIES
    ]
    outputs: Final[list[JsonValue]] = [
        {"type": "function_call_output", "call_id": call_id(city), "output": "sunny."} for city in CITIES
    ]
    return [{"role": "user", "content": ASK}, *calls, *outputs, {"role": "user", "content": final_text(marker)}]


def test_injection_points_count_a_marked_function_tool_toward_the_cap(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire, inject=True)
        body: Final = _responses(
            model,
            _marked_function_call_items(marker),
            instructions=SYSTEM,
            tools=[{**_TOOL, "cache_control": EPHEMERAL}],
        )
        _assert_completed(_send(gateway, body))
        labels: Final = anthropic_labels(_only_request(wire))
    assert labels == ["tool:lookup_weather", *(tool_use_label(city) for city in CITIES)]


def test_assistant_text_after_a_function_call_keeps_its_mark(gateway: Gateway) -> None:
    marker: Final = new_marker()
    items: Final[list[JsonValue]] = [
        {"role": "user", "content": ASK},
        {
            "type": "function_call",
            "call_id": call_id(CITIES[0]),
            "name": "lookup_weather",
            "arguments": json.dumps({"city": CITIES[0]}),
        },
        _item("assistant", _CHECK),
        {"type": "function_call_output", "call_id": call_id(CITIES[0]), "output": "sunny."},
        {"role": "user", "content": final_text(marker)},
    ]
    with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        _assert_completed(_send(gateway, _responses(model, items, tools=[_TOOL])))
        labels: Final = anthropic_labels(_only_request(wire))
    assert labels == [_ASSISTANT_LABEL]


_COUNTED_TOKENS: Final = 42


def _count_peer(request: Request) -> Reply:
    if request.target.endswith("/count_tokens"):
        return Reply(body=json.dumps({"input_tokens": _COUNTED_TOKENS}).encode())
    return anthropic_peer(request)


def test_input_tokens_counts_marked_items_without_their_marks(gateway: Gateway) -> None:
    marker: Final = new_marker()
    with wire_server(_count_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = {"model": model, "input": _many_marks(marker, assistant_marked=True), "tools": [_TOOL]}
        response: Final = gateway.request("POST", "/v1/responses/input_tokens", body)
        counted: Final = _only_request(wire)
    assert response.status_code == 200, response.text
    assert _JSON_OBJECT.validate_json(response.text) == {
        "object": "response.input_tokens",
        "input_tokens": _COUNTED_TOKENS,
        "litellm_count_source": "provider",
    }
    assert counted.target.endswith("/count_tokens"), counted.target
    assert anthropic_labels(counted) == []


_INPUT_SHAPES: Final = ("string", "list", "object")


def _shaped_input(shape: str, marker: str) -> JsonValue:
    marked: Final = _item("user", final_text(marker))
    match shape:
        case "string":
            return final_text(marker)
        case "list":
            return [_item("system", SYSTEM), marked]
        case _:
            return {"type": "message", **marked}


def _shaped_body(model: str, shape: str, marker: str, **fields: JsonValue) -> dict[str, JsonValue]:
    return {
        "model": model,
        "max_output_tokens": 64,
        "instructions": SYSTEM,
        "input": _shaped_input(shape, marker),
        **fields,
    }


def _error_message(answer: _Answer) -> JsonValue:
    if answer.status == 200:
        return None
    return object_value(_JSON_OBJECT.validate_json(answer.text)["error"]).get("message")


@pytest.mark.parametrize("shape", _INPUT_SHAPES)
def test_each_input_shape_reads_the_same_whether_or_not_the_route_keeps_item_marks(
    gateway: Gateway, shape: str
) -> None:
    kept_marker, stripped_marker = new_marker(), new_marker()
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = _unmapped_arn_deployment(scenario, wire)
        kept: Final = _send(gateway, _shaped_body(model, shape, kept_marker))
        kept_sent: Final = sent_without_cache_points(wire.drain(), kept_marker)
        stripped: Final = _send(gateway, _shaped_body(model, shape, stripped_marker, drop_params=True))
        stripped_sent: Final = sent_without_cache_points(wire.drain(), stripped_marker)
    assert (stripped.status, _error_message(stripped)) == (kept.status, _error_message(kept)), (
        kept.text,
        stripped.text,
    )
    assert stripped_sent == kept_sent


@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream"))
def test_object_input_on_a_mark_stripping_route_is_served(gateway: Gateway, stream: bool) -> None:
    marker: Final = new_marker()
    reply: Final = chat_reply(f"chatcmpl-{marker}", "gpt-5.4", "sunny", stream=stream)
    with wire_server(answering_model_discovery(lambda request: reply)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=_OPENAI_COMPATIBLE, api_base=f"{wire.url}/v1", api_key=PROVIDER_KEY, use_chat_completions_api=True
        )
        _assert_completed(_send(gateway, _shaped_body(model, "object", marker, stream=stream)), stream=stream)
        posted: Final = [request for request in wire.drain() if request.method == "POST"]
    assert [request.target for request in posted] == ["/v1/chat/completions"], [request.target for request in posted]
    assert _openai_message_marks(posted[0]) == []


@pytest.mark.parametrize("shape", _INPUT_SHAPES)
def test_input_tokens_reads_each_input_shape_without_item_marks(gateway: Gateway, shape: str) -> None:
    marker: Final = new_marker()
    with wire_server(_count_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_deployment(scenario, wire)
        body: Final = {
            key: value for key, value in _shaped_body(model, shape, marker).items() if key != "max_output_tokens"
        }
        response: Final = gateway.request("POST", "/v1/responses/input_tokens", body)
        counted: Final = wire.drain()
    assert response.status_code == 200, response.text
    assert _JSON_OBJECT.validate_json(response.text) == {
        "object": "response.input_tokens",
        "input_tokens": _COUNTED_TOKENS,
        "litellm_count_source": "provider",
    }
    assert [anthropic_labels(request) for request in counted] == [[]]


@pytest.mark.parametrize("shape", _INPUT_SHAPES)
def test_project_input_token_limit_reads_each_input_shape(gateway: Gateway, shape: str) -> None:
    marker: Final = new_marker()
    reply: Final = chat_reply(f"chatcmpl-{marker}", "gpt-5.4", "sunny", stream=False)
    with wire_server(answering_model_discovery(lambda request: reply)) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model=_OPENAI_COMPATIBLE, api_base=f"{wire.url}/v1", api_key=PROVIDER_KEY, use_chat_completions_api=True
        )
        team: Final = scenario.team()
        key: Final = scenario.key(team_id=team, project_id=scenario.project(team, model_itpm_limit={model: 1_000_000}))
        _assert_completed(_send(gateway, _shaped_body(model, shape, marker), key=key))
        posted: Final = [request for request in wire.drain() if request.method == "POST"]
    assert [request.target for request in posted] == ["/v1/chat/completions"], [request.target for request in posted]
    assert _openai_message_marks(posted[0]) == []