Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

39.7 KB
Raw Blame
import json
import re
import uuid
from collections.abc import Callable, Mapping, Sequence
from email.parser import BytesParser
from email.policy import HTTP
from types import MappingProxyType
from typing import Final

import anthropic
import httpx
import pytest
from integration._support.client import Gateway, Scenario, eventually, string_value
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter

_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, JsonValue], ...])
_TOOL_USE_ID: Final = "toolu_01"
_WEATHER: Final = "sunny, 21C"
_FIELD_ERROR: Final = f"tool_result {_TOOL_USE_ID} content.0.text: Input should be a valid string"
_CLAUDE_CODE: Final = MappingProxyType({"user-agent": "claude-cli/2.1.0 (external, cli)"})
_OPENAI_COUNT: Final = 4242
_GOOGLE_COUNT: Final = 777
_ANTHROPIC_COUNT: Final = 58
_BEDROCK_COUNT: Final = 1345
_GEMINI: Final = "gemini-4-argon"
_VERTEX_PROJECT: Final = "scripted-project"
_CITY: Final = "Paris"
_REDACTED_CITY: Final = "[CITY]"
_TOOLS: Final[list[JsonValue]] = [
    {
        "name": "get_weather",
        "description": "Get the weather for a city",
        "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
    }
]


def _messages(*blocks: JsonValue) -> list[JsonValue]:
    return [
        {"role": "user", "content": "What is the weather in Paris?"},
        {
            "role": "assistant",
            "content": [{"type": "tool_use", "id": _TOOL_USE_ID, "name": "get_weather", "input": {"city": "Paris"}}],
        },
        {"role": "user", "content": [{"type": "tool_result", "tool_use_id": _TOOL_USE_ID, "content": list(blocks)}]},
    ]


_NULL_TEXT: Final = _messages({"type": "text", "text": None})
_NUMBER_THEN_WEATHER: Final = _messages({"type": "text", "text": 5}, {"type": "text", "text": _WEATHER})
_WEATHER_ONLY: Final = _messages({"type": "text", "text": _WEATHER})
_WEATHER_THEN_MISSING: Final = _messages({"type": "text", "text": _WEATHER}, {"type": "text"})


def _body(model: str, messages: list[JsonValue], **extra: JsonValue) -> dict[str, JsonValue]:
    return {"model": model, "messages": messages, "tools": _TOOLS, **extra}


def _responses_reply() -> bytes:
    return json.dumps(
        {
            "id": f"resp_{uuid.uuid4().hex}",
            "object": "response",
            "created_at": 1789788253,
            "status": "completed",
            "model": "gpt-5.4-mini",
            "output": [
                {
                    "type": "message",
                    "id": f"msg_{uuid.uuid4().hex}",
                    "status": "completed",
                    "role": "assistant",
                    "content": [{"type": "output_text", "text": "It is sunny.", "annotations": []}],
                }
            ],
            "usage": {"input_tokens": 41, "output_tokens": 5, "total_tokens": 46},
        }
    ).encode()


def _chat_reply() -> bytes:
    return json.dumps(
        {
            "id": f"chatcmpl-{uuid.uuid4().hex}",
            "object": "chat.completion",
            "created": 1,
            "model": "mistral-small-latest",
            "choices": [
                {"index": 0, "message": {"role": "assistant", "content": "It is sunny."}, "finish_reason": "stop"}
            ],
            "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13},
        }
    ).encode()


def _anthropic_reply() -> bytes:
    return json.dumps(
        {
            "id": f"msg_{uuid.uuid4().hex}",
            "type": "message",
            "role": "assistant",
            "model": "claude-sonnet-4-6",
            "content": [{"type": "text", "text": "It is sunny."}],
            "stop_reason": "end_turn",
            "stop_sequence": None,
            "usage": {"input_tokens": 9, "output_tokens": 4},
        }
    ).encode()


def _openai_peer(request: Request) -> Reply:
    if request.target.endswith("/responses/input_tokens"):
        return Reply(body=json.dumps({"input_tokens": _OPENAI_COUNT}).encode())
    if request.target.endswith("/responses"):
        return Reply(body=_responses_reply())
    return Reply(status=404, body=json.dumps({"error": request.target}).encode())


def _chat_peer(request: Request) -> Reply:
    if request.target.endswith("/chat/completions"):
        return Reply(body=_chat_reply())
    return Reply(status=404, body=json.dumps({"error": request.target}).encode())


def _google_peer(request: Request) -> Reply:
    if request.target.endswith(":countTokens"):
        return Reply(body=json.dumps({"totalTokens": _GOOGLE_COUNT}).encode())
    return Reply(status=404, body=json.dumps({"error": request.target}).encode())


def _anthropic_peer(request: Request) -> Reply:
    if request.target.endswith("/v1/messages/count_tokens"):
        return Reply(body=json.dumps({"input_tokens": _ANTHROPIC_COUNT}).encode())
    if request.target.endswith("/v1/messages"):
        return Reply(body=_anthropic_reply())
    return Reply(status=404, body=json.dumps({"error": request.target}).encode())


def _bedrock_peer(request: Request) -> Reply:
    if request.target.endswith("/count-tokens"):
        return Reply(body=json.dumps({"inputTokens": _BEDROCK_COUNT}).encode())
    return Reply(status=404, body=json.dumps({"error": request.target}).encode())


def _openai_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
    return scenario.model(model="openai/gpt-5.4-mini", api_base=wire.url, api_key="synthetic-openai-key", **parameters)


def _mistral_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
    return scenario.model(
        model="mistral/mistral-small-latest", api_base=wire.url, api_key="synthetic-mistral-key", **parameters
    )


def _anthropic_model(scenario: Scenario, wire: Wire, **parameters: JsonValue) -> str:
    return scenario.model(
        model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-anthropic-key", **parameters
    )


def _google_model(gateway: Gateway, scenario: Scenario, wire: Wire, provider: str, **parameters: JsonValue) -> str:
    if provider == "gemini":
        return scenario.model(model=f"gemini/{_GEMINI}", api_base=wire.url, api_key="scripted-gemini-key", **parameters)
    return scenario.model(
        model=f"vertex_ai/{_GEMINI}",
        api_base=wire.url,
        api_key=None,
        vertex_project=_VERTEX_PROJECT,
        vertex_location="us-east5",
        vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")),
        **parameters,
    )


def _json(request: Request) -> dict[str, JsonValue]:
    return _JSON_OBJECT.validate_json(request.body)


def _only(requests: Sequence[Request], suffix: str) -> dict[str, JsonValue]:
    matching: Final = tuple(request for request in requests if request.target.endswith(suffix))
    assert len(matching) == 1, [request.target for request in requests]
    return _json(matching[0])


def _function_call_output(body: Mapping[str, JsonValue]) -> JsonValue:
    outputs: Final = tuple(
        item["output"]
        for item in _JSON_OBJECTS.validate_python(body["input"])
        if item.get("type") == "function_call_output"
    )
    assert len(outputs) == 1, body
    return outputs[0]


def _tool_message_text(body: Mapping[str, JsonValue]) -> str:
    contents: Final = tuple(
        message.get("content")
        for message in _JSON_OBJECTS.validate_python(body["messages"])
        if message.get("role") == "tool"
    )
    assert len(contents) == 1, body
    content: Final = contents[0]
    if isinstance(content, str):
        return content
    return "".join(str(part["text"]) for part in _JSON_OBJECTS.validate_python(content))


def _anthropic_error_message(status: int, text: str, payload: bytes) -> str:
    assert status == 400, text
    error: Final = _JSON_OBJECT.validate_json(payload)
    assert error["type"] == "error", text
    detail: Final = _JSON_OBJECT.validate_python(error["error"])
    assert detail["type"] == "invalid_request_error", text
    return str(detail["message"])


def _assert_field_rejection(gateway: Gateway, path: str, body: Mapping[str, JsonValue], **headers: str) -> None:
    response: Final = gateway.request("POST", path, body, headers=headers)
    assert _FIELD_ERROR in _anthropic_error_message(response.status_code, response.text, response.content), (
        response.text
    )


def _sdk(gateway: Gateway) -> anthropic.Anthropic:
    return anthropic.Anthropic(
        api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client
    )


def _assert_sdk_bad_request(raised: pytest.ExceptionInfo[anthropic.BadRequestError]) -> None:
    error: Final = raised.value
    assert error.status_code == 400, error
    assert _FIELD_ERROR in _anthropic_error_message(error.status_code, error.message, error.response.content), error


def test_openai_rejects_a_non_string_tool_result_text_on_both_routes(gateway: Gateway) -> None:
    client: Final = _sdk(gateway)
    with wire_server(_openai_peer) as wire, gateway.scenario() as scenario:
        model: Final = _openai_model(scenario, wire)
        with pytest.raises(anthropic.BadRequestError) as created:
            client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS)
        _assert_sdk_bad_request(created)
        with pytest.raises(anthropic.BadRequestError) as streamed:
            client.messages.create(model=model, max_tokens=64, messages=_NULL_TEXT, tools=_TOOLS, stream=True)
        _assert_sdk_bad_request(streamed)
        with pytest.raises(anthropic.BadRequestError) as counted:
            client.messages.count_tokens(model=model, messages=_NULL_TEXT, tools=_TOOLS)
        _assert_sdk_bad_request(counted)
        assert wire.drain() == ()


_DROP_SOURCES: Final = MappingProxyType(
    {
        "deployment": (MappingProxyType({"drop_params": True}), MappingProxyType({}), MappingProxyType({})),
        "body": (MappingProxyType({}), MappingProxyType({"drop_params": True}), MappingProxyType({})),
        "claude_code": (MappingProxyType({}), MappingProxyType({}), _CLAUDE_CODE),
    }
)


@pytest.mark.parametrize("source", sorted(_DROP_SOURCES))
def test_openai_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None:
    deployment, extra, headers = _DROP_SOURCES[source]
    with wire_server(_openai_peer) as wire, gateway.scenario() as scenario:
        model: Final = _openai_model(scenario, wire, **deployment)
        body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra)
        created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers)
        counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers)
        assert created.status_code == 200, created.text
        assert counted.status_code == 200, counted.text
        assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _OPENAI_COUNT, counted.text
        requests: Final = wire.drain()
        assert _function_call_output(_only(requests, "/responses")) == _WEATHER
        assert _function_call_output(_only(requests, "/responses/input_tokens")) == _WEATHER


def test_explicit_drop_params_false_wins_on_both_routes(gateway: Gateway) -> None:
    with wire_server(_openai_peer) as wire, gateway.scenario() as scenario:
        model: Final = _openai_model(scenario, wire, drop_params=True)
        body: Final = _body(model, _NULL_TEXT, drop_params=False)
        _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64}, **_CLAUDE_CODE)
        _assert_field_rejection(gateway, "/v1/messages/count_tokens", body, **_CLAUDE_CODE)
        assert wire.drain() == ()


def test_chat_bridge_and_local_count_reject_a_non_string_tool_result_text(gateway: Gateway) -> None:
    with wire_server(_chat_peer) as wire, gateway.scenario() as scenario:
        model: Final = _mistral_model(scenario, wire)
        body: Final = _body(model, _NULL_TEXT)
        _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64})
        _assert_field_rejection(gateway, "/v1/messages", {**body, "max_tokens": 64, "stream": True})
        _assert_field_rejection(gateway, "/v1/messages/count_tokens", body)
        assert wire.drain() == ()


@pytest.mark.parametrize("source", ("deployment", "body"))
def test_chat_bridge_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None:
    deployment, extra, _ = _DROP_SOURCES[source]
    with wire_server(_chat_peer) as wire, gateway.scenario() as scenario:
        model: Final = _mistral_model(scenario, wire, **deployment)
        body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra)
        created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64})
        counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body)
        assert created.status_code == 200, created.text
        assert counted.status_code == 200, counted.text
        assert _tool_message_text(_only(wire.drain(), "/chat/completions")) == _WEATHER


def test_count_without_a_deployment_rejects_and_honors_body_drop_params(gateway: Gateway) -> None:
    model: Final = f"integration-unrouted-{uuid.uuid4().hex}"
    _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(model, _NULL_TEXT))
    dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(model, _NULL_TEXT, drop_params=True))
    assert dropped.status_code == 200, dropped.text
    tokens: Final = _JSON_OBJECT.validate_json(dropped.content)["input_tokens"]
    assert isinstance(tokens, int) and tokens > 0, dropped.text


def _counted_contents(body: Mapping[str, JsonValue]) -> JsonValue:
    if "contents" in body:
        return body["contents"]
    return _JSON_OBJECT.validate_python(body["generateContentRequest"])["contents"]


@pytest.mark.parametrize("provider", ("gemini", "vertex_ai"))
def test_google_count_rejects_and_drops_a_non_string_tool_result_text(gateway: Gateway, provider: str) -> None:
    with wire_server(_google_peer) as wire, gateway.scenario() as scenario:
        strict: Final = _google_model(gateway, scenario, wire, provider)
        dropping: Final = _google_model(gateway, scenario, wire, provider, drop_params=True)
        _assert_field_rejection(gateway, "/v1/messages/count_tokens", _body(strict, _NULL_TEXT))
        assert tuple(request for request in wire.drain() if request.target.endswith(":countTokens")) == ()
        dropped: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _NUMBER_THEN_WEATHER))
        assert dropped.status_code == 200, dropped.text
        dropped_body: Final = _only(wire.drain(), ":countTokens")
        clean: Final = gateway.request("POST", "/v1/messages/count_tokens", _body(dropping, _WEATHER_ONLY))
        assert clean.status_code == 200, clean.text
        assert _JSON_OBJECT.validate_json(dropped.content)["input_tokens"] == _GOOGLE_COUNT, dropped.text
        assert _counted_contents(dropped_body) == _counted_contents(_only(wire.drain(), ":countTokens"))


def _forwarded_tool_result(body: Mapping[str, JsonValue]) -> JsonValue:
    last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1]
    return _JSON_OBJECTS.validate_python(last["content"])[0]["content"]


@pytest.mark.parametrize(
    ("headers", "forwarded"),
    ((MappingProxyType({}), [{"type": "text", "text": None}]), (_CLAUDE_CODE, [])),
    ids=("no-drop-params", "claude-code-drop-params"),
)
def test_native_anthropic_forwards_a_null_tool_result_text_unless_drop_params_drops_it(
    gateway: Gateway, headers: Mapping[str, str], forwarded: list[JsonValue]
) -> None:
    with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_model(scenario, wire)
        body: Final = _body(model, _NULL_TEXT)
        created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers)
        counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers)
        assert created.status_code == 200, created.text
        assert counted.status_code == 200, counted.text
        assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text
        requests: Final = wire.drain()
        assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded
        assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded


def test_bedrock_count_with_a_claude_code_user_agent_still_counts(gateway: Gateway) -> None:
    with wire_server(_bedrock_peer) as wire, gateway.scenario() as scenario:
        model: Final = scenario.model(
            model="bedrock/anthropic.claude-sonnet-4-6",
            api_base=wire.url,
            api_key=None,
            aws_access_key_id="AKIAINTEGRATIONTOOLRESULT",
            aws_secret_access_key="integration-tool-result-secret",
            aws_region_name="us-east-1",
        )
        counted: Final = gateway.request(
            "POST", "/v1/messages/count_tokens", _body(model, _WEATHER_ONLY), headers=_CLAUDE_CODE
        )
        assert counted.status_code == 200, counted.text
        assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _BEDROCK_COUNT, counted.text
        assert len(tuple(request for request in wire.drain() if request.target.endswith("/count-tokens"))) == 1


@pytest.mark.parametrize("provider", ("openai", "mistral"))
def test_valid_tool_result_text_reaches_the_upstream(gateway: Gateway, provider: str) -> None:
    peer: Final = _openai_peer if provider == "openai" else _chat_peer
    with wire_server(peer) as wire, gateway.scenario() as scenario:
        model: Final = (_openai_model if provider == "openai" else _mistral_model)(scenario, wire)
        body: Final = _body(model, _WEATHER_THEN_MISSING)
        created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64})
        counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body)
        assert created.status_code == 200, created.text
        assert counted.status_code == 200, counted.text
        requests: Final = wire.drain()
        if provider == "openai":
            assert _function_call_output(_only(requests, "/responses")) == f"{_WEATHER}\n"
            assert _function_call_output(_only(requests, "/responses/input_tokens")) == f"{_WEATHER}\n"
        else:
            assert _tool_message_text(_only(requests, "/chat/completions")) == _WEATHER


def _rewritten(content: JsonValue, rewrite: Callable[[str], str]) -> JsonValue:
    if isinstance(content, str):
        return rewrite(content)
    if isinstance(content, list):
        return [_rewritten(part, rewrite) for part in content]
    if isinstance(content, dict) and isinstance(text := content.get("text"), str):
        return {**content, "text": rewrite(text)}
    return content


def _redacted(content: JsonValue) -> JsonValue:
    return _rewritten(content, lambda text: text.replace(_CITY, _REDACTED_CITY))


def _redacted_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    return {**row, "content": _redacted(row["content"])} if "content" in row else dict(row)


def _digits_masked_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    if "content" not in row:
        return dict(row)
    return {**row, "content": _rewritten(row["content"], lambda text: re.sub(r"\d", "#", text))}


def _joined_tool_row(row: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    redacted: Final = _redacted_row(row)
    content: Final = redacted.get("content")
    if row.get("role") != "tool" or not isinstance(content, list):
        return redacted
    texts: Final = (
        part["text"] for part in _JSON_OBJECTS.validate_python(content) if isinstance(part.get("text"), str)
    )
    return {**redacted, "content": " | ".join(str(text) for text in texts)}


_ROW_REWRITES: Final = MappingProxyType({"join": _joined_tool_row, "digits": _digits_masked_row})


def _guardrail_peer(request: Request) -> Reply:
    rows: Final = _JSON_OBJECTS.validate_python(_json(request).get("structured_messages") or [])
    if request.target.startswith("/none/") or not rows:
        return Reply(body=json.dumps({"action": "NONE"}).encode())
    rewrite: Final = _ROW_REWRITES.get(request.target.split("/")[1], _redacted_row)
    verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [rewrite(row) for row in rows]}
    return Reply(body=json.dumps(verdict).encode())


def _delete_guardrail(gateway: Gateway, identity: str) -> None:
    deleted: Final = gateway.request("DELETE", f"/guardrails/{identity}")
    assert deleted.status_code == 200, deleted.text


def _guardrail(gateway: Gateway, scenario: Scenario, wire: Wire, mode: str) -> str:
    name: Final = f"tool-result-{mode}-{uuid.uuid4().hex}"
    params: Final[dict[str, JsonValue]] = {
        "guardrail": "generic_guardrail_api",
        "mode": "pre_call",
        "default_on": False,
        "api_base": f"{wire.url}/{mode}",
        "api_key": "synthetic-guardrail-key",
    }
    created: Final = gateway.post("/guardrails", {"guardrail": {"guardrail_name": name, "litellm_params": params}})
    scenario.cleanups.callback(_delete_guardrail, gateway, string_value(created["guardrail_id"]))
    return name


def _first_turn(messages: JsonValue) -> str:
    return json.dumps(_JSON_OBJECTS.validate_python(messages)[0])


def test_rewriting_guardrail_leaves_the_rejection_to_the_openai_route(gateway: Gateway) -> None:
    with (
        wire_server(_openai_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _openai_model(scenario, provider)
        body: Final = _body(model, _NULL_TEXT, max_tokens=64, guardrails=[_guardrail(gateway, scenario, guard, "rows")])
        _assert_field_rejection(gateway, "/v1/messages", body)
        _assert_field_rejection(gateway, "/v1/messages", {**body, "stream": True})
        assert provider.drain() == ()


@pytest.mark.parametrize("source", sorted(_DROP_SOURCES))
def test_rewriting_guardrail_then_drop_params_drops_the_malformed_text(gateway: Gateway, source: str) -> None:
    deployment, extra, headers = _DROP_SOURCES[source]
    with (
        wire_server(_openai_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _openai_model(scenario, provider, **deployment)
        guardrail: Final = _guardrail(gateway, scenario, guard, "rows")
        body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail], **extra)
        created: Final = gateway.request("POST", "/v1/messages", body, headers=headers)
        assert created.status_code == 200, created.text
        forwarded: Final = _only(provider.drain(), "/responses")
        first_turn: Final = _first_turn(forwarded["input"])
        assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn
        assert _function_call_output(forwarded) == _WEATHER, forwarded


@pytest.mark.parametrize(
    ("deployment", "forwarded"),
    (
        (MappingProxyType({}), [{"type": "text", "text": 5}, {"type": "text", "text": _WEATHER}]),
        (MappingProxyType({"drop_params": True}), [{"type": "text", "text": _WEATHER}]),
    ),
    ids=("no-drop-params", "deployment-drop-params"),
)
def test_rewriting_guardrail_forwards_the_malformed_text_to_anthropic_unless_drop_params_drops_it(
    gateway: Gateway, deployment: Mapping[str, JsonValue], forwarded: list[JsonValue]
) -> None:
    with (
        wire_server(_anthropic_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _anthropic_model(scenario, provider, **deployment)
        guardrail: Final = _guardrail(gateway, scenario, guard, "rows")
        body: Final = _body(model, _NUMBER_THEN_WEATHER, max_tokens=64, guardrails=[guardrail])
        created: Final = gateway.request("POST", "/v1/messages", body)
        assert created.status_code == 200, created.text
        forwarded_body: Final = _only(provider.drain(), "/v1/messages")
        first_turn: Final = _first_turn(forwarded_body["messages"])
        assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn
        assert _forwarded_tool_result(forwarded_body) == forwarded


_MORNING: Final = f"{_CITY} morning: sunny"
_EVENING: Final = f"{_CITY} evening: 18C"
_TOMORROW: Final = f"{_CITY} tomorrow: rain"
_WEEKEND: Final = f"{_CITY} weekend: clear"


def _text(value: JsonValue) -> dict[str, JsonValue]:
    return {"type": "text", "text": value}


def _tool_use(tool_use_id: str) -> dict[str, JsonValue]:
    return {"type": "tool_use", "id": tool_use_id, "name": "get_weather", "input": {"city": _CITY}}


def _tool_result(tool_use_id: str, *parts: JsonValue) -> dict[str, JsonValue]:
    return {"type": "tool_result", "tool_use_id": tool_use_id, "content": [_text(part) for part in parts]}


_SEVERAL_TOOL_RESULTS: Final[list[JsonValue]] = [
    {"role": "user", "content": f"What is the weather in {_CITY}?"},
    {"role": "assistant", "content": [_tool_use("toolu_01"), _tool_use("toolu_02"), _tool_use("toolu_03")]},
    {
        "role": "user",
        "content": [
            _tool_result("toolu_01", None, _MORNING, 5, _EVENING),
            _tool_result("toolu_02", _TOMORROW, 7.5),
            _tool_result("toolu_03", _WEEKEND),
        ],
    },
]


def _forwarded_tool_results(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
    last: Final = _JSON_OBJECTS.validate_python(body["messages"])[-1]
    return {
        string_value(block["tool_use_id"]): block["content"]
        for block in _JSON_OBJECTS.validate_python(last["content"])
        if block.get("type") == "tool_result"
    }


def _texts(content: JsonValue) -> tuple[JsonValue, ...]:
    if isinstance(content, str):
        return (content,)
    return tuple(part.get("text") for part in _JSON_OBJECTS.validate_python(content))


_RESTORED_TOOL_RESULTS: Final = MappingProxyType(
    {
        "rows": (
            [_text(None), _text(_redacted(_MORNING)), _text(5), _text(_redacted(_EVENING))],
            [_text(_redacted(_TOMORROW)), _text(7.5)],
        ),
        "join": (
            [_text(f"null | {_redacted(_MORNING)} | 5 | {_redacted(_EVENING)}")],
            [_text(f"{_redacted(_TOMORROW)} | 7.5")],
        ),
    }
)


@pytest.mark.parametrize("mode", sorted(_RESTORED_TOOL_RESULTS))
def test_rewriting_guardrail_puts_back_each_malformed_text_it_left_unchanged(gateway: Gateway, mode: str) -> None:
    with (
        wire_server(_anthropic_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _anthropic_model(scenario, provider)
        guardrail: Final = _guardrail(gateway, scenario, guard, mode)
        body: Final = _body(model, _SEVERAL_TOOL_RESULTS, max_tokens=64, guardrails=[guardrail])
        created: Final = gateway.request("POST", "/v1/messages", body)
        assert created.status_code == 200, created.text
        forwarded: Final = _only(provider.drain(), "/v1/messages")
        first_turn: Final = _first_turn(forwarded["messages"])
        assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn
        tool_results: Final = _forwarded_tool_results(forwarded)
        assert _CITY not in json.dumps(tool_results), tool_results
        assert (tool_results["toolu_01"], tool_results["toolu_02"]) == _RESTORED_TOOL_RESULTS[mode], tool_results
        assert _texts(tool_results["toolu_03"]) == (_redacted(_WEEKEND),), tool_results


def _null_text_answering(tool_use_id: str) -> list[JsonValue]:
    return [
        {"role": "user", "content": f"What is the weather in {_CITY}?"},
        {"role": "assistant", "content": [_tool_use(tool_use_id)]},
        {"role": "user", "content": [_tool_result(tool_use_id, None)]},
    ]


@pytest.mark.parametrize("tool_use_id", ("toolu_01", "functions.get_weather:0"))
def test_rewriting_guardrail_leaves_the_rejection_to_the_chat_bridge_for_any_tool_use_id(
    gateway: Gateway, tool_use_id: str
) -> None:
    with (
        wire_server(_chat_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _mistral_model(scenario, provider)
        guardrail: Final = _guardrail(gateway, scenario, guard, "rows")
        body: Final = _body(model, _null_text_answering(tool_use_id), max_tokens=64, guardrails=[guardrail])
        response: Final = gateway.request("POST", "/v1/messages", body)
        message: Final = _anthropic_error_message(response.status_code, response.text, response.content)
        assert "content.0.text: Input should be a valid string" in message, message
        assert provider.drain() == ()


def _files_peer(request: Request) -> Reply:
    if request.method == "POST" and request.target.endswith("/files"):
        uploaded: Final = {
            "id": f"file-{uuid.uuid4().hex}",
            "object": "file",
            "bytes": len(request.body),
            "created_at": 1789788253,
            "filename": "batch.jsonl",
            "purpose": "batch",
            "status": "processed",
        }
        return Reply(body=json.dumps(uploaded).encode())
    if request.target.endswith("/models"):
        return Reply(body=json.dumps({"object": "list", "data": []}).encode())
    return Reply(status=404, body=json.dumps({"error": request.target}).encode())


def _uploaded_records(request: Request) -> tuple[dict[str, JsonValue], ...]:
    envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
    parts: Final = BytesParser(policy=HTTP).parsebytes(envelope).iter_parts()
    files: Final = tuple(part.get_payload(decode=True) for part in parts if part.get_filename() is not None)
    assert len(files) == 1, request.headers
    payload: Final = files[0]
    assert isinstance(payload, bytes), payload
    return tuple(_JSON_OBJECT.validate_json(line) for line in payload.splitlines() if line.strip())


def _listed_models(gateway: Gateway) -> frozenset[str]:
    with httpx.Client(base_url=gateway.client.base_url, timeout=15, trust_env=False) as client:
        listed: Final = client.get("/v1/models", headers={"Authorization": f"Bearer {gateway.key}"})
    assert listed.status_code == 200, listed.text
    entries: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(listed.content)["data"])
    return frozenset(string_value(entry["id"]) for entry in entries)


def _uploaded_batch_body(
    gateway: Gateway, scenario: Scenario, guard: Wire, mode: str, messages: list[JsonValue]
) -> dict[str, JsonValue]:
    with wire_server(_files_peer) as provider:
        model: Final = _openai_model(scenario, provider)
        key: Final = scenario.key(metadata={"guardrails": [_guardrail(gateway, scenario, guard, mode)]})
        eventually(
            lambda: tuple(_listed_models(gateway) for _ in range(8)),
            lambda listings: all(model in listing for listing in listings),
            seconds=90,
        )
        record: Final = {
            "custom_id": f"weather-{uuid.uuid4().hex}",
            "method": "POST",
            "url": "/v1/messages",
            "body": _body(model, messages, max_tokens=64),
        }
        uploaded: Final = gateway.request_multipart(
            "/v1/files",
            {"purpose": "batch", "model": model},
            {"file": ("batch.jsonl", json.dumps(record).encode() + b"\n", "application/jsonl")},
            key=key,
        )
        assert uploaded.status_code == 200, uploaded.text
        uploads: Final = tuple(request for request in provider.drain() if request.target.endswith("/files"))
    assert len(uploads) == 1, [request.target for request in uploads]
    (forwarded,) = _uploaded_records(uploads[0])
    assert forwarded["custom_id"] == record["custom_id"], forwarded
    return _JSON_OBJECT.validate_python(forwarded["body"])


def test_batch_upload_behind_a_rewriting_guardrail_keeps_the_malformed_text(gateway: Gateway) -> None:
    with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario:
        forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "rows", _NUMBER_THEN_WEATHER)
    first_turn: Final = _first_turn(forwarded_body["messages"])
    assert _REDACTED_CITY in first_turn and _CITY not in first_turn, first_turn
    assert _forwarded_tool_result(forwarded_body) == [_text(5), _text(_WEATHER)], forwarded_body


def test_guardrail_that_rewrites_nothing_forwards_a_null_tool_result_text_unchanged(gateway: Gateway) -> None:
    with (
        wire_server(_anthropic_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _anthropic_model(scenario, provider)
        guardrail: Final = _guardrail(gateway, scenario, guard, "none")
        created: Final = gateway.request(
            "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, guardrails=[guardrail])
        )
        assert created.status_code == 200, created.text
        assert len(guard.drain()) == 1
        assert _forwarded_tool_result(_only(provider.drain(), "/v1/messages")) == [{"type": "text", "text": None}]


def _forwarded_chat_turns(wire: Wire) -> tuple[dict[str, JsonValue], ...]:
    return _JSON_OBJECTS.validate_python(_only(wire.drain(), "/chat/completions")["messages"])


def test_mistral_receives_the_tool_reply_whose_only_text_drop_params_dropped(gateway: Gateway) -> None:
    with wire_server(_chat_peer) as wire, gateway.scenario() as scenario:
        model: Final = _mistral_model(scenario, wire)
        created: Final = gateway.request(
            "POST", "/v1/messages", _body(model, _NULL_TEXT, max_tokens=64, drop_params=True)
        )
        assert created.status_code == 200, created.text
        turns: Final = _forwarded_chat_turns(wire)
        assert [turn["role"] for turn in turns] == ["user", "assistant", "tool"], turns
        calls: Final = _JSON_OBJECTS.validate_python(turns[1]["tool_calls"])
        assert [call["id"] for call in calls] == [turns[2]["tool_call_id"]], turns
        assert _tool_message_text({"messages": list(turns)}) == "", turns


_WEATHER_CALL: Final[dict[str, JsonValue]] = {
    "id": "call_weather",
    "type": "function",
    "function": {"name": "get_weather", "arguments": json.dumps({"city": _CITY})},
}


def test_mistral_chat_completions_keeps_an_empty_tool_reply(gateway: Gateway) -> None:
    ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}"
    messages: Final[list[JsonValue]] = [
        {"role": "system", "content": ""},
        {"role": "user", "content": ask},
        {"role": "assistant", "content": ""},
        {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]},
        {"role": "tool", "tool_call_id": "call_weather", "content": ""},
        {"role": "user", "content": ""},
    ]
    with wire_server(_chat_peer) as wire, gateway.scenario() as scenario:
        model: Final = _mistral_model(scenario, wire)
        response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages})
        assert response.status_code == 200, response.text
        turns: Final = _forwarded_chat_turns(wire)
        assert [(turn["role"], turn.get("content")) for turn in turns] == [
            ("user", ask),
            ("assistant", None),
            ("tool", ""),
        ], turns
        assert turns[2]["tool_call_id"] == "call_weather", turns


def test_mistral_chat_completions_drops_an_orphan_tool_message_and_sends_a_null_tool_reply_as_empty_text(
    gateway: Gateway,
) -> None:
    ask: Final = f"What is the weather in {_CITY}? {uuid.uuid4().hex}"
    messages: Final[list[JsonValue]] = [
        {"role": "user", "content": ask},
        {"role": "tool", "tool_call_id": "call_orphan", "content": ""},
        {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]},
        {"role": "tool", "tool_call_id": "call_weather", "content": None},
    ]
    with wire_server(_chat_peer) as wire, gateway.scenario() as scenario:
        model: Final = _mistral_model(scenario, wire)
        response: Final = gateway.request("POST", "/v1/chat/completions", {"model": model, "messages": messages})
        assert response.status_code == 200, response.text
        turns: Final = _forwarded_chat_turns(wire)
    assert [(turn["role"], turn.get("tool_call_id"), turn.get("content")) for turn in turns] == [
        ("user", None, ask),
        ("assistant", None, None),
        ("tool", "call_weather", ""),
    ], turns


_NATIVE_DROP_PARAMS_CASES: Final = MappingProxyType(
    {
        "body": (_DROP_SOURCES["body"], [_text(_WEATHER)]),
        "deployment": (_DROP_SOURCES["deployment"], [_text(_WEATHER)]),
        "explicit-false": (
            (MappingProxyType({"drop_params": True}), MappingProxyType({"drop_params": False}), _CLAUDE_CODE),
            [_text(5), _text(_WEATHER)],
        ),
    }
)


@pytest.mark.parametrize("case", sorted(_NATIVE_DROP_PARAMS_CASES))
def test_native_anthropic_resolves_drop_params_the_same_way_on_both_routes(gateway: Gateway, case: str) -> None:
    (deployment, extra, headers), forwarded = _NATIVE_DROP_PARAMS_CASES[case]
    with wire_server(_anthropic_peer) as wire, gateway.scenario() as scenario:
        model: Final = _anthropic_model(scenario, wire, **deployment)
        body: Final = _body(model, _NUMBER_THEN_WEATHER, **extra)
        created: Final = gateway.request("POST", "/v1/messages", {**body, "max_tokens": 64}, headers=headers)
        counted: Final = gateway.request("POST", "/v1/messages/count_tokens", body, headers=headers)
        assert created.status_code == 200, created.text
        assert counted.status_code == 200, counted.text
        assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == _ANTHROPIC_COUNT, counted.text
        requests: Final = wire.drain()
    assert _forwarded_tool_result(_only(requests, "/v1/messages")) == forwarded
    assert _forwarded_tool_result(_only(requests, "/v1/messages/count_tokens")) == forwarded


_CARD_NUMBER: Final = 4111111111111111
_CARD_NULL_WEATHER: Final = _messages(_text(_CARD_NUMBER), _text(None), _text(_WEATHER))
_SCANNED_CARD_NULL_WEATHER: Final = ("4111111111111111", "null", _WEATHER)
_MASKED_CARD_NULL_WEATHER: Final = [_text("################"), _text(None), _text("sunny, ##C")]


def _scanned_tool_texts(guard: Wire) -> tuple[JsonValue, ...]:
    requests: Final = guard.drain()
    assert len(requests) == 1, [request.target for request in requests]
    rows: Final = _JSON_OBJECTS.validate_python(_json(requests[0])["structured_messages"])
    tool_rows: Final = tuple(row for row in rows if row.get("role") == "tool")
    assert len(tool_rows) == 1, rows
    return _texts(tool_rows[0]["content"])


def test_masking_guardrail_scans_a_numeric_tool_result_text_before_anthropic_gets_it(gateway: Gateway) -> None:
    with (
        wire_server(_anthropic_peer) as provider,
        wire_server(_guardrail_peer) as guard,
        gateway.scenario() as scenario,
    ):
        model: Final = _anthropic_model(scenario, provider)
        guardrail: Final = _guardrail(gateway, scenario, guard, "digits")
        created: Final = gateway.request(
            "POST", "/v1/messages", _body(model, _CARD_NULL_WEATHER, max_tokens=64, guardrails=[guardrail])
        )
        assert created.status_code == 200, created.text
        scanned: Final = _scanned_tool_texts(guard)
        forwarded: Final = _forwarded_tool_result(_only(provider.drain(), "/v1/messages"))
    assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned
    assert forwarded == _MASKED_CARD_NULL_WEATHER, forwarded


def test_masking_guardrail_scans_a_numeric_tool_result_text_in_a_batch_upload(gateway: Gateway) -> None:
    with wire_server(_guardrail_peer) as guard, gateway.scenario() as scenario:
        forwarded_body: Final = _uploaded_batch_body(gateway, scenario, guard, "digits", _CARD_NULL_WEATHER)
        scanned: Final = _scanned_tool_texts(guard)
    assert scanned == _SCANNED_CARD_NULL_WEATHER, scanned
    assert _forwarded_tool_result(forwarded_body) == _MASKED_CARD_NULL_WEATHER, forwarded_body