Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

75.0 KB
Raw Blame
import asyncio
import json
import threading
import time
from collections import Counter
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Final
from uuid import uuid4

import anthropic
import httpx
import openai
import pytest
from integration._support.client import Gateway, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter

from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE
from litellm.responses.utils import ResponsesAPIRequestUtils

pytestmark: Final = pytest.mark.shares_worker_state

JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
REDACTED: Final = "REDACTED_BY_LITELM"
TOOL_INPUT: Final = {"key": "order-123", "sort_key": "created_at"}
ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-4-5-20250929"


def _prompt_storage_config(
    tmp_path: Path,
    *,
    store_prompts: bool = True,
    local_cache: bool = False,
    model_list: tuple[dict[str, JsonValue], ...] = (),
) -> Path:
    config: Final = tmp_path / f"spend-log-content-{uuid4()}.json"
    settings: Final = {"cache": True, "cache_params": {"type": "local"}} if local_cache else {}
    config.write_text(
        json.dumps(
            {
                "model_list": list(model_list),
                "general_settings": {
                    "master_key": "os.environ/LITELLM_MASTER_KEY",
                    "database_url": "os.environ/DATABASE_URL",
                    "disable_responses_id_security": True,
                    "store_model_in_db": True,
                    "store_prompts_in_spend_logs": store_prompts,
                    "proxy_batch_write_at": 1,
                    "proxy_batch_polling_interval": 1,
                },
                "litellm_settings": settings,
            }
        )
    )
    return config


def _json_object(body: bytes) -> dict[str, JsonValue]:
    return JSON_OBJECT.validate_json(body)


def _answering_model_listing(respond: Callable[[Request], Reply]) -> Callable[[Request], Reply]:
    def answer(request: Request) -> Reply:
        if request.method == "GET":
            assert request.target == "/v1/models", request.target
            return Reply(body=b'{"object":"list","data":[]}')
        return respond(request)

    return answer


def _provider_calls(requests: tuple[Request, ...]) -> tuple[Request, ...]:
    return tuple(request for request in requests if request.method != "GET" or request.target != "/v1/models")


def _objects(value: JsonValue) -> tuple[dict[str, JsonValue], ...]:
    assert isinstance(value, list)
    return tuple(object_value(item) for item in value)


def _sse_events(body: str) -> tuple[dict[str, JsonValue], ...]:
    return tuple(
        _json_object(line.removeprefix("data:").strip().encode())
        for line in body.splitlines()
        if line.startswith("data:") and line.removeprefix("data:").strip() != "[DONE]"
    )


def _spend_request_id(response_id: str, *, responses_api: bool = False) -> str:
    if not responses_api:
        return response_id
    decoded: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id)
    request_id: Final = decoded.get("response_id")
    return string_value(request_id) if isinstance(request_id, str) else response_id


def _stored_row(response_id: str, *, responses_api: bool = False) -> dict[str, JsonValue]:
    request_id: Final = _spend_request_id(response_id, responses_api=responses_api)
    rows: Final = eventually(
        lambda: read_rows(
            'SELECT request_id, proxy_server_request, response, status FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
            (request_id,),
        ),
        lambda values: len(values) == 1,
        seconds=70,
    )
    assert rows[0]["request_id"] == request_id
    return rows[0]


def _stored_cache_hit_row(response_id: str) -> dict[str, JsonValue]:
    request_id_prefix: Final = f"{response_id}_cache_hit"
    rows: Final = eventually(
        lambda: read_rows(
            'SELECT request_id, proxy_server_request, response, status, cache_hit FROM "LiteLLM_SpendLogs" '
            "WHERE LEFT(request_id, LENGTH(%s)) = %s",
            (request_id_prefix, request_id_prefix),
        ),
        lambda values: len(values) == 1,
        seconds=70,
    )
    row: Final = rows[0]
    assert string_value(row["request_id"]).startswith(request_id_prefix), row
    assert row["cache_hit"] == "True", row
    assert row["proxy_server_request"] is not None, row
    assert row["response"] is not None, row
    return row


def _stored_rows(response_ids: tuple[str, ...], *, responses_api: bool = False) -> tuple[dict[str, JsonValue], ...]:
    request_ids: Final = tuple(
        _spend_request_id(response_id, responses_api=responses_api) for response_id in response_ids
    )
    placeholders: Final = ", ".join("%s" for _ in request_ids)
    rows: Final = eventually(
        lambda: read_rows(
            'SELECT request_id, proxy_server_request, response, status FROM "LiteLLM_SpendLogs" '
            f"WHERE request_id IN ({placeholders})",
            request_ids,
        ),
        lambda values: len(values) == len(request_ids),
        seconds=70,
    )
    observed_ids: Final = tuple(string_value(row["request_id"]) for row in rows)
    assert Counter(observed_ids) == Counter(request_ids), rows
    rows_by_id: Final = {string_value(row["request_id"]): row for row in rows}
    return tuple(rows_by_id[request_id] for request_id in request_ids)


def _chat_completion(response_id: str, text: str, *, logprobs: bool = False) -> dict[str, JsonValue]:
    logprob_content: Final = [
        {
            "token": token,
            "logprob": -0.1,
            "bytes": [115],
            "top_logprobs": [{"token": token, "logprob": -0.1}],
        }
        for token in ("sort", "_key")
    ]
    choice: Final = {
        "index": 0,
        "message": {"role": "assistant", "content": text},
        "finish_reason": "stop",
        **({"logprobs": {"content": logprob_content}} if logprobs else {}),
    }
    return {
        "id": response_id,
        "object": "chat.completion",
        "created": 1,
        "model": "gpt-4o-mini",
        "choices": [choice],
        "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
        "system_fingerprint": "fp_scripted",
    }


def _chat_stream(response_id: str, text: str, *, include_usage: bool = False) -> Reply:
    base: Final = {
        "id": response_id,
        "object": "chat.completion.chunk",
        "created": 1,
        "model": "gpt-4o-mini",
    }
    frames: Final = (
        {**base, "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}]},
        {
            **base,
            "choices": [
                {
                    "index": 0,
                    "delta": {"content": text},
                    "finish_reason": None,
                    "logprobs": {
                        "content": [
                            {"token": "sort", "logprob": -0.1, "top_logprobs": [{"token": "sort", "logprob": -0.1}]}
                        ],
                    },
                }
            ],
        },
        {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
        *(
            (
                {
                    **base,
                    "choices": [],
                    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
                },
            )
            if include_usage
            else ()
        ),
    )
    chunks: Final = tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in frames) + (b"data: [DONE]\n\n",)
    return Reply(content_type="text/event-stream", chunks=chunks)


def _anthropic_message(
    response_id: str,
    text: str,
    *,
    tool_input: dict[str, JsonValue] | None = None,
) -> dict[str, JsonValue]:
    content: Final = (
        [{"type": "tool_use", "id": "toolu_scripted", "name": "lookup", "input": tool_input}]
        if tool_input is not None
        else [{"type": "text", "text": text}]
    )
    return {
        "id": response_id,
        "type": "message",
        "role": "assistant",
        "model": "claude-sonnet-4-5-20250929",
        "content": content,
        "stop_reason": "tool_use" if tool_input is not None else "end_turn",
        "stop_sequence": None,
        "usage": {"input_tokens": 1, "output_tokens": 1},
    }


def _anthropic_sse(response_id: str, text: str) -> tuple[bytes, ...]:
    events: Final = (
        (
            "message_start",
            {
                "type": "message_start",
                "message": {
                    "id": response_id,
                    "type": "message",
                    "role": "assistant",
                    "model": "claude-sonnet-4-5-20250929",
                    "content": [],
                    "stop_reason": None,
                    "stop_sequence": None,
                    "usage": {"input_tokens": 1, "output_tokens": 0},
                },
            },
        ),
        (
            "content_block_start",
            {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
        ),
        (
            "content_block_delta",
            {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}},
        ),
        ("content_block_stop", {"type": "content_block_stop", "index": 0}),
        (
            "message_delta",
            {
                "type": "message_delta",
                "delta": {"stop_reason": "end_turn", "stop_sequence": None},
                "usage": {"output_tokens": 1},
            },
        ),
        ("message_stop", {"type": "message_stop"}),
    )
    return tuple(f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode() for event, payload in events)


def _responses_text(response_body: dict[str, JsonValue]) -> str:
    output: Final = _objects(response_body["output"])
    content: Final = _objects(output[0]["content"])
    return string_value(content[0]["text"])


def test_stored_chat_response_keeps_logprob_tokens(gateway: Gateway, tmp_path: Path) -> None:
    response_body: Final = {
        "id": f"chatcmpl-logprobs-{uuid4()}",
        "object": "chat.completion",
        "created": 1,
        "model": "gpt-4o-mini",
        "choices": [
            {
                "index": 0,
                "message": {"role": "assistant", "content": "sort_key"},
                "finish_reason": "stop",
                "logprobs": {
                    "content": [
                        {
                            "token": "sort",
                            "logprob": -0.1,
                            "bytes": [115],
                            "top_logprobs": [{"token": "sort", "logprob": -0.1}],
                        },
                        {
                            "token": "_key",
                            "logprob": -0.2,
                            "bytes": [95],
                            "top_logprobs": [{"token": "_key", "logprob": -0.2}],
                        },
                    ]
                },
            }
        ],
        "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
        "system_fingerprint": "fp_scripted",
    }

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        return Reply(body=json.dumps(response_body).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(
            model="openai/gpt-4o-mini",
            api_base=wire.url,
            api_key="synthetic-openai-key",
        )
        api_key: Final = scenario.key(key_alias=f"spend-log-h1-{uuid4()}", models=[model])
        response: Final = isolated.request(
            "POST",
            "/v1/chat/completions",
            {
                "model": model,
                "messages": [{"role": "user", "content": "hi"}],
                "logprobs": True,
                "top_logprobs": 1,
                "prompt_cache_key": "tenant-42-cache",
                "aws_secret_access_key": "AKIAEXAMPLESECRET",
                "secret_fields": {"raw_headers": {"authorization": "Bearer secret-h1"}},
                "metadata": {"user_api_key_alias": "alias-h1", "user_api_key_hash": "hash-h1"},
            },
            key=api_key,
        )
        assert response.status_code == 200, response.text
        caller_response: Final = _json_object(response.content)
        response_id: Final = string_value(caller_response["id"])
        caller_choice: Final = object_value(_objects(caller_response["choices"])[0])
        caller_message: Final = object_value(caller_choice["message"])
        assert caller_message["role"] == "assistant"
        assert caller_message["content"] == "sort_key"
        assert caller_response["system_fingerprint"] == "fp_scripted"
        upstream: Final = _provider_calls(wire.drain())
        post_requests: Final = tuple(request for request in upstream if request.method == "POST")
        assert len(post_requests) == 1
        upstream_body: Final = _json_object(post_requests[0].body)
        assert upstream_body["logprobs"] is True
        assert upstream_body["top_logprobs"] == 1
        assert upstream_body["messages"] == [{"role": "user", "content": "hi"}]
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_response: Final = object_value(row["response"])
        logprob_content: Final = stored_response["choices"][0]["logprobs"]["content"]
        response_tokens: Final = [item["token"] for item in logprob_content]
        top_logprob_tokens: Final = [item["top_logprobs"][0]["token"] for item in logprob_content]
        assert response_tokens == ["sort", "_key"]
        assert top_logprob_tokens == ["sort", "_key"]
        assert stored_response["system_fingerprint"] == REDACTED
        assert stored_request["prompt_cache_key"] == REDACTED
        assert stored_request["aws_secret_access_key"] == REDACTED
        assert "secret_fields" not in stored_request
        stored_metadata: Final = object_value(stored_request["metadata"])
        assert stored_metadata["user_api_key_alias"] == REDACTED
        assert stored_metadata["user_api_key_hash"] == REDACTED


def test_stored_messages_keep_tool_use_input(gateway: Gateway, tmp_path: Path) -> None:
    tool_input: Final = {"key": "order-123", "sort_key": "created_at"}
    tool_result: Final = [
        {"type": "tool_result", "tool_use_id": "toolu_01", "content": [{"type": "text", "text": "shipped"}]},
        {"type": "text", "text": "Now order-456"},
    ]
    response_body: Final = {
        "id": f"msg-tool-use-{uuid4()}",
        "type": "message",
        "role": "assistant",
        "model": "claude-sonnet-4-5-20250929",
        "content": [
            {
                "type": "tool_use",
                "id": "toolu_02",
                "name": "get_order",
                "input": {
                    "key": "order-456",
                    "partition_key": "tenant_42",
                    "access_level": "admin",
                    "token_type": "bearer",
                },
            }
        ],
        "stop_reason": "tool_use",
        "stop_sequence": None,
        "usage": {"input_tokens": 8, "output_tokens": 4},
    }

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        received: Final = _json_object(request.body)
        assert received["messages"][1]["content"][0]["input"] == tool_input
        return Reply(body=json.dumps(response_body).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(
            model="anthropic/claude-sonnet-4-5-20250929",
            api_base=wire.url,
            api_key="synthetic-anthropic-key",
        )
        response: Final = isolated.request(
            "POST",
            "/v1/messages",
            {
                "model": model,
                "max_tokens": 64,
                "aws_secret_access_key": "AKIAEXAMPLESECRET",
                "messages": [
                    {"role": "user", "content": "Look up order order-123."},
                    {
                        "role": "assistant",
                        "content": [
                            {
                                "type": "tool_use",
                                "id": "toolu_01",
                                "name": "get_order",
                                "input": tool_input,
                            }
                        ],
                    },
                    {"role": "user", "content": tool_result},
                ],
            },
        )
        assert response.status_code == 200, response.text
        caller_response: Final = _json_object(response.content)
        response_id: Final = string_value(caller_response["id"])
        caller_tool_input: Final = _objects(caller_response["content"])[0]["input"]
        assert caller_tool_input == {
            "key": "order-456",
            "partition_key": "tenant_42",
            "access_level": "admin",
            "token_type": "bearer",
        }
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_response: Final = object_value(row["response"])
        assert stored_request["messages"][1]["content"][0]["input"] == tool_input
        stored_response_tool_arguments: Final = stored_response["choices"][0]["message"]["tool_calls"][0]["function"][
            "arguments"
        ]
        assert json.loads(stored_response_tool_arguments) == {
            "key": "order-456",
            "partition_key": "tenant_42",
            "access_level": "admin",
            "token_type": "bearer",
        }
        assert stored_request["aws_secret_access_key"] == REDACTED
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert _json_object(observed[0].body)["messages"][1]["content"][0]["input"] == tool_input


def test_previous_response_id_replay_sends_real_tool_payloads(gateway: Gateway, tmp_path: Path) -> None:
    function_arguments: Final = {"sort_key": "created_at", "access_level": "admin"}
    function_output: Final = {
        "status": "active",
        "token_type": "bearer",
        "partition_key": "tenant_42",
    }

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        return Reply(body=json.dumps(_anthropic_message(f"msg-responses-replay-{uuid4()}", "OK")).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path), workers=2) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(
            model="anthropic/claude-sonnet-4-5-20250929",
            api_base=wire.url,
            api_key="synthetic-anthropic-key",
        )
        first_response: Final = isolated.request(
            "POST",
            "/v1/responses",
            {
                "model": model,
                "input": [
                    {"role": "user", "content": "Fetch my account settings."},
                    {
                        "type": "function_call",
                        "call_id": "call_1",
                        "name": "get_settings",
                        "arguments": function_arguments,
                    },
                    {
                        "type": "function_call_output",
                        "call_id": "call_1",
                        "output": function_output,
                    },
                    {"role": "user", "content": "Acknowledge with OK"},
                ],
                "aws_secret_access_key": "AKIAEXAMPLESECRET",
            },
        )
        assert first_response.status_code == 200, first_response.text
        first_body: Final = object_value(first_response.json())
        response_id: Final = string_value(first_body["id"])
        assert _responses_text(first_body) == "OK"
        first_row: Final = _stored_row(response_id, responses_api=True)
        stored_request: Final = object_value(first_row["proxy_server_request"])
        assert stored_request["input"][1]["arguments"] == function_arguments
        assert stored_request["input"][2]["output"] == function_output
        assert stored_request["aws_secret_access_key"] == REDACTED
        second_response: Final = isolated.request(
            "POST",
            "/v1/responses",
            {"model": model, "previous_response_id": response_id, "input": "List the values"},
        )
        assert second_response.status_code == 200, second_response.text
        second_body: Final = object_value(second_response.json())
        second_response_id: Final = string_value(second_body["id"])
        assert second_response_id != response_id
        assert _responses_text(second_body) == "OK"
        second_row: Final = _stored_row(second_response_id, responses_api=True)
        second_stored_request: Final = object_value(second_row["proxy_server_request"])
        assert second_stored_request["input"] == "List the values"
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 2
        second_request: Final = _json_object(observed[1].body)
        assert second_request["messages"][1]["role"] == "assistant"
        assert second_request["messages"][2]["role"] == "user"
        assistant_content: Final = _objects(object_value(second_request["messages"][1])["content"])
        user_content: Final = _objects(object_value(second_request["messages"][2])["content"])
        tool_use: Final = tuple(block for block in assistant_content if block.get("type") == "tool_use")
        tool_result_blocks: Final = tuple(block for block in user_content if block.get("type") == "tool_result")
        assert len(tool_use) == 1
        assert len(tool_result_blocks) == 1
        assert tool_use[0]["input"] == function_arguments
        replayed_output: Final = tool_result_blocks[0]["content"]
        assert isinstance(replayed_output, str)
        assert JSON_OBJECT.validate_json(replayed_output) == function_output
        assert REDACTED not in replayed_output


def _openai_sdk_chat_response_id(
    isolated: Gateway,
    model: str,
    *,
    client_kind: str,
    messages: list[dict[str, JsonValue]],
) -> str:
    if client_kind == "sync":
        with openai.OpenAI(
            base_url=f"{isolated.client.base_url}/v1",
            api_key=isolated.key,
            max_retries=0,
            http_client=httpx.Client(trust_env=False, timeout=30),
        ) as client:
            response: Final = client.chat.completions.create(
                model=model,
                messages=messages,
                logprobs=True,
                top_logprobs=1,
                extra_body={"prompt_cache_key": "tenant-42-cache", "aws_secret_access_key": "AKIAEXAMPLESECRET"},
            )
            assert response.choices[0].message.content == "sort_key"
            assert response.choices[0].logprobs is not None
            assert response.choices[0].logprobs.content[0].token == "sort"
            return response.id

    async def call() -> str:
        async with openai.AsyncOpenAI(
            base_url=f"{isolated.client.base_url}/v1",
            api_key=isolated.key,
            max_retries=0,
            http_client=httpx.AsyncClient(trust_env=False, timeout=30),
        ) as client:
            response: Final = await client.chat.completions.create(
                model=model,
                messages=messages,
                logprobs=True,
                top_logprobs=1,
                extra_body={"prompt_cache_key": "tenant-42-cache", "aws_secret_access_key": "AKIAEXAMPLESECRET"},
            )
            assert response.choices[0].message.content == "sort_key"
            assert response.choices[0].logprobs is not None
            assert response.choices[0].logprobs.content[0].token == "sort"
            return response.id

    return asyncio.run(call())


def _openai_sdk_stream_response_id(isolated: Gateway, model: str, messages: list[dict[str, JsonValue]]) -> str:
    async def call() -> str:
        async with openai.AsyncOpenAI(
            base_url=f"{isolated.client.base_url}/v1",
            api_key=isolated.key,
            max_retries=0,
            http_client=httpx.AsyncClient(trust_env=False, timeout=30),
        ) as client:
            stream: Final = await client.chat.completions.create(
                model=model,
                messages=messages,
                logprobs=True,
                top_logprobs=1,
                stream=True,
                stream_options={"include_usage": True},
                extra_body={"prompt_cache_key": "tenant-42-cache", "aws_secret_access_key": "AKIAEXAMPLESECRET"},
            )
            chunks: Final = [chunk async for chunk in stream]
            assert chunks[0].choices[0].delta.content == ""
            assert chunks[-1].usage is not None
            assert chunks[-1].usage.total_tokens == 2
            return chunks[0].id

    return asyncio.run(call())


@pytest.mark.parametrize("client_kind", ("sync", "async"), ids=("sync", "async"))
def test_chat_sdk_keeps_logprob_tokens(gateway: Gateway, tmp_path: Path, client_kind: str) -> None:
    response_id_from_wire: Final = f"chatcmpl-sdk-logprobs-{uuid4()}"

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["messages"] == [{"role": "user", "content": "hi"}]
        assert received["logprobs"] is True
        return Reply(body=json.dumps(_chat_completion(response_id_from_wire, "sort_key", logprobs=True)).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(
            model="openai/gpt-4o-mini",
            api_base=wire.url,
            api_key="synthetic-openai-key",
        )
        response_id: Final = _openai_sdk_chat_response_id(
            isolated,
            model,
            client_kind=client_kind,
            messages=[{"role": "user", "content": "hi"}],
        )
        assert response_id == response_id_from_wire
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_response: Final = object_value(row["response"])
        assert [entry["token"] for entry in stored_response["choices"][0]["logprobs"]["content"]] == ["sort", "_key"]
        assert stored_request["prompt_cache_key"] == REDACTED
        assert stored_request["aws_secret_access_key"] == REDACTED
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert _json_object(observed[0].body)["messages"] == [{"role": "user", "content": "hi"}]


def test_chat_sdk_stream_include_usage_masks_request_fields(gateway: Gateway, tmp_path: Path) -> None:
    response_id_from_wire: Final = f"chatcmpl-sdk-stream-{uuid4()}"
    messages: Final = [
        {"role": "user", "content": "stream control"},
        {
            "role": "assistant",
            "tool_calls": [
                {
                    "id": "call_stream",
                    "type": "function",
                    "function": {"name": "lookup", "arguments": '{"sort_key":"created_at"}'},
                }
            ],
        },
        {"role": "tool", "tool_call_id": "call_stream", "content": "done"},
    ]

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["stream"] is True
        assert received["stream_options"] == {"include_usage": True}
        assert received["messages"] == messages
        return _chat_stream(response_id_from_wire, "sort_key", include_usage=True)

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(
            model="openai/gpt-4o-mini",
            api_base=wire.url,
            api_key="synthetic-openai-key",
        )
        response_id: Final = _openai_sdk_stream_response_id(isolated, model, messages)
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        assert stored_request["prompt_cache_key"] == REDACTED
        assert stored_request["aws_secret_access_key"] == REDACTED
        assert len(_stored_rows((response_id,))) == 1
        observed: Final = _provider_calls(wire.drain())
        post_requests: Final = tuple(request for request in observed if request.method == "POST")
        assert len(post_requests) == 1


@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
def test_chat_stream_usage_the_proxy_adds_keeps_its_marker_out_of_the_stored_request(
    gateway: Gateway, tmp_path: Path
) -> None:
    response_id: Final = f"chatcmpl-proxy-usage-{uuid4()}"
    with (
        wire_server(
            _answering_model_listing(lambda request: _chat_stream(response_id, "sort_key", include_usage=True))
        ) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
        response: Final = isolated.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": "stream marker"}], "stream": True},
        )
        assert response.status_code == 200, response.text
        assert {string_value(event["id"]) for event in _sse_events(response.text)} == {response_id}, response.text
        stored_request: Final = object_value(_stored_row(response_id)["proxy_server_request"])
        assert stored_request["messages"] == [{"role": "user", "content": "stream marker"}], stored_request
        assert "_litellm_stream_usage_injected" not in stored_request, stored_request
        observed: Final = _provider_calls(wire.drain())
        assert [_json_object(request.body)["stream_options"] for request in observed] == [{"include_usage": True}]


def test_chat_history_keeps_string_tool_arguments_and_tool_content(gateway: Gateway, tmp_path: Path) -> None:
    arguments: Final = '{"sort_key":"created_at"}'
    tool_content: Final = "tool-result-created_at"
    response_id_from_wire: Final = f"chatcmpl-history-{uuid4()}"
    messages: Final = [
        {"role": "user", "content": "history control"},
        {
            "role": "assistant",
            "tool_calls": [
                {
                    "id": "call_history",
                    "type": "function",
                    "function": {"name": "lookup", "arguments": arguments},
                }
            ],
        },
        {"role": "tool", "tool_call_id": "call_history", "content": tool_content},
    ]

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["messages"] == messages
        return Reply(body=json.dumps(_chat_completion(response_id_from_wire, "done")).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
        response: Final = isolated.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": messages},
        )
        assert response.status_code == 200, response.text
        response_id: Final = string_value(_json_object(response.content)["id"])
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        assert stored_request["messages"][1]["tool_calls"][0]["function"]["arguments"] == arguments
        assert stored_request["messages"][2]["content"] == tool_content
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert _json_object(observed[0].body)["messages"] == messages


def _anthropic_sdk_message_response_id(
    isolated: Gateway,
    model: str,
    *,
    client_kind: str,
    messages: list[dict[str, JsonValue]],
    expected_input: dict[str, JsonValue],
) -> str:
    if client_kind == "sync":
        with anthropic.Anthropic(
            base_url=str(isolated.client.base_url),
            api_key=isolated.key,
            max_retries=0,
            http_client=httpx.Client(trust_env=False, timeout=30),
        ) as client:
            response: Final = client.messages.create(model=model, max_tokens=64, messages=messages)
            block: Final = response.content[0]
            assert block.type == "tool_use"
            assert block.input == expected_input
            return response.id

    async def call() -> str:
        async with anthropic.AsyncAnthropic(
            base_url=str(isolated.client.base_url),
            api_key=isolated.key,
            max_retries=0,
            http_client=httpx.AsyncClient(trust_env=False, timeout=30),
        ) as client:
            response: Final = await client.messages.create(model=model, max_tokens=64, messages=messages)
            block: Final = response.content[0]
            assert block.type == "tool_use"
            assert block.input == expected_input
            return response.id

    return asyncio.run(call())


@pytest.mark.parametrize("client_kind", ("sync", "async"), ids=("sync", "async"))
def test_messages_sdk_keeps_tool_use_input(gateway: Gateway, tmp_path: Path, client_kind: str) -> None:
    request_tool_input: Final = {"key": "order-123", "sort_key": "created_at"}
    response_tool_input: Final = {
        "key": "order-456",
        "partition_key": "tenant_42",
        "access_level": "admin",
        "token_type": "bearer",
    }
    response_id_from_wire: Final = f"msg-sdk-tool-{uuid4()}"
    messages: Final = [
        {"role": "user", "content": "SDK tool control"},
        {
            "role": "assistant",
            "content": [{"type": "tool_use", "id": "toolu_sdk", "name": "lookup", "input": request_tool_input}],
        },
    ]

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["messages"][1]["content"][0]["input"] == request_tool_input
        return Reply(
            body=json.dumps(
                _anthropic_message(response_id_from_wire, "unused", tool_input=response_tool_input)
            ).encode()
        )

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        response_id: Final = _anthropic_sdk_message_response_id(
            isolated,
            model,
            client_kind=client_kind,
            messages=messages,
            expected_input=response_tool_input,
        )
        assert response_id == response_id_from_wire
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_response: Final = object_value(row["response"])
        assert stored_request["messages"][1]["content"][0]["input"] == request_tool_input
        assert (
            JSON_OBJECT.validate_json(
                stored_response["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"]
            )
            == response_tool_input
        )
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert _json_object(observed[0].body)["messages"][1]["content"][0]["input"] == request_tool_input


def test_messages_stream_keeps_tool_use_input(gateway: Gateway, tmp_path: Path) -> None:
    request_tool_input: Final = {"key": "order-123", "sort_key": "created_at"}
    response_id_from_wire: Final = f"msg-stream-tool-{uuid4()}"
    messages: Final = [
        {"role": "user", "content": "stream tool control"},
        {
            "role": "assistant",
            "content": [{"type": "tool_use", "id": "toolu_stream", "name": "lookup", "input": request_tool_input}],
        },
    ]

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["stream"] is True
        assert received["messages"][1]["content"][0]["input"] == request_tool_input
        return Reply(content_type="text/event-stream", chunks=_anthropic_sse(response_id_from_wire, "streamed"))

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        async_client: Final = anthropic.AsyncAnthropic(
            base_url=str(isolated.client.base_url),
            api_key=isolated.key,
            max_retries=0,
            http_client=httpx.AsyncClient(trust_env=False, timeout=30),
        )

        async def call() -> str:
            async with async_client:
                stream: Final = await async_client.messages.create(
                    model=model,
                    max_tokens=64,
                    messages=messages,
                    stream=True,
                )
                events: Final = [event async for event in stream]
                assert events[0].type == "message_start"
                assert events[-1].type == "message_stop"
                return events[0].message.id

        response_id: Final = asyncio.run(call())
        assert response_id == response_id_from_wire
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        assert stored_request["messages"][1]["content"][0]["input"] == request_tool_input
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1


def test_responses_stream_keeps_function_call_arguments(gateway: Gateway, tmp_path: Path) -> None:
    function_arguments: Final = {"sort_key": "created_at", "access_level": "admin"}
    response_id_from_wire: Final = f"msg-responses-stream-{uuid4()}"

    def respond(request: Request) -> Reply:
        assert request.target == "/v1/messages"
        received: Final = _json_object(request.body)
        assert received["stream"] is True
        assert "created_at" in request.body.decode()
        return Reply(content_type="text/event-stream", chunks=_anthropic_sse(response_id_from_wire, "streamed"))

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        response: Final = isolated.request(
            "POST",
            "/v1/responses",
            {
                "model": model,
                "stream": True,
                "input": [
                    {"role": "user", "content": "stream response control"},
                    {
                        "type": "function_call",
                        "call_id": "call_stream",
                        "name": "lookup",
                        "arguments": function_arguments,
                    },
                ],
            },
        )
        assert response.status_code == 200, response.text
        events: Final = _sse_events(response.text)
        completed_event: Final = next(event for event in events if event["type"] == "response.completed")
        completed_response: Final = object_value(completed_event["response"])
        response_id: Final = string_value(completed_response["id"])
        assert _responses_text(completed_response) == "streamed"
        row: Final = _stored_row(response_id, responses_api=True)
        stored_request: Final = object_value(row["proxy_server_request"])
        assert stored_request["input"][1]["arguments"] == function_arguments
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert "created_at" in observed[0].body.decode()


def test_native_responses_keeps_logprob_tokens(gateway: Gateway, tmp_path: Path) -> None:
    response_id_from_wire: Final = f"resp-native-logprobs-{uuid4()}"
    response_body: Final = {
        "id": response_id_from_wire,
        "object": "response",
        "created_at": 1,
        "status": "completed",
        "model": "gpt-4o-mini",
        "prompt_cache_key": "tenant-42",
        "output": [
            {
                "type": "message",
                "id": f"msg-native-{uuid4()}",
                "status": "completed",
                "role": "assistant",
                "content": [
                    {
                        "type": "output_text",
                        "text": "sort",
                        "annotations": [],
                        "logprobs": [
                            {
                                "token": "sort",
                                "logprob": -0.1,
                                "bytes": [115],
                                "top_logprobs": [{"token": "sort", "logprob": -0.1}],
                            }
                        ],
                    }
                ],
            }
        ],
        "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
    }

    def respond(request: Request) -> Reply:
        assert request.target.endswith("/responses"), request.target
        assert not request.target.endswith("/chat/completions"), request.target
        received: Final = _json_object(request.body)
        assert received["include"] == ["message.output_text.logprobs"]
        assert received["top_logprobs"] == 1
        assert received["prompt_cache_key"] == "tenant-42"
        return Reply(body=json.dumps(response_body).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
        response: Final = isolated.request(
            "POST",
            "/v1/responses",
            {
                "model": model,
                "input": "native response logprob control",
                "include": ["message.output_text.logprobs"],
                "top_logprobs": 1,
                "prompt_cache_key": "tenant-42",
            },
        )
        assert response.status_code == 200, response.text
        caller_body: Final = object_value(response.json())
        response_id: Final = string_value(caller_body["id"])
        assert _responses_text(caller_body) == "sort"
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_response: Final = object_value(row["response"])
        stored_output: Final = object_value(_objects(stored_response["output"])[0])
        stored_content: Final = object_value(_objects(stored_output["content"])[0])
        stored_logprobs: Final = _objects(stored_content["logprobs"])
        assert stored_logprobs[0]["token"] == "sort"
        assert stored_request["prompt_cache_key"] == REDACTED
        assert stored_response["prompt_cache_key"] == REDACTED
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert observed[0].target.endswith("/responses"), observed[0].target


def test_malformed_tool_blocks_keep_only_recognized_content(gateway: Gateway, tmp_path: Path) -> None:
    extra_blocks: Final = [
        {"type": "tool_use", "api_key": "sk-sibling-secret", "input": {"sort_key": "created_at"}},
        {"type": {"bad": 1}, "input": {"api_key": "sk-malformed-secret"}},
    ]
    response_id_from_wire: Final = f"chat-extra-blocks-{uuid4()}"

    def respond(request: Request) -> Reply:
        assert request.method == "POST"
        return Reply(body=json.dumps(_chat_completion(response_id_from_wire, "done")).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
        response: Final = isolated.request(
            "POST",
            "/v1/chat/completions",
            {
                "model": model,
                "messages": [{"role": "user", "content": "extra blocks control"}],
                "extra_blocks": extra_blocks,
            },
        )
        assert response.status_code == 200, response.text
        response_id: Final = string_value(_json_object(response.content)["id"])
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_blocks: Final = _objects(stored_request["extra_blocks"])
        assert stored_blocks[0]["api_key"] == REDACTED
        assert object_value(stored_blocks[1]["input"])["api_key"] == REDACTED
        assert object_value(stored_blocks[0]["input"])["sort_key"] == "created_at"
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1
        assert _json_object(observed[0].body)["messages"] == [{"role": "user", "content": "extra blocks control"}]


def test_messages_tool_input_handles_mixed_values_and_truncation(gateway: Gateway, tmp_path: Path) -> None:
    long_partition_key: Final = "x" * 5000
    tool_input: Final = {
        "sort_key": 7,
        "access_level": ["admin"],
        "token_type": "",
        "partition_key": long_partition_key,
        "key": "dup",
        "sort_key_copy": "dup",
    }
    response_id_from_wire: Final = f"msg-mixed-tool-input-{uuid4()}"

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["messages"][1]["content"][0]["input"] == tool_input
        return Reply(body=json.dumps(_anthropic_message(response_id_from_wire, "done")).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        response: Final = isolated.request(
            "POST",
            "/v1/messages",
            {
                "model": model,
                "max_tokens": 64,
                "messages": [
                    {"role": "user", "content": "mixed tool control"},
                    {
                        "role": "assistant",
                        "content": [{"type": "tool_use", "id": "toolu_mixed", "name": "lookup", "input": tool_input}],
                    },
                ],
            },
        )
        assert response.status_code == 200, response.text
        response_id: Final = string_value(_json_object(response.content)["id"])
        row: Final = _stored_row(response_id)
        stored_request: Final = object_value(row["proxy_server_request"])
        stored_tool_input: Final = _objects(object_value(stored_request["messages"][1])["content"])[0]["input"]
        stored_tool_input_object: Final = object_value(stored_tool_input)
        assert stored_tool_input_object["sort_key"] == 7
        assert stored_tool_input_object["access_level"] == ["admin"]
        assert stored_tool_input_object["token_type"] == ""
        assert stored_tool_input_object["key"] == "dup"
        assert stored_tool_input_object["sort_key_copy"] == "dup"
        partition_key: Final = string_value(stored_tool_input_object["partition_key"])
        assert REDACTED not in partition_key
        assert LITELLM_TRUNCATED_PAYLOAD_FIELD in partition_key
        assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in partition_key
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1


def test_messages_without_auth_create_no_spend_row(gateway: Gateway, tmp_path: Path) -> None:
    marker: Final = f"unauthenticated-spend-marker-{uuid4()}"

    def respond(_: Request) -> Reply:
        raise AssertionError("Unauthenticated requests must not reach the upstream")

    with (
        wire_server(respond) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
    ):
        response: Final = isolated.client.post(
            "/v1/messages",
            json={
                "model": "missing-model",
                "max_tokens": 8,
                "messages": [{"role": "user", "content": marker}],
            },
        )
        assert response.status_code == 401, response.text
        rows: Final = eventually(
            lambda: read_rows(
                'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE proxy_server_request::text LIKE %s',
                (f"%{marker}%",),
            ),
            lambda values: bool(values),
            seconds=1,
            return_last_on_timeout=True,
        )
        assert rows == []
        assert wire.drain() == ()


def test_messages_upstream_error_keeps_tool_input(gateway: Gateway, tmp_path: Path) -> None:
    marker: Final = f"upstream-error-{uuid4()}"
    tool_input: Final = {"key": marker, "sort_key": "created_at"}
    error_body: Final = {
        "type": "error",
        "error": {"type": "invalid_request_error", "message": "synthetic upstream error"},
    }

    def respond(request: Request) -> Reply:
        received: Final = _json_object(request.body)
        assert received["messages"][1]["content"][0]["input"] == tool_input
        return Reply(status=400, content_type="application/json", body=json.dumps(error_body).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        response: Final = isolated.request(
            "POST",
            "/v1/messages",
            {
                "model": model,
                "max_tokens": 64,
                "messages": [
                    {"role": "user", "content": marker},
                    {
                        "role": "assistant",
                        "content": [{"type": "tool_use", "id": "toolu_error", "name": "lookup", "input": tool_input}],
                    },
                ],
            },
        )
        assert 400 <= response.status_code < 500, response.text
        assert response.status_code != 500
        assert "synthetic upstream error" in response.text
        rows: Final = eventually(
            lambda: read_rows(
                'SELECT proxy_server_request, status FROM "LiteLLM_SpendLogs" WHERE proxy_server_request::text LIKE %s',
                (f"%{marker}%",),
            ),
            lambda values: len(values) == 1,
            seconds=70,
        )
        stored_request: Final = object_value(rows[0]["proxy_server_request"])
        assert object_value(stored_request["messages"][1])["content"][0]["input"] == tool_input
        assert rows[0]["status"] == "failure"
        observed: Final = _provider_calls(wire.drain())
        assert len(observed) == 1


def test_store_prompts_off_keeps_chat_and_messages_representation_equal(gateway: Gateway, tmp_path: Path) -> None:
    chat_response_id: Final = f"chat-store-off-{uuid4()}"
    messages_response_id: Final = f"msg-store-off-{uuid4()}"

    def respond(request: Request) -> Reply:
        if request.target.endswith("/chat/completions"):
            return Reply(body=json.dumps(_chat_completion(chat_response_id, "chat")).encode())
        return Reply(body=json.dumps(_anthropic_message(messages_response_id, "messages")).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(
            gateway,
            tmp_path,
            {},
            config=_prompt_storage_config(tmp_path, store_prompts=False),
        ) as isolated,
        isolated.scenario() as scenario,
    ):
        chat_model: Final = scenario.model(
            model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key"
        )
        messages_model: Final = scenario.model(
            model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key"
        )
        chat_response: Final = isolated.request(
            "POST",
            "/v1/chat/completions",
            {"model": chat_model, "messages": [{"role": "user", "content": "store off chat"}]},
        )
        messages_response: Final = isolated.request(
            "POST",
            "/v1/messages",
            {
                "model": messages_model,
                "max_tokens": 8,
                "messages": [{"role": "user", "content": "store off messages"}],
            },
        )
        assert chat_response.status_code == 200, chat_response.text
        assert messages_response.status_code == 200, messages_response.text
        chat_id: Final = string_value(_json_object(chat_response.content)["id"])
        messages_id: Final = string_value(_json_object(messages_response.content)["id"])
        chat_row: Final = _stored_row(chat_id)
        messages_row: Final = _stored_row(messages_id)
        assert object_value(chat_row["proxy_server_request"]) == {}
        assert object_value(messages_row["proxy_server_request"]) == {}
        assert len(_provider_calls(wire.drain())) == 2


def test_identical_messages_requests_have_distinct_spend_rows(gateway: Gateway, tmp_path: Path) -> None:
    marker: Final = f"identical-messages-{uuid4()}"
    request_body: Final = {
        "model": "",
        "max_tokens": 8,
        "messages": [{"role": "user", "content": marker}],
    }

    def respond(_: Request) -> Reply:
        return Reply(body=json.dumps(_anthropic_message(f"msg-identical-{uuid4()}", "same")).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        body: Final = {**request_body, "model": model}
        response_ids: Final = tuple(
            string_value(_json_object(isolated.request("POST", "/v1/messages", body).content)["id"]) for _ in range(3)
        )
        assert len(set(response_ids)) == 3
        assert len(_stored_rows(response_ids)) == 3
        assert len(_provider_calls(wire.drain())) == 3


def test_chat_cache_hit_keeps_logprob_tokens(gateway: Gateway, tmp_path: Path) -> None:
    def respond(_: Request) -> Reply:
        return Reply(body=json.dumps(_chat_completion(f"chat-cache-{uuid4()}", "sort_key", logprobs=True)).encode())

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(
            gateway,
            tmp_path,
            {},
            config=_prompt_storage_config(tmp_path, local_cache=True),
            workers=2,
        ) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key")
        body: Final = {
            "model": model,
            "messages": [{"role": "user", "content": "cache logprob control"}],
            "logprobs": True,
            "top_logprobs": 1,
            "prompt_cache_key": "tenant-42-cache",
            "aws_secret_access_key": "AKIAEXAMPLESECRET",
            "secret_fields": {"raw_headers": {"authorization": "Bearer secret-cache"}},
        }
        first: Final = isolated.request("POST", "/v1/chat/completions", body)
        second: Final = isolated.request("POST", "/v1/chat/completions", body)
        assert first.status_code == 200, first.text
        assert second.status_code == 200, second.text
        first_id: Final = string_value(_json_object(first.content)["id"])
        second_id: Final = string_value(_json_object(second.content)["id"])
        second_row: Final = _stored_cache_hit_row(second_id)
        stored_request: Final = object_value(second_row["proxy_server_request"])
        stored_response: Final = object_value(second_row["response"])
        assert [entry["token"] for entry in stored_response["choices"][0]["logprobs"]["content"]] == ["sort", "_key"]
        assert stored_request["prompt_cache_key"] == REDACTED
        assert stored_request["aws_secret_access_key"] == REDACTED
        assert "secret_fields" not in stored_request
        assert stored_response["system_fingerprint"] == REDACTED
        assert len(_provider_calls(wire.drain())) == 1
        assert len(_stored_rows((first_id,))) == 1


def test_messages_cache_hit_keeps_tool_input(gateway: Gateway, tmp_path: Path) -> None:
    tool_input: Final = {"key": "cache-order", "sort_key": "created_at"}

    def respond(_: Request) -> Reply:
        return Reply(
            body=json.dumps(_anthropic_message(f"msg-cache-{uuid4()}", "done", tool_input=tool_input)).encode()
        )

    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(
            gateway,
            tmp_path,
            {},
            config=_prompt_storage_config(tmp_path, local_cache=True),
            workers=2,
        ) as isolated,
        isolated.scenario() as scenario,
    ):
        model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
        body: Final = {
            "model": model,
            "max_tokens": 64,
            "aws_secret_access_key": "AKIAEXAMPLESECRET",
            "secret_fields": {"raw_headers": {"authorization": "Bearer secret-cache"}},
            "messages": [
                {"role": "user", "content": "cache tool control"},
                {
                    "role": "assistant",
                    "content": [{"type": "tool_use", "id": "toolu_cache", "name": "lookup", "input": tool_input}],
                },
            ],
        }
        first: Final = isolated.request("POST", "/v1/messages", body)
        second: Final = isolated.request("POST", "/v1/messages", body)
        assert first.status_code == 200, first.text
        assert second.status_code == 200, second.text
        first_id: Final = string_value(_json_object(first.content)["id"])
        second_id: Final = string_value(_json_object(second.content)["id"])
        second_row: Final = _stored_cache_hit_row(second_id)
        stored_request: Final = object_value(second_row["proxy_server_request"])
        assert stored_request["messages"][1]["content"][0]["input"] == tool_input
        assert stored_request["aws_secret_access_key"] == REDACTED
        assert "secret_fields" not in stored_request
        assert len(_provider_calls(wire.drain())) == 1
        assert len(_stored_rows((first_id,))) == 1


def _burst_case(
    index: int,
    chat_model: str,
    messages_model: str,
    *,
    prefix: str,
) -> tuple[str, str, dict[str, JsonValue]]:
    marker: Final = f"{prefix}-{uuid4()}"
    match index % 5:
        case 0:
            return (
                "chat_nonstream",
                marker,
                {
                    "model": chat_model,
                    "messages": [{"role": "user", "content": marker}],
                    "logprobs": True,
                    "top_logprobs": 1,
                },
            )
        case 1:
            return (
                "chat_stream",
                marker,
                {
                    "model": chat_model,
                    "messages": [{"role": "user", "content": marker}],
                    "logprobs": True,
                    "top_logprobs": 1,
                    "stream": True,
                },
            )
        case 2:
            return (
                "messages_nonstream",
                marker,
                {
                    "model": messages_model,
                    "max_tokens": 16,
                    "messages": [
                        {"role": "user", "content": marker},
                        {
                            "role": "assistant",
                            "content": [
                                {
                                    "type": "tool_use",
                                    "id": "toolu_burst",
                                    "name": "lookup",
                                    "input": {"sort_key": marker},
                                }
                            ],
                        },
                    ],
                },
            )
        case 3:
            return (
                "messages_stream",
                marker,
                {
                    "model": messages_model,
                    "max_tokens": 16,
                    "messages": [
                        {"role": "user", "content": marker},
                        {
                            "role": "assistant",
                            "content": [
                                {
                                    "type": "tool_use",
                                    "id": "toolu_burst_stream",
                                    "name": "lookup",
                                    "input": {"sort_key": marker},
                                }
                            ],
                        },
                    ],
                    "stream": True,
                },
            )
        case _:
            return (
                "responses_stream" if index % 2 else "responses_nonstream",
                marker,
                {
                    "model": messages_model,
                    "input": [
                        {"role": "user", "content": marker},
                        {
                            "type": "function_call",
                            "call_id": "call_burst",
                            "name": "lookup",
                            "arguments": {"sort_key": marker},
                        },
                    ],
                    **({"stream": True} if index % 2 else {}),
                },
            )


def _burst_marker(request: Request, prefix: str) -> str:
    tokens: Final = request.body.decode().replace('"', " ").replace(",", " ").split()
    marker: Final = next(
        (token.strip("[]{}:,") for token in tokens if token.startswith(prefix)),
        None,
    )
    assert marker is not None, f"No {prefix} marker in {request.target}: {request.body.decode()}"
    return marker


def _burst_model_list(chat_model: str, messages_model: str, api_base: str) -> tuple[dict[str, JsonValue], ...]:
    return (
        {
            "model_name": chat_model,
            "litellm_params": {
                "model": "openai/gpt-4o-mini",
                "api_base": api_base,
                "api_key": "synthetic-openai-key",
            },
        },
        {
            "model_name": messages_model,
            "litellm_params": {
                "model": ANTHROPIC_MODEL,
                "api_base": api_base,
                "api_key": "synthetic-anthropic-key",
            },
        },
    )


def _assert_burst_upstream(
    requests: tuple[Request, ...],
    prefix: str,
    markers: tuple[str, ...],
    expected_posts: int,
) -> None:
    assert len(requests) == expected_posts
    assert all(request.method == "POST" for request in requests)
    assert Counter(_burst_marker(request, prefix) for request in requests) == Counter(markers)


def _burst_response_id(response: httpx.Response, kind: str, marker: str) -> str:
    assert response.status_code == 200, f"{kind} {marker}: {response.text}"
    if kind.endswith("_nonstream"):
        body: Final = _json_object(response.content)
        if kind == "chat_nonstream":
            assert object_value(_objects(body["choices"])[0])["message"]["content"] == marker
        elif kind == "messages_nonstream":
            assert _objects(body["content"])[0]["text"] == marker
        else:
            assert _responses_text(body) == marker
        return string_value(body["id"])
    events: Final = _sse_events(response.text)
    if kind == "chat_stream":
        assert marker in response.text
        return string_value(events[0]["id"])
    if kind == "messages_stream":
        assert marker in response.text
        return string_value(object_value(events[0]["message"])["id"])
    completed: Final = next(event for event in events if event["type"] == "response.completed")
    completed_response: Final = object_value(completed["response"])
    assert _responses_text(completed_response) == marker
    return string_value(completed_response["id"])


def _burst_endpoint(kind: str) -> str:
    match kind:
        case "chat_nonstream" | "chat_stream":
            return "/v1/chat/completions"
        case "messages_nonstream" | "messages_stream":
            return "/v1/messages"
        case "responses_nonstream" | "responses_stream":
            return "/v1/responses"
        case _:
            raise AssertionError(f"Unknown burst request kind: {kind}")


async def _send_burst(
    isolated: Gateway,
    cases: tuple[tuple[str, str, dict[str, JsonValue]], ...],
) -> tuple[httpx.Response, ...]:
    async with httpx.AsyncClient(
        base_url=str(isolated.client.base_url),
        headers={"Authorization": f"Bearer {isolated.key}"},
        timeout=180,
        trust_env=False,
    ) as client:
        return tuple(await asyncio.gather(*(client.post(_burst_endpoint(kind), json=body) for kind, _, body in cases)))


def _assert_burst_row(
    row: dict[str, JsonValue],
    kind: str,
    marker: str,
    *,
    require_chat_logprobs: bool = True,
) -> None:
    stored_request: Final = object_value(row["proxy_server_request"])
    stored_response: Final = object_value(row["response"])
    assert marker in json.dumps(stored_request)
    assert marker in json.dumps(stored_response)
    if kind == "chat_nonstream" and require_chat_logprobs:
        choice: Final = _objects(stored_response["choices"])[0]
        logprobs: Final = _objects(object_value(choice["logprobs"])["content"])[0]
        assert logprobs["token"] == "sort"
    if kind == "chat_stream":
        choice: Final = _objects(stored_response["choices"])[0]
        assert object_value(choice["message"])["content"] == marker


@pytest.mark.timeout(240)
def test_concurrent_mixed_requests_land_once(gateway: Gateway, tmp_path: Path) -> None:
    def respond(request: Request) -> Reply:
        marker: Final = _burst_marker(request, "audit-x1")
        response_id: Final = f"{'chatcmpl' if request.target.endswith('/chat/completions') else 'msg'}-{uuid4()}"
        body: Final = _json_object(request.body)
        if request.target.endswith("/chat/completions"):
            if body.get("stream") is True:
                return _chat_stream(response_id, marker)
            return Reply(body=json.dumps(_chat_completion(response_id, marker, logprobs=True)).encode())
        if body.get("stream") is True:
            return Reply(content_type="text/event-stream", chunks=_anthropic_sse(response_id, marker))
        return Reply(body=json.dumps(_anthropic_message(response_id, marker)).encode())

    chat_model: Final = f"integration-x1-chat-{uuid4().hex}"
    messages_model: Final = f"integration-x1-messages-{uuid4().hex}"
    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(
            gateway,
            tmp_path,
            {},
            config=_prompt_storage_config(
                tmp_path,
                model_list=_burst_model_list(chat_model, messages_model, wire.url),
            ),
            workers=2,
        ) as isolated,
    ):
        cases: Final = tuple(_burst_case(index, chat_model, messages_model, prefix="audit-x1") for index in range(30))
        responses: Final = asyncio.run(_send_burst(isolated, cases))
        response_ids: Final = tuple(
            _burst_response_id(response, kind, marker) for response, (kind, marker, _) in zip(responses, cases)
        )
        assert len(set(response_ids)) == len(response_ids)
        rows: Final = tuple(
            _stored_row(response_id, responses_api=kind.startswith("responses"))
            for response_id, (kind, _, _) in zip(response_ids, cases)
        )
        for row, (kind, marker, _) in zip(rows, cases):
            _assert_burst_row(row, kind, marker)
        _assert_burst_upstream(
            _provider_calls(wire.drain()),
            "audit-x1",
            tuple(marker for _, marker, _ in cases),
            30,
        )


@pytest.mark.timeout(240)
def test_slow_upstream_burst_lands_once(gateway: Gateway, tmp_path: Path) -> None:
    def respond(request: Request) -> Reply:
        time.sleep(1)
        marker: Final = _burst_marker(request, "audit-x2")
        response_id: Final = f"{'chatcmpl' if request.target.endswith('/chat/completions') else 'msg'}-{uuid4()}"
        body: Final = _json_object(request.body)
        if request.target.endswith("/chat/completions"):
            if body.get("stream") is True:
                return _chat_stream(response_id, marker)
            return Reply(body=json.dumps(_chat_completion(response_id, marker, logprobs=True)).encode())
        if body.get("stream") is True:
            return Reply(content_type="text/event-stream", chunks=_anthropic_sse(response_id, marker))
        return Reply(body=json.dumps(_anthropic_message(response_id, marker)).encode())

    chat_model: Final = f"integration-x2-chat-{uuid4().hex}"
    messages_model: Final = f"integration-x2-messages-{uuid4().hex}"
    with (
        wire_server(_answering_model_listing(respond)) as wire,
        owned_proxy(
            gateway,
            tmp_path,
            {},
            config=_prompt_storage_config(
                tmp_path,
                model_list=_burst_model_list(chat_model, messages_model, wire.url),
            ),
            workers=2,
        ) as isolated,
    ):
        cases: Final = tuple(_burst_case(index, chat_model, messages_model, prefix="audit-x2") for index in range(15))
        responses: Final = asyncio.run(_send_burst(isolated, cases))
        response_ids: Final = tuple(
            _burst_response_id(response, kind, marker) for response, (kind, marker, _) in zip(responses, cases)
        )
        assert len(set(response_ids)) == len(response_ids)
        rows: Final = tuple(
            _stored_row(response_id, responses_api=kind.startswith("responses"))
            for response_id, (kind, _, _) in zip(response_ids, cases)
        )
        for row, (kind, marker, _) in zip(rows, cases):
            _assert_burst_row(row, kind, marker)
        _assert_burst_upstream(
            _provider_calls(wire.drain()),
            "audit-x2",
            tuple(marker for _, marker, _ in cases),
            15,
        )


@pytest.mark.timeout(240)
def test_upstream_stop_returns_errors_and_recovers(gateway: Gateway, tmp_path: Path) -> None:
    release: Final = threading.Event()

    def stopped_respond(_: Request) -> Reply:
        release.wait(timeout=5)
        return Reply(status=503, content_type="application/json", body=b'{"error":"synthetic upstream stopped"}')

    with (
        owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path), workers=2) as isolated,
        isolated.scenario() as scenario,
    ):
        with ThreadPoolExecutor(max_workers=1) as executor:
            with wire_server(_answering_model_listing(stopped_respond)) as wire:
                failed_model: Final = scenario.model(
                    model="openai/gpt-4o-mini",
                    api_base=wire.url,
                    api_key="synthetic-openai-key",
                )
                failed_cases: Final = tuple(
                    (
                        "chat_nonstream",
                        marker,
                        {
                            "model": failed_model,
                            "messages": [{"role": "user", "content": marker}],
                        },
                    )
                    for marker in (f"audit-x3-{uuid4()}" for _ in range(10))
                )
                future: Final = executor.submit(asyncio.run, _send_burst(isolated, failed_cases))
                arrived_provider_calls: Final = eventually(
                    lambda: _provider_calls(wire.drain()),
                    lambda requests: len(requests) >= 1,
                    seconds=20,
                )
                release.set()
            failed_upstream: Final = (*arrived_provider_calls, *_provider_calls(wire.drain()))
            assert failed_upstream
            failed_responses: Final = future.result(timeout=60)
            assert all(response.status_code >= 400 and response.text for response in failed_responses)
            health: Final = isolated.request("GET", "/health/liveliness")
            assert health.status_code == 200, health.text

            def recovered_respond(request: Request) -> Reply:
                marker: Final = _burst_marker(request, "audit-x3-recovery")
                return Reply(body=json.dumps(_chat_completion(f"chatcmpl-{uuid4()}", marker)).encode())

            with wire_server(_answering_model_listing(recovered_respond)) as recovered_wire:
                recovered_model: Final = scenario.model(
                    model="openai/gpt-4o-mini",
                    api_base=recovered_wire.url,
                    api_key="synthetic-openai-key",
                )
                recovery_markers: Final = tuple(f"audit-x3-recovery-{uuid4()}" for _ in range(5))
                recovered_cases: Final = tuple(
                    (
                        "chat_nonstream",
                        marker,
                        {
                            "model": recovered_model,
                            "messages": [{"role": "user", "content": marker}],
                        },
                    )
                    for marker in recovery_markers
                )
                recovered_responses: Final = asyncio.run(_send_burst(isolated, recovered_cases))
                recovered_ids: Final = tuple(
                    _burst_response_id(response, kind, marker)
                    for response, (kind, marker, _) in zip(recovered_responses, recovered_cases)
                )
                assert len(set(recovered_ids)) == 5
                recovered_rows: Final = _stored_rows(recovered_ids)
                for row, (_, marker, _) in zip(recovered_rows, recovered_cases):
                    _assert_burst_row(row, "chat_nonstream", marker, require_chat_logprobs=False)
                assert len(_provider_calls(recovered_wire.drain())) == 5