Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

46.4 KB
Raw Blame
import asyncio
import json
from collections.abc import AsyncIterator, Callable, Generator, Mapping
from contextlib import contextmanager
from datetime import datetime
from itertools import product
from types import MappingProxyType
from typing import Final, cast
from uuid import uuid4

import httpx
import pytest
import respx
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict

import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.prompt_cache_prediction import NativePredictionTarget, TokenCounter
from litellm.proxy.hooks.autorouter_baseline_cache import AutoRouterBaselineCache, CapturedBaselineObservation
from litellm.router import Router
from litellm.types.router import RetryPolicy
from litellm.types.utils import CallTypes, StandardLoggingRoutingDecision

pytestmark: Final = pytest.mark.asyncio


_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])


_OBJECTS: Final = TypeAdapter(dict[str, object])


_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]])


_MESSAGES_JSON: Final = """[{"role":"user","content":[
    {"type":"text","text":"stable","cache_control":{"type":"ephemeral","ttl":"1h"}},
    {"type":"text","text":"question"}]}]"""


_MODELS: Final = _MESSAGES.validate_json("""[
    {"model_name":"test-router","litellm_params":{"model":"auto_router/complexity_router",
      "complexity_router_config":{"tiers":{"SIMPLE":"sonnet","MEDIUM":"sonnet","COMPLEX":"sonnet",
      "REASONING":{"model_name":"opus","litellm_params":{"max_tokens":16}}},"session_affinity":false,
      "keyword_tier_rules":[{"keywords":["USE_OPUS"],"tier":"REASONING"}]}}},
    {"model_name":"sonnet","litellm_params":{"model":"anthropic/claude-sonnet-5","api_key":"test-selected"},
      "model_info":{"id":"selected"}},
    {"model_name":"opus","litellm_params":{"model":"anthropic/claude-opus-5","api_key":"test-selected"},
      "model_info":{"id":"baseline"}}]""")


def _message(completed: bool, model: str) -> Mapping[str, JsonValue]:
    return _JSON_OBJECT.validate_json(f"""{{
        "id":"msg_baseline_test","type":"message","role":"assistant","model":{json.dumps(model)},
        "content":{'[{"type":"text","text":"OK"}]' if completed else "[]"},
        "stop_reason":{'"end_turn"' if completed else "null"},"stop_sequence":null,
        "usage":{{"input_tokens":1000,"output_tokens":{10 if completed else 0},
          "cache_creation_input_tokens":5000,"cache_read_input_tokens":0,
          "cache_creation":{{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5000}}}}}}""")


_EVENTS: Final = _MESSAGES.validate_json("""[
    {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}},
    {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"OK"}},
    {"type":"content_block_stop","index":0},
    {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}},
    {"type":"message_stop"}
]""")


async def _count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
    assert model == "claude-opus-5"
    return 6000 if "question" in json.dumps(_JSON_OBJECT.validate_python(body)) else 5000


class _CallContext(TypedDict):
    litellm_logging_obj: NotRequired[ReadOnly[Logging]]
    litellm_call_id: ReadOnly[str]
    litellm_metadata: ReadOnly[Mapping[str, object]]
    litellm_session_id: ReadOnly[str]


def _kwargs(logging_obj: Logging, trusted: bool = True, *, explicit_logging: bool = True) -> _CallContext:
    context: Final = _OBJECTS.validate_json(
        '{"max_tokens":16,"litellm_metadata":{"user_api_key_hash":"test-caller-hash"}}'
    )
    Router._record_routing_decision(  # pyright: ignore[reportUnknownMemberType, reportPrivateUsage]  # production trusted stamp owner
        context,
        StandardLoggingRoutingDecision(
            router_model_name="test-router",
            router_type="complexity",
            routed_model="sonnet",
            cause="heuristic_scorer",
            conversation_continuing=True,
            savings_baseline_model="anthropic/claude-opus-5",
            savings_baseline_deployment_id="baseline",
        ),
    )
    metadata: Final = _OBJECTS.validate_python(context["litellm_metadata"])
    if not trusted:
        metadata["_autorouter_baseline_route"] = _JSON_OBJECT.validate_json(
            '{"router_name":"test-router","baseline_model":"anthropic/claude-opus-5","baseline_deployment_id":"baseline"}'
        )
    envelope: Final[_CallContext] = {
        "litellm_call_id": logging_obj.litellm_call_id,
        "litellm_session_id": "baseline-session",
        "litellm_metadata": metadata,
    }
    supplied: Final[_CallContext] = {**envelope, "litellm_logging_obj": logging_obj}
    return supplied if explicit_logging else envelope


def _stream(logging_obj: Logging) -> bool:
    return logging_obj.stream is True  # pyright: ignore[reportUnknownMemberType]  # normalize the legacy Logging flag


def _sse(completed: bool = True, model: str = "claude-sonnet-5") -> tuple[bytes, ...]:
    events: Final = (
        {
            "type": "message_start",
            "message": _message(False, model),
        },
        *_EVENTS,
    )
    return tuple(
        f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
        for event in (events if completed else events[:-1])
    )


def _upstream(request: httpx.Request) -> httpx.Response:
    body: Final = _JSON_OBJECT.validate_json(request.content)
    model: Final = body.get("model")
    assert isinstance(model, str)
    stream: Final = body.get("stream") is True
    content: Final = b"".join(_sse(model=model)) if stream else json.dumps(_message(True, model)).encode()
    return httpx.Response(
        200,
        content=content,
        request=request,
        headers=MappingProxyType({"content-type": "text/event-stream" if stream else "application/json"}),
    )


def _error(request: httpx.Request, code: int, message: str) -> httpx.Response:
    return httpx.Response(
        code,
        text='{"type":"error","error":{"type":"rate_limit_error","message":' + json.dumps(message) + "}}",
        headers=MappingProxyType({"retry-after": "0"}),
        request=request,
    )


@contextmanager
def _transport(upstream: Callable[[httpx.Request], httpx.Response]) -> Generator[respx.Route]:
    with respx.mock() as transport:
        yield transport.post("https://api.anthropic.com/v1/messages").mock(side_effect=upstream)


class _NativeOptions(TypedDict):
    api_key: NotRequired[ReadOnly[str]]
    num_retries: NotRequired[ReadOnly[int]]


async def _call(
    target: Router | None,
    logging_obj: Logging,
    *,
    trusted: bool = True,
    messages: str = _MESSAGES_JSON,
    explicit_logging: bool = True,
) -> None:
    invoke: Final = target.anthropic_messages if target else litellm.anthropic_messages  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # legacy native call signatures
    options: Final = _NativeOptions() if target else _NativeOptions(api_key="test-selected", num_retries=0)
    response: Final[object] = await invoke(  # pyright: ignore[reportUnknownVariableType]  # native Router returns an opaque SDK result
        model="test-router" if target else "anthropic/claude-sonnet-5",
        max_tokens=16,
        stream=_stream(logging_obj),
        messages=_MESSAGES.validate_json(messages),
        **options,
        **_kwargs(logging_obj, trusted, explicit_logging=explicit_logging),
    )
    assert response is not None
    if _stream(logging_obj):
        assert isinstance(response, AsyncIterator)
        stream: Final = cast(AsyncIterator[object], response)  # cast-ok: iterator checked; all items satisfy object
        assert tuple([chunk async for chunk in stream])


class _Capture(CustomLogger):
    def __init__(self, call_id: str) -> None:
        self.call_id: Final = call_id
        self.payloads: Final[asyncio.Queue[Mapping[str, object]]] = asyncio.Queue()

    async def async_log_success_event(
        self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
    ) -> None:
        payload: Final = _OBJECTS.validate_python(kwargs.get("standard_logging_object"))
        if payload.get("litellm_call_id") == self.call_id:
            self.payloads.put_nowait(payload)

    async def payload(self) -> Mapping[str, object]:
        return await asyncio.wait_for(self.payloads.get(), timeout=20)


class _Rig:
    def __init__(
        self,
        monkeypatch: pytest.MonkeyPatch,
        *,
        retries: int = 0,
        count: TokenCounter = _count,
        models: list[dict[str, JsonValue]] = _MODELS,
    ) -> None:
        self.router: Final = Router(
            model_list=models,
            num_retries=retries,
            retry_policy=RetryPolicy(RateLimitErrorRetries=retries),
            disable_cooldowns=True,
        )

        def router() -> Router:
            return self.router

        self.hook: Final = AutoRouterBaselineCache(None, router=router, token_counter=count)
        self.call_id: Final = uuid4().hex
        self.capture: Final = _Capture(self.call_id)
        monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
        for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
            monkeypatch.delenv(name, raising=False)
        monkeypatch.setattr(litellm, "callbacks", [self.hook])
        for name in ("success_callback", "failure_callback", "_async_failure_callback"):
            monkeypatch.setattr(litellm, name, [])
        monkeypatch.setattr(litellm, "_async_success_callback", [self.capture])

    def logging(self, stream: bool = False) -> Logging:
        return Logging(
            model="anthropic/claude-sonnet-5",
            messages=_MESSAGES.validate_json(_MESSAGES_JSON),
            stream=stream,
            call_type=CallTypes.anthropic_messages.value,
            start_time=datetime.now(),
            litellm_call_id=self.call_id,
            function_id=self.call_id,
            kwargs={"litellm_session_id": "baseline-session"},
        )


def _observation(payload: Mapping[str, object]) -> CapturedBaselineObservation:
    encoded: Final = payload["autorouter_baseline_observation"]
    assert isinstance(encoded, str)
    assert "test-selected" not in encoded and "stable" not in encoded and "x-api-key" not in encoded
    return CapturedBaselineObservation.model_validate_json(encoded)


@pytest.mark.parametrize("stream,baseline", ((False, False), (True, False), (False, True), (True, True)))
async def test_native_logging_captures_usage_without_publishing_hypothetical_savings(
    monkeypatch: pytest.MonkeyPatch,
    stream: bool,
    baseline: bool,
) -> None:
    rig: Final = _Rig(monkeypatch)
    messages: Final = _MESSAGES_JSON.replace("question", "question USE_OPUS") if baseline else _MESSAGES_JSON
    with _transport(_upstream):
        await _call(rig.router, rig.logging(stream), messages=messages)
        payload: Final = await rig.capture.payload()
    captured: Final = _observation(payload)
    assert payload["autorouter_savings"] is None
    assert _OBJECTS.validate_python(payload["autorouter_savings_estimate"])["reason"] == "pending_projection"
    assert captured.observation.outcome == "complete"
    assert captured.observation.baseline_equivalent == baseline
    assert captured.observation.usage is not None and captured.observation.usage.completion_tokens == 10
    assert captured.observation.plan is not None and captured.observation.plan.total_tokens == 6000


async def test_count_failure_preserves_initial_observed_equivalence(monkeypatch: pytest.MonkeyPatch) -> None:
    async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
        return None

    rig: Final = _Rig(monkeypatch, count=count)
    with _transport(_upstream):
        await _call(rig.router, rig.logging(), messages=_MESSAGES_JSON.replace("question", "question USE_OPUS"))
        captured: Final = _observation(await rig.capture.payload())
    assert captured.observation.baseline_equivalent and captured.observation.usage is not None
    assert captured.observation.plan is None and captured.observation.reason == "token_count_unavailable"


async def test_native_retry_is_uncertain_even_when_final_response_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
    rig: Final = _Rig(monkeypatch, retries=1)

    def upstream(request: httpx.Request) -> httpx.Response:
        return _upstream(request) if route.call_count else _error(request, 429, "retry")

    with _transport(upstream) as route:
        await _call(rig.router, rig.logging())
        captured: Final = _observation(await rig.capture.payload())
        assert route.call_count == 2
    assert captured.observation.outcome == "uncertain"
    assert captured.observation.reason == "retried_request"


async def test_caller_cannot_forge_an_observation_scope(monkeypatch: pytest.MonkeyPatch) -> None:
    rig: Final = _Rig(monkeypatch)
    with _transport(_upstream):
        await _call(None, rig.logging(), trusted=False)
        payload: Final = await rig.capture.payload()
    assert payload["autorouter_baseline_observation"] is None
    assert payload["autorouter_savings"] is None


@pytest.mark.parametrize(
    "model,key,endpoint",
    (
        ("claude-sonnet-5", "test-first", None),
        ("claude-opus-5", "test-second", None),
        ("claude-opus-5", "test-first", "https://example.test"),
    ),
)
async def test_count_memo_is_scoped_to_provider_recipient(model: str, key: str, endpoint: str | None) -> None:
    counts: Final = iter((5000, 6000))

    async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
        return next(counts)

    collector: Final = AutoRouterBaselineCache(None, token_counter=count)
    original: Final = NativePredictionTarget("claude-opus-5", "test-first")
    other: Final = NativePredictionTarget(model, key, endpoint)
    assert await collector._count(original, {}) == 5000  # pyright: ignore[reportPrivateUsage]
    assert await collector._count(other, {}) == 6000  # pyright: ignore[reportPrivateUsage]
    assert await collector._count(original, {}) == 5000  # pyright: ignore[reportPrivateUsage]


@pytest.mark.parametrize("stream", (False, True))
async def test_provider_counting_does_not_hold_the_inference_response(
    monkeypatch: pytest.MonkeyPatch,
    stream: bool,
) -> None:
    counting: Final = asyncio.Event()
    release: Final = asyncio.Event()

    async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
        counting.set()
        await release.wait()
        return await _count(model, api_key, body)

    rig: Final = _Rig(monkeypatch, count=count)
    try:
        with _transport(_upstream):
            await asyncio.wait_for(_call(rig.router, rig.logging(stream)), timeout=2)
            await asyncio.wait_for(counting.wait(), timeout=2)
            assert rig.capture.payloads.empty()
            release.set()
            assert _observation(await rig.capture.payload()).observation.plan is not None
    finally:
        release.set()


@pytest.mark.parametrize("baseline_effort", (None, "medium"))
@pytest.mark.parametrize(
    "automatic_system, caching",
    (
        (None, "explicit"),
        ("stable system", "request"),
        ([{"type": "text", "text": "stable system"}], "request"),
        ("stable system", "global"),
        ([{"type": "text", "text": "stable system"}], "configured"),
    ),
)
async def test_native_tier_switch_uses_baseline_settings_and_preserves_history(
    monkeypatch: pytest.MonkeyPatch,
    baseline_effort: str | None,
    automatic_system: str | list[dict[str, str]] | None,
    caching: str,
) -> None:
    from litellm.proxy.spend_tracking.baseline_accounting import BaselineHistory, advance_baseline_history

    models: Final = _MESSAGES.validate_python(
        [
            {
                "model_name": "test-router",
                "litellm_params": {
                    "model": "auto_router/complexity_router",
                    "complexity_router_config": {
                        "tiers": {
                            "SIMPLE": {"model_name": "sonnet", "litellm_params": {"reasoning_effort": "low"}},
                            "MEDIUM": {"model_name": "sonnet", "litellm_params": {"reasoning_effort": "low"}},
                            "COMPLEX": {"model_name": "sonnet", "litellm_params": {"reasoning_effort": "high"}},
                            "REASONING": "opus",
                        },
                        "session_affinity": False,
                        "keyword_tier_rules": [{"keywords": ["ESCALATE"], "tier": "COMPLEX"}],
                    },
                },
            },
            _MODELS[1],
            {
                "model_name": "opus",
                "model_info": {"id": "baseline"},
                "litellm_params": {
                    "model": "anthropic/claude-opus-5",
                    "api_key": "test-selected",
                    **({"reasoning_effort": baseline_effort} if baseline_effort else {}),
                },
            },
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models)
    monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", caching == "global")
    controls: Final = (
        {
            "cache_control_injection_points": [
                {"location": "message", "role": "system", "control": {"type": "ephemeral", "ttl": "1h"}},
                {"location": "message", "index": -1, "control": {"type": "ephemeral", "ttl": "1h"}},
            ]
        }
        if caching == "configured"
        else {"enable_prompt_caching": caching == "request"}
    )
    captures: Final[asyncio.Queue[CapturedBaselineObservation]] = asyncio.Queue()
    with _transport(_upstream) as route:
        for suffix in ("", " ESCALATE"):
            log: Final = rig.logging()
            await rig.router.anthropic_messages(
                model="test-router",
                max_tokens=4096,
                messages=(
                    [{"role": "user", "content": "question" + suffix}]
                    if automatic_system is not None
                    else _MESSAGES.validate_json(_MESSAGES_JSON.replace("question", "question" + suffix))
                ),
                system=automatic_system,
                **controls,
                litellm_logging_obj=log,
                litellm_call_id=rig.call_id,
                litellm_metadata={"user_api_key_hash": "test-caller-hash"},
                litellm_session_id="native-tiers",
            )
            captures.put_nowait(_observation(await rig.capture.payload()))
        first_wire, second_wire = (_JSON_OBJECT.validate_json(call.request.content) for call in route.calls)
    assert (first_wire.get("thinking"), first_wire.get("output_config")) != (
        second_wire.get("thinking"),
        second_wire.get("output_config"),
    )
    first, second = (captures.get_nowait() for _ in range(2))
    assert first.scope == second.scope
    assert first.observation.plan is not None and second.observation.plan is not None
    assert first.observation.plan.breakpoints[0] == second.observation.plan.breakpoints[0]
    assert len(first.observation.plan.breakpoints) == (2 if automatic_system is not None else 1)
    history, _ = advance_baseline_history(
        BaselineHistory(first_at=0.0),
        (first.observation.model_copy(update={"request_id": "first", "started_at": 10000.0, "available_at": 10001.0}),),
    )
    _, result = advance_baseline_history(
        history,
        (
            second.observation.model_copy(
                update={"request_id": "second", "started_at": 10020.0, "available_at": 10021.0}
            ),
        ),
    )
    assert result[0].usage is not None and result[0].usage.prompt_tokens_details.cached_tokens == 5000


@pytest.mark.parametrize("call_type", (CallTypes.acompletion, CallTypes.aresponses, CallTypes.anthropic_messages))
async def test_plain_requests_do_not_initialize_or_warn(
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
    call_type: CallTypes,
) -> None:
    rig: Final = _Rig(monkeypatch)
    logging: Final = rig.logging()
    await rig.hook.async_pre_call_deployment_hook(
        {
            "litellm_logging_obj": logging,
            "litellm_metadata": {"session_id": "ordinary"},
        },
        call_type,
    )
    assert logging.baseline_cache_context is None
    assert "baseline observation could not be initialized" not in caplog.text
    assert not rig.hook.counts


async def test_plain_fallback_invalidates_existing_autorouter_capture(monkeypatch: pytest.MonkeyPatch) -> None:
    rig: Final = _Rig(monkeypatch)
    logging: Final = rig.logging()
    await rig.hook.async_pre_call_deployment_hook(_kwargs(logging), CallTypes.anthropic_messages)
    assert logging.baseline_cache_context is not None
    await rig.hook.async_pre_call_deployment_hook({"litellm_logging_obj": logging}, CallTypes.anthropic_messages)
    assert logging.baseline_observation is not None
    assert logging.baseline_observation.observation.reason == "retried_request"


async def test_native_count_finishing_after_quarter_worker_budget_keeps_plan_and_spend(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from litellm.litellm_core_utils import logging_worker
    from litellm.litellm_core_utils.logging_worker import LoggingWorker

    counted: Final = asyncio.Event()
    loop: Final = asyncio.get_running_loop()
    real_time: Final = loop.time
    monkeypatch.setattr(loop, "time", lambda: real_time() + (2.05 if counted.is_set() else 0.0))

    async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
        counted.set()
        await asyncio.sleep(0.001)
        return await _count(model, api_key, body)

    worker: Final = LoggingWorker(timeout=8.0)
    monkeypatch.setattr(logging_worker, "GLOBAL_LOGGING_WORKER", worker)
    rig: Final = _Rig(monkeypatch, count=count)
    try:
        with _transport(_upstream):
            await _call(rig.router, rig.logging())
            payload: Final = await rig.capture.payload()
        observed: Final = _observation(payload).observation
        assert observed.outcome == "complete" and observed.reason is None
        assert observed.plan is not None and observed.plan.breakpoints[0].prefix_tokens == 5000
        assert payload["response_cost"] is not None and worker._timeout_total == 0
    finally:
        await worker.stop()


@pytest.mark.parametrize(
    "options,on_deployment",
    (
        ({"thinking": {"type": "enabled", "budget_tokens": 2048}}, False),
        ({"extra_body": {"speed": "fast", "output_config": {"effort": "high"}}}, False),
        *product(
            (
                {"container": {"id": "container_test"}},
                {"mcp_servers": [{"type": "url", "name": "test", "url": "https://example.com/mcp"}]},
                {"inference_geo": "us"},
                {"safeguards": [{"type": "default"}]},
            ),
            (False, True),
        ),
    ),
)
async def test_native_baseline_identity_keeps_the_actual_transformed_body(
    monkeypatch: pytest.MonkeyPatch, options: dict[str, JsonValue], on_deployment: bool
) -> None:
    models: Final = _MESSAGES.validate_python(
        [
            *_MODELS[:2],
            {
                **_MODELS[2],
                "litellm_params": {
                    **_JSON_OBJECT.validate_python(_MODELS[2]["litellm_params"]),
                    **(options if on_deployment else {}),
                },
            },
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models)
    log: Final = rig.logging()
    with _transport(_upstream):
        await rig.router.anthropic_messages(
            model="test-router",
            max_tokens=16,
            messages=_MESSAGES.validate_json(_MESSAGES_JSON.replace("question", "question USE_OPUS")),
            litellm_logging_obj=log,
            litellm_call_id=rig.call_id,
            litellm_metadata={"user_api_key_hash": "test-caller-hash"},
            litellm_session_id="native-identical",
            **({} if on_deployment else options),
        )
        observed: Final = _observation(await rig.capture.payload()).observation
    assert observed.outcome == "complete"
    assert log.baseline_cache_context is not None
    assert observed.baseline_equivalent and observed.usage is not None, (
        log.baseline_cache_context.baseline_body,
        log.baseline_cache_context.selected_body_digest,
    )

    from litellm.proxy.spend_tracking.baseline_accounting import BaselineHistory, advance_baseline_history

    _, estimates = advance_baseline_history(BaselineHistory(), (observed,))
    assert estimates[0].provenance == "observed_identical" and estimates[0].usage == observed.usage


@pytest.mark.parametrize("tier_limit", (8, 16))
@pytest.mark.parametrize("extra", ({}, {"max_tokens": 8}))
async def test_native_baseline_identity_respects_caller_limit_and_tier_override(
    monkeypatch: pytest.MonkeyPatch, tier_limit: int, extra: dict[str, int]
) -> None:
    models: Final = _MESSAGES.validate_python(
        [
            {
                "model_name": "test-router",
                "litellm_params": {
                    "model": "auto_router/complexity_router",
                    "complexity_router_config": {
                        "tiers": {
                            "SIMPLE": {"model_name": "opus", "litellm_params": {"max_tokens": tier_limit}},
                            "MEDIUM": {"model_name": "opus", "litellm_params": {"max_tokens": tier_limit}},
                            "COMPLEX": "opus",
                            "REASONING": "opus",
                        },
                        "session_affinity": False,
                    },
                },
            },
            {**_MODELS[2], "litellm_params": {**_MODELS[2]["litellm_params"], "max_tokens": 64}},
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models)
    with _transport(_upstream) as route:
        await rig.router.anthropic_messages(
            model="test-router",
            max_tokens=8,
            messages=_MESSAGES.validate_json(_MESSAGES_JSON),
            litellm_logging_obj=rig.logging(),
            litellm_call_id=rig.call_id,
            litellm_metadata={"user_api_key_hash": "test-caller-hash"},
            litellm_session_id="native-limits",
            extra_body=extra,
        )
        observed: Final = _observation(await rig.capture.payload()).observation
        wire: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content)
    assert wire["max_tokens"] == tier_limit
    assert observed.baseline_equivalent == (tier_limit == 8)


@pytest.mark.parametrize("nested", (False, True))
async def test_native_baseline_projection_matches_wire_parameter_placement(
    monkeypatch: pytest.MonkeyPatch,
    nested: bool,
) -> None:
    counted: Final[asyncio.Queue[Mapping[str, JsonValue]]] = asyncio.Queue()

    async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
        counted.put_nowait(body)
        return await _count(model, api_key, body)

    models: Final = _MESSAGES.validate_python(
        [
            {
                **entry,
                "litellm_params": {
                    **_JSON_OBJECT.validate_python(entry["litellm_params"]),
                    "model": "anthropic/claude-opus-5",
                },
            }
            if entry["model_name"] == "sonnet"
            else entry
            for entry in _MODELS
        ]
    )
    rig: Final = _Rig(monkeypatch, count=count, models=models)
    settings: Final = {"speed": "standard", "thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}}
    with _transport(_upstream) as route:
        await rig.router.anthropic_messages(
            model="test-router",
            max_tokens=4096,
            messages=_MESSAGES.validate_json(_MESSAGES_JSON),
            litellm_logging_obj=rig.logging(),
            litellm_call_id=rig.call_id,
            litellm_metadata={"user_api_key_hash": "test-caller-hash"},
            litellm_session_id="native-placement",
            **({"extra_body": settings} if nested else settings),
        )
        captured: Final = _observation(await rig.capture.payload())
        wire: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content)
    assert captured.observation.plan is not None and not captured.observation.baseline_equivalent
    projected: Final = counted.get_nowait()
    assert {key: projected[key] for key in settings if key in projected} == {
        key: wire[key] for key in settings if key in wire
    }
    assert {key: wire[key] for key in settings if key in wire} == ({} if nested else settings)


_PARITY_TOOL: Final = {"name": "custom", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}
_PARITY_SYSTEM: Final = [{"type": "text", "text": "stable system", "cache_control": {"type": "ephemeral"}}]
_PARITY_POINTS: Final = [{"location": "message", "role": "system"}, {"location": "message", "index": -1}]


@pytest.mark.parametrize(
    "caller,selected,baseline,summary",
    (
        pytest.param({"extra_body": {"cache_control": {"type": "ephemeral"}}}, {}, {}, False, id="envelope-control"),
        pytest.param({"extra_body": {"system": _PARITY_SYSTEM}}, {}, {}, False, id="envelope-system"),
        pytest.param(
            {"extra_body": {"messages": _MESSAGES.validate_json(_MESSAGES_JSON)}}, {}, {}, False, id="envelope-messages"
        ),
        pytest.param({}, {"tools": [_PARITY_TOOL]}, {}, False, id="selected-tool-mark"),
        pytest.param({}, {}, {"tools": [_PARITY_TOOL]}, False, id="baseline-tool-mark"),
        pytest.param({}, {"system": _PARITY_SYSTEM}, {"system": "baseline system"}, False, id="selected-system-mark"),
        pytest.param({}, {"system": "selected system"}, {"system": _PARITY_SYSTEM}, False, id="baseline-system-mark"),
        pytest.param({"system": None}, {}, {"system": "configured system"}, False, id="null-system"),
        pytest.param({"thinking": None}, {}, {"thinking": {"type": "adaptive"}}, False, id="null-thinking"),
        pytest.param({"tools": None}, {}, {"tools": [_PARITY_TOOL]}, False, id="null-tools"),
        pytest.param({"verbosity": "low", "instructions": "ignored"}, {}, {}, False, id="ignored-native-options"),
        pytest.param(
            {"messages": _MESSAGES.validate_json(_MESSAGES_JSON.replace("stable", " "))},
            {},
            {},
            False,
            id="empty-marked-block",
        ),
        pytest.param({"thinking": {"type": "adaptive"}}, {}, {}, True, id="reasoning-summary"),
        pytest.param(
            {"thinking": {"type": "adaptive"}, "additional_drop_params": ["thinking.display"]},
            {},
            {},
            True,
            id="drop-nested-option",
        ),
        pytest.param(
            {"cache_control_injection_points": _PARITY_POINTS},
            {"tools": [{**_PARITY_TOOL, "name": f"custom_{index}"} for index in range(4)]},
            {},
            False,
            id="configured-cap",
        ),
    ),
)
async def test_native_baseline_projection_matches_direct_baseline_request(
    monkeypatch: pytest.MonkeyPatch,
    caller: dict[str, JsonValue],
    selected: dict[str, JsonValue],
    baseline: dict[str, JsonValue],
    summary: bool,
) -> None:
    counted: Final[asyncio.Queue[Mapping[str, JsonValue]]] = asyncio.Queue()

    async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
        counted.put_nowait(body)
        return await _count(model, api_key, body)

    def upstream(request: httpx.Request) -> httpx.Response:
        body: Final = _JSON_OBJECT.validate_json(request.content)
        model: Final = body.get("model")
        assert isinstance(model, str)
        return httpx.Response(
            200,
            request=request,
            json={
                **_message(True, model),
                "usage": {"input_tokens": 6000, "output_tokens": 10},
            },
        )

    models: Final = _MESSAGES.validate_python(
        [
            _MODELS[0],
            {
                **_MODELS[1],
                "litellm_params": {**_JSON_OBJECT.validate_python(_MODELS[1]["litellm_params"]), **selected},
            },
            {
                **_MODELS[2],
                "litellm_params": {**_JSON_OBJECT.validate_python(_MODELS[2]["litellm_params"]), **baseline},
            },
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models, count=count)
    monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False)
    monkeypatch.setattr(litellm, "reasoning_auto_summary", summary)
    monkeypatch.delenv("LITELLM_REASONING_AUTO_SUMMARY", raising=False)
    request: Final = {
        "messages": [{"role": "user", "content": "question"}],
        **({"system": "stable system"} if "system" not in selected and "system" not in baseline else {}),
        "max_tokens": 4096,
        "enable_prompt_caching": True,
        **caller,
    }
    with _transport(upstream) as route:
        await rig.router.anthropic_messages(model="opus", **_JSON_OBJECT.validate_python(request))
        direct: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content)
        await rig.router.anthropic_messages(
            model="test-router",
            litellm_logging_obj=rig.logging(),
            litellm_call_id=rig.call_id,
            litellm_metadata={"user_api_key_hash": "test-caller-hash"},
            litellm_session_id="native-projection-parity",
            **_JSON_OBJECT.validate_python(request),
        )
        captured: Final = _observation(await rig.capture.payload())
    assert captured.observation.plan is not None, captured.observation.reason
    projected: Final = counted.get_nowait()
    assert {key: value for key, value in projected.items() if key not in ("metadata", "stream")} == {
        key: value for key, value in direct.items() if key not in ("metadata", "stream")
    }


@pytest.mark.parametrize(
    "selected,baseline,usage_field,observed_value,multiplier",
    (
        ({}, {"speed": "fast"}, "speed", "standard", 3.0),
        ({"inference_geo": "us"}, {}, "inference_geo", "us", 1.0),
    ),
)
async def test_native_baseline_prices_projected_settings_without_changing_actual_spend(
    monkeypatch: pytest.MonkeyPatch,
    selected: dict[str, JsonValue],
    baseline: dict[str, JsonValue],
    usage_field: str,
    observed_value: str,
    multiplier: float,
) -> None:
    from litellm.proxy.spend_tracking.baseline_accounting import BaselineHistory, advance_baseline_history
    from litellm.proxy.spend_tracking.savings import baseline_cost_snapshot, price_baseline_comparison
    from litellm.types.utils import ModelInfo

    def upstream(request: httpx.Request) -> httpx.Response:
        body: Final = _JSON_OBJECT.validate_json(request.content)
        model: Final = body.get("model")
        assert isinstance(model, str)
        assert body.get(usage_field) == selected.get(usage_field)
        return httpx.Response(
            200,
            request=request,
            json={
                **_message(True, model),
                "usage": {
                    "input_tokens": 6000,
                    "output_tokens": 10,
                    usage_field: observed_value,
                    "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
                },
            },
        )

    rig: Final = _Rig(
        monkeypatch,
        models=_MESSAGES.validate_python(
            [
                _MODELS[0],
                {
                    **_MODELS[1],
                    "litellm_params": {
                        **_JSON_OBJECT.validate_python(_MODELS[1]["litellm_params"]),
                        **selected,
                    },
                },
                {
                    **_MODELS[2],
                    "litellm_params": {
                        **_JSON_OBJECT.validate_python(_MODELS[2]["litellm_params"]),
                        **baseline,
                    },
                },
            ]
        ),
    )
    monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False)
    with _transport(upstream):
        response: Final = _JSON_OBJECT.validate_python(
            await rig.router.anthropic_messages(
                model="test-router",
                max_tokens=16,
                messages=[{"role": "user", "content": "question"}],
                litellm_logging_obj=rig.logging(),
                litellm_call_id=rig.call_id,
                litellm_metadata={"user_api_key_hash": "test-caller-hash"},
                litellm_session_id="baseline-speed",
            )
        )
        payload: Final = await rig.capture.payload()
    captured: Final = _observation(payload)
    _, estimates = advance_baseline_history(BaselineHistory(), (captured.observation,))
    estimate: Final = estimates[0]
    assert captured.prices is not None
    prices: Final[ModelInfo] = {
        **captured.prices,
        "input_cost_per_token": 1e-6,
        "output_cost_per_token": 2e-6,
        "provider_specific_entry": {"fast": 3.0, "us": 2.0},
    }
    actual: Final = payload["response_cost"]
    assert isinstance(actual, float)
    snapshot: Final = baseline_cost_snapshot(
        captured.model,
        prices,
        actual,
        _OBJECTS.validate_python(payload["cost_breakdown"]),
        None,
    )
    comparison: Final = price_baseline_comparison(snapshot, estimate.usage, estimate.provenance)
    assert comparison is not None and snapshot.actual_token_cost is not None, estimate.reason
    assert comparison.baseline == pytest.approx(
        actual + (6000 * 1e-6 + 10 * 2e-6) * multiplier - snapshot.actual_token_cost
    )
    assert comparison.actual == actual
    assert _JSON_OBJECT.validate_python(response["usage"])[usage_field] == observed_value


async def test_native_request_rewritten_after_capture_preserves_spend_without_guessing_baseline(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class RewriteSystem(CustomLogger):
        async def async_pre_call_deployment_hook(
            self, kwargs: Mapping[str, object], call_type: CallTypes | None
        ) -> dict[str, object]:
            return {**kwargs, "system": "hook system"}

    rig: Final = _Rig(monkeypatch)
    monkeypatch.setattr(litellm, "callbacks", [rig.hook, RewriteSystem()])
    with _transport(_upstream) as route:
        await _call(rig.router, rig.logging())
        payload: Final = await rig.capture.payload()
        wire: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content)
    observed: Final = _observation(payload).observation
    assert wire["system"] == "hook system"
    assert observed.reason == "unsupported_request_transformation" and observed.plan is None
    assert observed.usage is not None
    actual: Final = payload["response_cost"]
    assert isinstance(actual, float) and actual > 0


@pytest.mark.parametrize("history", ("long_session", "non_ascii"))
async def test_native_baseline_models_long_and_non_ascii_history(monkeypatch: pytest.MonkeyPatch, history: str) -> None:
    rounds: Final = tuple(
        message
        for index in range(1200)
        for message in (
            {"role": "assistant", "content": [{"type": "tool_use", "id": f"t{index}", "name": "Read", "input": {}}]},
            {"role": "user", "content": [{"type": "tool_result", "tool_use_id": f"t{index}", "content": "ok"}]},
        )
    )
    prefix: Final = (
        [{"role": "user", "content": "start"}, *rounds]
        if history == "long_session"
        else [{"role": "user", "content": "a" * 400_000 + "é"}, {"role": "assistant", "content": "ok"}]
    )
    messages: Final = json.dumps([*prefix, *_MESSAGES.validate_json(_MESSAGES_JSON)])
    rig: Final = _Rig(monkeypatch)
    with _transport(_upstream):
        await _call(rig.router, rig.logging(), messages=messages)
        observed: Final = _observation(await rig.capture.payload()).observation
    assert observed.outcome == "complete" and observed.plan is not None


async def test_native_baseline_abstains_after_selected_tier_compaction(monkeypatch: pytest.MonkeyPatch) -> None:
    from litellm.router_strategy.complexity_router.context_compaction import compaction_executor

    monkeypatch.setitem(
        litellm.model_cost,
        "summary-fixture",
        {
            "litellm_provider": "anthropic",
            "mode": "chat",
            "max_input_tokens": 32000,
            "max_output_tokens": 4096,
            "supports_anthropic_compaction": True,
        },
    )
    models: Final = _MESSAGES.validate_python(
        [
            {
                "model_name": "test-router",
                "litellm_params": {
                    "model": "auto_router/complexity_router",
                    "complexity_router_config": {
                        "tiers": {"SIMPLE": "sonnet", "MEDIUM": "opus", "COMPLEX": "opus", "REASONING": "opus"},
                        "keyword_tier_rules": [{"keywords": ["answer"], "tier": "SIMPLE"}],
                        "session_affinity": False,
                        "enable_context_window_escalation": False,
                        "max_tokens_from_tier_model": False,
                        "context_compaction": {"model": "compactor", "max_tokens": 512},
                    },
                },
            },
            {
                "model_name": "sonnet",
                "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "test-selected"},
                "model_info": {"id": "selected", "max_input_tokens": 512, "max_output_tokens": 64},
            },
            {
                "model_name": "opus",
                "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "test-selected"},
                "model_info": {"id": "baseline", "max_input_tokens": 200000, "max_output_tokens": 4096},
            },
            {
                "model_name": "compactor",
                "litellm_params": {"model": "anthropic/summary-fixture", "api_key": "test-compactor"},
                "model_info": {"id": "compactor"},
            },
        ]
    )

    async def summarize(protocol: object, request: object, parent_model: object = None) -> Mapping[str, object]:
        return {
            "stop_reason": "compaction",
            "content": [{"type": "compaction", "content": "compacted", "signature": "s"}],
            "usage": {"input_tokens": 0, "output_tokens": 0},
        }

    messages: Final = json.dumps(
        [
            {"role": "user", "content": "Background detail. " * 300},
            {"role": "assistant", "content": "Recorded"},
            {"role": "user", "content": "Answer briefly"},
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models)
    token: Final = compaction_executor.set(summarize)
    try:
        with _transport(_upstream) as route:
            await _call(rig.router, rig.logging(), messages=messages)
            observed: Final = _observation(await rig.capture.payload()).observation
            wire: Final = route.calls.last.request.content.decode()
    finally:
        compaction_executor.reset(token)
    assert "compacted" in wire and "Background detail" not in wire
    assert observed.reason == "unsupported_request_transformation" and observed.plan is None
    assert observed.usage is not None


async def test_selected_tier_cache_markers_do_not_hide_an_unmarked_baseline_plan(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    models: Final = _MESSAGES.validate_python(
        [
            _MODELS[0],
            {
                "model_name": "sonnet",
                "litellm_params": {
                    "model": "anthropic/claude-sonnet-5",
                    "api_key": "test-selected",
                    "cache_control_injection_points": [{"location": "message", "role": "user", "index": -1}],
                },
                "model_info": {"id": "selected"},
            },
            _MODELS[2],
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models)
    with _transport(_upstream) as route:
        await _call(
            rig.router,
            rig.logging(),
            messages='[{"role":"user","content":[{"type":"text","text":"stable"},{"type":"text","text":"question"}]}]',
        )
        observed: Final = _observation(await rig.capture.payload()).observation
        wire: Final = route.calls.last.request.content.decode()
    assert "cache_control" in wire
    assert observed.outcome == "complete" and not observed.baseline_equivalent
    assert observed.reason is None and observed.plan is not None and not observed.plan.breakpoints


@pytest.mark.parametrize("recovery", ("retry", "fallback"))
async def test_tier_pins_never_enter_the_caller_snapshot_on_later_routing_passes(
    monkeypatch: pytest.MonkeyPatch, recovery: str
) -> None:
    pinned: Final = {"model_name": "first", "litellm_params": {"reasoning_effort": "high", "max_tokens": 777}}
    models: Final = _MESSAGES.validate_python(
        [
            {
                "model_name": "test-router",
                "litellm_params": {
                    "model": "auto_router/complexity_router",
                    "complexity_router_config": {
                        "tiers": {"SIMPLE": pinned, "MEDIUM": pinned, "COMPLEX": pinned, "REASONING": "opus"},
                        "session_affinity": False,
                    },
                },
            },
            {
                "model_name": "fallback-router",
                "litellm_params": {
                    "model": "auto_router/complexity_router",
                    "complexity_router_config": {
                        "tiers": {"SIMPLE": "sonnet", "MEDIUM": "sonnet", "COMPLEX": "sonnet", "REASONING": "opus"},
                        "session_affinity": False,
                    },
                },
            },
            {
                "model_name": "first",
                "litellm_params": {
                    "model": "anthropic/claude-sonnet-5" if recovery == "retry" else "anthropic/claude-haiku-5",
                    "api_key": "test-selected",
                },
                "model_info": {"id": "first"},
            },
            *_MODELS[1:],
        ]
    )
    rig: Final = _Rig(monkeypatch, models=models, retries=1 if recovery == "retry" else 0)
    rig.router.fallbacks = [{"test-router": ["fallback-router"]}]

    def upstream(request: httpx.Request) -> httpx.Response:
        return _upstream(request) if route.call_count else _error(request, 429, "first attempt")

    log: Final = rig.logging()
    with _transport(upstream) as route:
        await _call(rig.router, log)
        await rig.capture.payload()
        first_wire: Final = _JSON_OBJECT.validate_json(route.calls[0].request.content)
    assert first_wire.get("output_config") == {"effort": "high"} and first_wire.get("max_tokens") == 777
    context: Final = log.baseline_cache_context
    assert context is not None and context.baseline_body is not None
    assert context.baseline_body.get("max_tokens") == 16
    assert "output_config" not in context.baseline_body and "thinking" not in context.baseline_body