Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

17.3 KB
Raw Blame
import asyncio
import datetime
from collections.abc import Mapping
from copy import deepcopy
from typing import Final, cast

import pytest
from prometheus_client import REGISTRY
from prometheus_client.samples import Sample

import litellm
from litellm.integrations.prometheus import PrometheusLogger
from litellm.types.integrations.prometheus import (
    PrometheusMetricLabels,
    UserAPIKeyLabelNames,
    UserAPIKeyLabelValues,
    get_input_sequence_length_bucket,
)
from litellm.types.utils import StandardLoggingPayload

LATENCY_METRICS: Final = (
    "litellm_llm_api_latency_metric",
    "litellm_llm_api_time_to_first_token_metric",
    "litellm_request_total_latency_metric",
)
FLAG: Final = "prometheus_emit_input_sequence_length_label"


def _clear_prometheus_registry() -> None:
    for collector in tuple(REGISTRY._collector_to_names):  # pyright: ignore[reportPrivateUsage]  # test registry reset
        REGISTRY.unregister(collector)


@pytest.fixture(autouse=True)
def isolated_registry(monkeypatch: pytest.MonkeyPatch):
    _clear_prometheus_registry()
    monkeypatch.setattr(litellm, FLAG, False)
    yield
    _clear_prometheus_registry()


@pytest.mark.parametrize("metric", LATENCY_METRICS)
def test_input_sequence_length_label_is_opt_in(monkeypatch: pytest.MonkeyPatch, metric: str):
    assert UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value not in PrometheusMetricLabels.get_labels(metric)

    monkeypatch.setattr(litellm, FLAG, True)
    assert UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value in PrometheusMetricLabels.get_labels(metric)


def test_input_sequence_length_label_stays_off_non_latency_metrics(monkeypatch: pytest.MonkeyPatch):
    monkeypatch.setattr(litellm, FLAG, True)
    assert UserAPIKeyLabelNames.INPUT_SEQUENCE_LENGTH.value not in PrometheusMetricLabels.get_labels(
        "litellm_proxy_total_requests_metric"
    )


@pytest.mark.parametrize(
    "prompt_tokens, expected",
    [
        (None, "unknown"),
        (0, "0-1k"),
        (999, "0-1k"),
        (1_000, "1k-4k"),
        (3_999, "1k-4k"),
        (4_000, "4k-16k"),
        (15_999, "4k-16k"),
        (16_000, "16k-64k"),
        (63_999, "16k-64k"),
        (64_000, "64k+"),
        (10_000_000, "64k+"),
        (-1, "unknown"),
    ],
)
def test_input_sequence_length_bucket_boundaries(prompt_tokens: int | None, expected: str):
    assert get_input_sequence_length_bucket(prompt_tokens) == expected


def test_user_api_key_label_values_carries_input_sequence_length():
    values: Final = UserAPIKeyLabelValues(input_sequence_length="4k-16k")

    assert values.input_sequence_length == "4k-16k"
    assert values.model_dump()["input_sequence_length"] == "4k-16k"


def _assert_latency_metrics(expected: str | None, stream: bool = True, queue_time: float = 0) -> None:
    samples: Final = tuple(sample for metric in REGISTRY.collect() for sample in metric.samples)
    for metric, duration in zip(LATENCY_METRICS, (2, 1, 3 + queue_time)):
        counts: Final = tuple(sample for sample in samples if sample.name == f"{metric}_count")
        sums: Final = tuple(sample for sample in samples if sample.name == f"{metric}_sum")
        buckets: Final = tuple(sample for sample in samples if sample.name == f"{metric}_bucket")
        if not stream and metric == "litellm_llm_api_time_to_first_token_metric":
            assert not counts and not sums and not buckets
            continue
        assert len(counts) == len(sums) == 1
        assert counts[0].value == 1
        assert sums[0].value == pytest.approx(duration)
        assert buckets and any(sample.labels["le"] == "+Inf" for sample in buckets)
        assert all(sample.value == int(float(sample.labels["le"]) >= duration) for sample in buckets)
        assert all(sample.labels.get("input_sequence_length") == expected for sample in (*counts, *sums, *buckets))


def _non_target_samples() -> tuple[Sample, ...]:
    return tuple(
        sample
        for metric in REGISTRY.collect()
        if metric.name not in LATENCY_METRICS
        for sample in metric.samples
        if "input_sequence_length" in sample.labels and not sample.name.endswith("_created")
    )


def _standard_logging_payload(now: datetime.datetime, prompt_tokens: int) -> StandardLoggingPayload:
    return cast(
        StandardLoggingPayload,
        {
            "id": "t",
            "call_type": "completion",
            "response_cost": 0.001,
            "status": "success",
            "total_tokens": prompt_tokens + 20,
            "prompt_tokens": prompt_tokens,
            "completion_tokens": 20,
            "startTime": now - datetime.timedelta(seconds=3),
            "endTime": now,
            "completionStartTime": now - datetime.timedelta(seconds=1),
            "model": "gpt-4o-mini",
            "model_id": "model-123",
            "model_group": "gpt-4o-mini",
            "api_base": "https://api.openai.com",
            "custom_llm_provider": "openai",
            "request_tags": [],
            "stream": True,
            "metadata": {
                "user_api_key_hash": "h",
                "user_api_key_alias": "a",
                "user_api_key_team_id": "t",
                "user_api_key_team_alias": "ta",
                "user_api_key_user_id": "u",
                "user_api_key_user_email": "[email protected]",
                "user_api_key_org_id": None,
                "user_api_key_org_alias": None,
                "requester_metadata": None,
                "user_api_key_end_user_id": None,
                "usage_object": None,
            },
            "hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None},
        },
    )


def _success_kwargs(
    now: datetime.datetime, prompt_tokens: int, requester_metadata: Mapping[str, object] | None = None
) -> Mapping[str, object]:
    payload: Final = _standard_logging_payload(now, prompt_tokens)
    return {
        "model": "gpt-4o-mini",
        "litellm_params": {"metadata": {}},
        "standard_logging_object": {
            **payload,
            "metadata": {**payload["metadata"], "requester_metadata": requester_metadata},
        },
        "stream": True,
        "start_time": now - datetime.timedelta(seconds=3),
        "api_call_start_time": now - datetime.timedelta(seconds=2),
        "completion_start_time": now - datetime.timedelta(seconds=1),
        "end_time": now,
    }


@pytest.mark.asyncio
@pytest.mark.parametrize("flag_at_request_time", (True, False))
async def test_logger_emits_bucket_from_its_startup_label_set(
    monkeypatch: pytest.MonkeyPatch, flag_at_request_time: bool
):
    now: Final = datetime.datetime.now()
    monkeypatch.setattr(litellm, FLAG, True)
    logger: Final = PrometheusLogger()
    monkeypatch.setattr(litellm, FLAG, flag_at_request_time)

    await logger.async_log_success_event(dict(_success_kwargs(now, prompt_tokens=4_000)), None, now, now)

    _assert_latency_metrics("4k-16k")


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("response", "combined_usage", "expected"),
    (
        ({"id": "moderation", "results": []}, None, "unknown"),
        ({"usage": None}, None, "unknown"),
        ({"usage": {}}, None, "unknown"),
        ({"usage": {"completion_tokens": 3}}, None, "unknown"),
        ({"usage": {"total_tokens": 5}}, None, "unknown"),
        ({"usage": {"prompt_tokens": 0}}, None, "0-1k"),
        ({"usage": {"prompt_tokens": 4_000}}, None, "4k-16k"),
        ({"usage": {"input_tokens": 0, "output_tokens": 3, "total_tokens": 3}}, None, "0-1k"),
        ({"usage": {"input_tokens": 4_000, "output_tokens": 3, "total_tokens": 4_003}}, None, "4k-16k"),
        (litellm.ModelResponse(usage=litellm.Usage(prompt_tokens=0)), None, "0-1k"),
        (None, litellm.Usage(prompt_tokens=0), "0-1k"),
    ),
)
@pytest.mark.parametrize("include_usage_metadata", (True, False))
async def test_logger_distinguishes_missing_usage_from_reported_zero(
    monkeypatch: pytest.MonkeyPatch,
    response: object,
    combined_usage: object,
    expected: str,
    include_usage_metadata: bool,
):
    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup

    now: Final = datetime.datetime.now()
    monkeypatch.setattr(litellm, FLAG, True)
    logger: Final = PrometheusLogger()
    response_obj: Final = response.model_dump() if isinstance(response, litellm.ModelResponse) else response
    usage: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
        response_obj=response_obj if isinstance(response_obj, dict) else None,
        combined_usage_object=combined_usage if isinstance(combined_usage, litellm.Usage) else None,
    )

    payload: Final = _standard_logging_payload(now, usage.get("prompt_tokens", 0))
    await logger.async_log_success_event(
        {
            **_success_kwargs(now, prompt_tokens=usage.get("prompt_tokens", 0)),
            "combined_usage_object": combined_usage,
            "standard_logging_object": {
                **payload,
                "metadata": {**payload["metadata"], "usage_object": usage if include_usage_metadata else None},
            },
        },
        response,
        now,
        now,
    )

    _assert_latency_metrics(expected)


@pytest.mark.asyncio
@pytest.mark.parametrize("total_tokens", (None, 0, 5_000))
@pytest.mark.parametrize("prompt_tokens", (0, 4_000))
async def test_upstream_total_only_usage_has_unknown_input_length(
    monkeypatch: pytest.MonkeyPatch, total_tokens: int | None, prompt_tokens: int
):
    import httpx

    from litellm.litellm_core_utils.litellm_logging import Logging, StandardLoggingPayloadSetup
    from litellm.proxy.pass_through_endpoints.upstream_usage_headers import apply_upstream_reported_usage

    now: Final = datetime.datetime.now()
    monkeypatch.setattr(litellm, FLAG, True)
    logger: Final = PrometheusLogger()
    logging_obj: Final = Logging(
        model="gpt-4o-mini",
        messages=[],
        stream=True,
        call_type="pass_through_endpoint",
        start_time=now,
        litellm_call_id="test-call-id",
        function_id="1",
    )
    headers: Final = httpx.Headers(
        {
            "x-litellm-response-cost": "0.001",
            **({"x-litellm-total-tokens": str(total_tokens)} if total_tokens is not None else {}),
        }
    )
    reported: Final = apply_upstream_reported_usage(logging_obj=logging_obj, headers=headers)
    assert reported is not None
    combined_usage: Final = logging_obj.model_call_details.get("combined_usage_object")
    response: Final = {"usage": {"prompt_tokens": prompt_tokens}}
    usage: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response, combined_usage)
    payload: Final = _standard_logging_payload(now, usage.get("prompt_tokens", 0))

    await logger.async_log_success_event(
        {
            **logging_obj.model_call_details,
            **_success_kwargs(now, usage.get("prompt_tokens", 0)),
            "standard_logging_object": {**payload, "metadata": {**payload["metadata"], "usage_object": usage}},
        },
        response,
        now,
        now,
    )

    _assert_latency_metrics("unknown" if total_tokens is not None else get_input_sequence_length_bucket(prompt_tokens))


@pytest.mark.asyncio
async def test_logger_built_with_flag_off_emits_no_bucket_label(monkeypatch: pytest.MonkeyPatch):
    now: Final = datetime.datetime.now()
    logger: Final = PrometheusLogger()
    monkeypatch.setattr(litellm, FLAG, True)

    await logger.async_log_success_event(dict(_success_kwargs(now, prompt_tokens=4_000)), None, now, now)

    _assert_latency_metrics(None)


@pytest.mark.asyncio
@pytest.mark.parametrize("flag_at_startup", (True, False))
@pytest.mark.parametrize("stream", (True, False))
@pytest.mark.parametrize(
    "metadata",
    (
        None,
        {},
        {"input_sequence_length": None},
        {"input_sequence_length": False},
        {"input_sequence_length": True},
        {"input_sequence_length": 0},
        {"input_sequence_length": []},
        {"input_sequence_length": {}},
        {"input_sequence_length": ""},
        {"input_sequence_length": "from-metadata"},
    ),
)
async def test_custom_input_length_label_is_scoped_to_target_histograms(
    monkeypatch: pytest.MonkeyPatch, flag_at_startup: bool, stream: bool, metadata: Mapping[str, object] | None
):
    now: Final = datetime.datetime.now()
    monkeypatch.setattr(litellm, "custom_prometheus_metadata_labels", ["input_sequence_length"])
    kwargs: Final = {
        **_success_kwargs(now, prompt_tokens=4_000, requester_metadata=metadata),
        "stream": stream,
        "litellm_params": {"metadata": {"queue_time_seconds": 0.25}},
    }
    original_kwargs: Final = deepcopy(kwargs)
    baseline_logger: Final = PrometheusLogger()
    await baseline_logger.async_log_success_event(kwargs, None, now, now)
    baseline_samples: Final = _non_target_samples()
    _clear_prometheus_registry()
    monkeypatch.setattr(litellm, FLAG, flag_at_startup)
    logger: Final = PrometheusLogger()
    monkeypatch.setattr(litellm, FLAG, not flag_at_startup)

    await logger.async_log_success_event(kwargs, None, now, now)

    assert kwargs == original_kwargs
    custom_value: Final = (metadata or {}).get("input_sequence_length")
    expected: Final = custom_value if isinstance(custom_value, str) else ("4k-16k" if flag_at_startup else "None")
    _assert_latency_metrics(expected, stream=stream, queue_time=0.25)
    assert all(logger.get_labels_for_metric(metric).count("input_sequence_length") == 1 for metric in LATENCY_METRICS)
    non_target_samples: Final = _non_target_samples()
    assert {
        "litellm_requests_metric_total",
        "litellm_spend_metric_total",
        "litellm_total_tokens_metric_total",
        "litellm_request_queue_time_seconds_count",
        "litellm_deployment_success_responses_total",
    }.issubset({sample.name for sample in non_target_samples})
    assert non_target_samples == baseline_samples
    queue_sum: Final = tuple(
        sample for sample in non_target_samples if sample.name == "litellm_request_queue_time_seconds_sum"
    )
    assert len(queue_sum) == 1 and queue_sum[0].value == 0.25


@pytest.mark.asyncio
@pytest.mark.parametrize("enabled", (True, False))
async def test_concurrent_requests_keep_independent_buckets(monkeypatch: pytest.MonkeyPatch, enabled: bool):
    now: Final = datetime.datetime.now()
    monkeypatch.setattr(litellm, FLAG, enabled)
    logger: Final = PrometheusLogger()
    cases: Final = (
        (None, "unknown"),
        (0, "0-1k"),
        (1_000, "1k-4k"),
        (4_000, "4k-16k"),
        (16_000, "16k-64k"),
        (64_000, "64k+"),
    )
    calls: Final = tuple(
        (
            {**_success_kwargs(now, prompt_tokens=tokens or 0), "stream": stream},
            {"usage": {"prompt_tokens": tokens}} if tokens is not None else None,
        )
        for tokens, _ in cases
        for stream in (True, False)
        for _ in range(2)
    )
    original_calls: Final = deepcopy(calls)

    await asyncio.gather(*(logger.async_log_success_event(kwargs, response, now, now) for kwargs, response in calls))

    assert calls == original_calls
    samples: Final = tuple(sample for metric in REGISTRY.collect() for sample in metric.samples)
    for metric, duration in zip(LATENCY_METRICS, (2, 1, 3)):
        expected_count: Final = 2 if metric == "litellm_llm_api_time_to_first_token_metric" else 4
        counts: Final = tuple(sample for sample in samples if sample.name == f"{metric}_count")
        sums: Final = tuple(sample for sample in samples if sample.name == f"{metric}_sum")
        buckets: Final = tuple(sample for sample in samples if sample.name == f"{metric}_bucket")
        expected: Final = (
            {bucket: expected_count for _, bucket in cases} if enabled else {None: expected_count * len(cases)}
        )
        assert len(counts) == len(sums) == len(expected)
        assert {sample.labels.get("input_sequence_length"): sample.value for sample in counts} == expected
        assert {sample.labels.get("input_sequence_length"): sample.value for sample in sums} == {
            bucket: count * duration for bucket, count in expected.items()
        }
        assert sum(sample.value for sample in buckets if sample.labels["le"] == "+Inf") == expected_count * len(cases)
        assert all(
            sample.value
            == expected[sample.labels.get("input_sequence_length")] * int(float(sample.labels["le"]) >= duration)
            for sample in buckets
        )


@pytest.mark.asyncio
@pytest.mark.parametrize("enabled", (True, False))
async def test_failed_request_does_not_observe_latency(monkeypatch: pytest.MonkeyPatch, enabled: bool):
    now: Final = datetime.datetime.now()
    monkeypatch.setattr(litellm, FLAG, enabled)
    monkeypatch.setattr(litellm, "custom_prometheus_metadata_labels", ["input_sequence_length"])
    logger: Final = PrometheusLogger()
    kwargs: Final = {
        **_success_kwargs(now, prompt_tokens=4_000),
        "standard_logging_object": {**_standard_logging_payload(now, 4_000), "status": "failure"},
        "exception": RuntimeError("upstream request failed"),
    }

    await logger.async_log_failure_event(kwargs, None, now, now)

    samples: Final = tuple(sample for metric in REGISTRY.collect() for sample in metric.samples)
    assert not any(sample.name.startswith(LATENCY_METRICS) for sample in samples)
    for metric in ("litellm_llm_api_failed_requests_metric_total", "litellm_deployment_failure_responses_total"):
        counts: Final = tuple(sample for sample in samples if sample.name == metric)
        assert len(counts) == 1
        assert counts[0].value == 1
        assert counts[0].labels["input_sequence_length"] == "None"