Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

3.5 KB
Raw Blame
from typing import Final

import pytest

from litellm.telemetry.consent import OFF, REQUIRES, TelemetryConsent
from litellm.telemetry.records import TelemetryGroup
from litellm.telemetry.report import report_to_json
from litellm.telemetry.sample import sample_report

_REQUEST_FIELDS: Final = {
    TelemetryGroup.REQUEST_SUCCESS: {
        "endpoint",
        "litellm_status",
        "handled_by_rust",
        "latency_to_first_byte_ms",
        "request_count",
    },
    TelemetryGroup.TOKEN_INFO: {"provider_cache_hit", "input_tokens", "cache_read_tokens"},
    TelemetryGroup.REQUEST_TAXONOMY: {"provider", "deployment_hash"},
    TelemetryGroup.EVENT_DETAILS: {"block_count", "block_types", "header_keys"},
}


def _with_requirements(group: TelemetryGroup) -> frozenset[TelemetryGroup]:
    required: Final = REQUIRES[group]
    parents: Final[frozenset[TelemetryGroup]] = _with_requirements(required) if required is not None else frozenset()
    return frozenset({group}) | parents


@pytest.mark.asyncio
async def test_off_builds_no_report() -> None:
    assert await sample_report(OFF, litellm_version="1.0.0") is None


@pytest.mark.asyncio
@pytest.mark.parametrize("group", list(_REQUEST_FIELDS))
async def test_each_request_group_adds_exactly_its_own_fields(group: TelemetryGroup) -> None:
    groups: Final = _with_requirements(group)
    with_group: Final = await sample_report(TelemetryConsent(groups), litellm_version="1.0.0")
    without_group: Final = await sample_report(TelemetryConsent(groups - {group}), litellm_version="1.0.0")
    assert with_group is not None and without_group is not None
    rows_with: Final = report_to_json(with_group)["requests"]
    assert isinstance(rows_with, list) and isinstance(rows_with[0], dict)
    rows_without: Final = report_to_json(without_group).get("requests", [{}])
    assert isinstance(rows_without, list) and isinstance(rows_without[0], dict)
    assert _REQUEST_FIELDS[group] <= set(rows_with[0]) - set(rows_without[0])


@pytest.mark.asyncio
async def test_heartbeat_only_sample_is_just_the_instance_header() -> None:
    report: Final = await sample_report(TelemetryConsent(frozenset({TelemetryGroup.HEARTBEAT})), litellm_version="2.0")
    assert report is not None
    assert report_to_json(report) == {
        "schema_version": 1,
        "instance": {"instance_id": "0" * 32, "litellm_version": "2.0", "groups": ["heartbeat"]},
        "window_start": 0.0,
        "window_end": 300.0,
        "dropped_records": 0,
    }


@pytest.mark.asyncio
async def test_without_taxonomy_rows_from_different_providers_with_the_same_status_merge() -> None:
    consent: Final = TelemetryConsent(_with_requirements(TelemetryGroup.TOKEN_INFO))
    report: Final = await sample_report(consent, litellm_version="1.0.0")
    assert report is not None
    assert len(report.requests) == 2
    assert all(key.provider is None for key, _ in report.requests)
    assert report.attempts == ()


@pytest.mark.asyncio
async def test_instance_configuration_adds_config_keys_and_page_navigation_adds_ui_events() -> None:
    consent: Final = TelemetryConsent(
        frozenset({TelemetryGroup.HEARTBEAT, TelemetryGroup.INSTANCE_CONFIGURATION, TelemetryGroup.PAGE_NAVIGATION})
    )
    report: Final = await sample_report(consent, litellm_version="1.0.0")
    assert report is not None
    json_report: Final = report_to_json(report)
    instance: Final = json_report["instance"]
    assert isinstance(instance, dict) and instance["config_keys"]
    assert json_report["ui_events"]
    assert "requests" not in json_report