Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

4.1 KB
Raw Blame
from typing import Final

import pytest

from pydantic import ValidationError

from litellm.proxy.telemetry.settings import EnvPolicy, TelemetrySettings, describe_errors, env_policy, load_settings
from litellm.telemetry.consent import OFF, TelemetryConsent
from litellm.telemetry.records import TelemetryGroup

_HEARTBEAT = TelemetryConsent(frozenset({TelemetryGroup.HEARTBEAT}))
_STORED = TelemetryConsent(frozenset({TelemetryGroup.HEARTBEAT, TelemetryGroup.REQUEST_SUCCESS}))


def _policy(settings: TelemetrySettings) -> EnvPolicy:
    policy = env_policy(settings)
    assert isinstance(policy, EnvPolicy), policy
    return policy


def test_without_env_vars_the_stored_groups_apply_and_nothing_is_reported_as_set() -> None:
    policy = _policy(TelemetrySettings())
    assert policy.effective(_STORED) is _STORED
    assert policy.effective(None) == OFF
    assert policy.set_variables == ()
    assert policy.locked_off is False


def test_the_veto_wins_over_both_pinned_and_stored_groups() -> None:
    policy = _policy(TelemetrySettings(disabled=True, groups="heartbeat"))
    assert policy.effective(_STORED) == OFF
    assert policy.locked_off is True


def test_pinned_groups_override_stored_groups() -> None:
    policy = _policy(TelemetrySettings(groups="Heartbeat, "))
    assert policy.effective(_STORED) == _HEARTBEAT
    assert policy.locked_off is False


def test_an_empty_groups_variable_pins_telemetry_off() -> None:
    policy = _policy(TelemetrySettings(groups=""))
    assert policy.effective(_STORED) == OFF
    assert policy.locked_off is True


@pytest.mark.parametrize(
    ("groups", "message"),
    [
        ("heartbeat,full", "unknown telemetry group 'full'"),
        ("token_info", "telemetry group 'token_info' needs 'request_success' turned on too"),
    ],
)
def test_invalid_pinned_groups_fail_settings_validation(
    monkeypatch: pytest.MonkeyPatch, groups: str, message: str
) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_GROUPS", groups)
    error = load_settings()
    assert isinstance(error, ValidationError)
    assert message in describe_errors(error)


@pytest.mark.parametrize("endpoint", ["not a url", "ftp://telemetry.example/v1", "telemetry.example/v1"])
def test_a_non_http_endpoint_fails_settings_validation(monkeypatch: pytest.MonkeyPatch, endpoint: str) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_ENDPOINT", endpoint)
    assert isinstance(load_settings(), ValidationError)


def test_validation_errors_name_the_variable_but_never_echo_its_value(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_ENDPOINT", "ftp://[email protected]")
    error = load_settings()
    assert isinstance(error, ValidationError)
    described = describe_errors(error)
    assert "LITELLM_TELEMETRY_ENDPOINT" in described
    assert "secret-token" not in described


def test_the_flush_interval_is_not_configurable(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_FLUSH_INTERVAL_SECONDS", "1")
    settings = load_settings()
    assert isinstance(settings, TelemetrySettings)
    assert "flush_interval_seconds" not in TelemetrySettings.model_fields


def test_set_variables_lists_every_telemetry_env_var_present_but_never_its_value(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_ENDPOINT", "https://[email protected]")
    monkeypatch.setenv("litellm_telemetry_disabled", "1")
    policy = _policy(TelemetrySettings())
    assert policy.set_variables == ("LITELLM_TELEMETRY_DISABLED", "LITELLM_TELEMETRY_ENDPOINT")
    assert policy.vetoed is True


def test_an_http_destination_without_an_endpoint_fails_settings_validation(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_DESTINATION", "http")
    error: Final = load_settings()
    assert isinstance(error, ValidationError)
    assert describe_errors(error).startswith("LITELLM_TELEMETRY_DESTINATION:")


def test_a_negative_settle_timeout_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("LITELLM_TELEMETRY_SETTLE_TIMEOUT_SECONDS", "-1")
    assert isinstance(load_settings(), ValidationError)