The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from typing import Final
import pytest
from pydantic import ValidationError
from litellm.proxy.telemetry.settings import EnvPolicy, TelemetrySettings, describe_errors, env_policy, load_settings
from litellm.telemetry.consent import OFF, TelemetryConsent
from litellm.telemetry.records import TelemetryGroup
_HEARTBEAT = TelemetryConsent(frozenset({TelemetryGroup.HEARTBEAT}))
_STORED = TelemetryConsent(frozenset({TelemetryGroup.HEARTBEAT, TelemetryGroup.REQUEST_SUCCESS}))
def _policy(settings: TelemetrySettings) -> EnvPolicy:
policy = env_policy(settings)
assert isinstance(policy, EnvPolicy), policy
return policy
def test_without_env_vars_the_stored_groups_apply_and_nothing_is_reported_as_set() -> None:
policy = _policy(TelemetrySettings())
assert policy.effective(_STORED) is _STORED
assert policy.effective(None) == OFF
assert policy.set_variables == ()
assert policy.locked_off is False
def test_the_veto_wins_over_both_pinned_and_stored_groups() -> None:
policy = _policy(TelemetrySettings(disabled=True, groups="heartbeat"))
assert policy.effective(_STORED) == OFF
assert policy.locked_off is True
def test_pinned_groups_override_stored_groups() -> None:
policy = _policy(TelemetrySettings(groups="Heartbeat, "))
assert policy.effective(_STORED) == _HEARTBEAT
assert policy.locked_off is False
def test_an_empty_groups_variable_pins_telemetry_off() -> None:
policy = _policy(TelemetrySettings(groups=""))
assert policy.effective(_STORED) == OFF
assert policy.locked_off is True
@pytest.mark.parametrize(
("groups", "message"),
[
("heartbeat,full", "unknown telemetry group 'full'"),
("token_info", "telemetry group 'token_info' needs 'request_success' turned on too"),
],
)
def test_invalid_pinned_groups_fail_settings_validation(
monkeypatch: pytest.MonkeyPatch, groups: str, message: str
) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_GROUPS", groups)
error = load_settings()
assert isinstance(error, ValidationError)
assert message in describe_errors(error)
@pytest.mark.parametrize("endpoint", ["not a url", "ftp://telemetry.example/v1", "telemetry.example/v1"])
def test_a_non_http_endpoint_fails_settings_validation(monkeypatch: pytest.MonkeyPatch, endpoint: str) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_ENDPOINT", endpoint)
assert isinstance(load_settings(), ValidationError)
def test_validation_errors_name_the_variable_but_never_echo_its_value(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_ENDPOINT", "ftp://[email protected]")
error = load_settings()
assert isinstance(error, ValidationError)
described = describe_errors(error)
assert "LITELLM_TELEMETRY_ENDPOINT" in described
assert "secret-token" not in described
def test_the_flush_interval_is_not_configurable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_FLUSH_INTERVAL_SECONDS", "1")
settings = load_settings()
assert isinstance(settings, TelemetrySettings)
assert "flush_interval_seconds" not in TelemetrySettings.model_fields
def test_set_variables_lists_every_telemetry_env_var_present_but_never_its_value(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_ENDPOINT", "https://[email protected]")
monkeypatch.setenv("litellm_telemetry_disabled", "1")
policy = _policy(TelemetrySettings())
assert policy.set_variables == ("LITELLM_TELEMETRY_DISABLED", "LITELLM_TELEMETRY_ENDPOINT")
assert policy.vetoed is True
def test_an_http_destination_without_an_endpoint_fails_settings_validation(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_DESTINATION", "http")
error: Final = load_settings()
assert isinstance(error, ValidationError)
assert describe_errors(error).startswith("LITELLM_TELEMETRY_DESTINATION:")
def test_a_negative_settle_timeout_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_TELEMETRY_SETTLE_TIMEOUT_SECONDS", "-1")
assert isinstance(load_settings(), ValidationError)