Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

21.2 KB
Raw Blame
from collections.abc import Iterator
from typing import Final

import pytest

from litellm.proxy.common_utils.callback_config_validation import (
    callback_config_error,
    conflicting_callback_var_error,
    conflicting_span_scope_error,
    cross_entry_family_error,
    logging_metadata_config_error,
)


def test_callback_config_error_rejects_invalid_langfuse_environment():
    for callback in ["langfuse", "langfuse_otel"]:
        error = callback_config_error(callback, {"langfuse_environment": "Production"})
        assert error is not None and "langfuse_environment" in error

    assert callback_config_error("langfuse", {"langfuse_environment": "team-a-prod"}) is None
    assert callback_config_error("langfuse", {"langfuse_public_key": "pk"}) is None


def test_callback_config_error_rejects_an_unknown_langfuse_span_scope():
    for bad in ["everything", "LLM_ONLY", "llm-only", ""]:
        error = callback_config_error("langfuse_otel", {"langfuse_span_scope": bad})
        assert error is not None and "langfuse_span_scope" in error and "llm_only" in error

    assert callback_config_error("langfuse_otel", {"langfuse_span_scope": "llm_only"}) is None
    assert callback_config_error("langfuse_otel", {"langfuse_span_scope": "full"}) is None


def test_a_span_scope_on_a_callback_that_does_not_read_it_is_rejected():
    """Only langfuse_otel filters on the scope. Accepting it on the classic Langfuse
    callback or on an unrelated backend would store a setting that never takes
    effect, with the full tree still exported."""
    for callback_name in ["langfuse", "datadog", "otel", None]:
        error = callback_config_error(callback_name, {"langfuse_span_scope": "llm_only"})
        assert error is not None and "langfuse_span_scope" in error and "langfuse_otel" in error

    assert callback_config_error("langfuse", {"langfuse_environment": "team-a-prod"}) is None


def test_a_bad_span_scope_is_reported_even_when_the_environment_is_fine():
    error = callback_config_error(
        "langfuse_otel", {"langfuse_environment": "team-a-prod", "langfuse_span_scope": "everything"}
    )
    assert error is not None and "langfuse_span_scope" in error


@pytest.mark.parametrize(
    "new_vars, stored, rejected",
    [
        ({"langfuse_span_scope": "llm_only"}, [{"langfuse_span_scope": "full"}], True),
        ({"langfuse_span_scope": "full"}, [{"langfuse_public_key": "pk"}, {"langfuse_span_scope": "llm_only"}], True),
        ({"langfuse_span_scope": "llm_only"}, [{"langfuse_span_scope": "llm_only"}], False),
        ({"langfuse_span_scope": "llm_only"}, [{"langfuse_public_key": "pk"}], False),
        ({"langfuse_span_scope": "llm_only"}, [], False),
        ({"langfuse_public_key": "pk"}, [{"langfuse_span_scope": "llm_only"}], False),
        (None, [{"langfuse_span_scope": "llm_only"}], False),
    ],
)
def test_one_span_scope_per_team(new_vars, stored, rejected):
    """The entries flatten last-wins, so a second scope would export whichever entry
    was stored last. An entry that names no scope leaves the stored one in charge."""
    error = conflicting_span_scope_error(new_vars, stored)
    assert (error is not None) is rejected
    if rejected:
        assert "langfuse_span_scope" in error and stored[-1]["langfuse_span_scope"] in error


def test_key_logging_entries_may_not_disagree_on_the_span_scope():
    disagreeing = {
        "logging": [
            {
                "callback_name": "langfuse_otel",
                "callback_type": "success",
                "callback_vars": {"langfuse_span_scope": "full"},
            },
            {
                "callback_name": "langfuse_otel",
                "callback_type": "failure",
                "callback_vars": {"langfuse_span_scope": "llm_only"},
            },
        ]
    }
    error = logging_metadata_config_error(disagreeing)
    assert error is not None and "langfuse_span_scope" in error and "'full'" in error

    agreeing = {
        "logging": [
            {
                "callback_name": "langfuse_otel",
                "callback_type": "success",
                "callback_vars": {"langfuse_span_scope": "llm_only"},
            },
            {
                "callback_name": "langfuse_otel",
                "callback_type": "failure",
                "callback_vars": {"langfuse_span_scope": "llm_only"},
            },
            {"callback_name": "otel", "callback_type": "success", "callback_vars": {}},
        ]
    }
    assert logging_metadata_config_error(agreeing) is None


@pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"])
@pytest.mark.parametrize("bad", ["1.5", "-0.1", "abc", "nan", "inf"])
def test_callback_config_error_rejects_out_of_range_arize_sampling_rate(var, bad):
    error = callback_config_error("arize", {var: bad})
    assert error is not None and var in error and repr(bad) in error


@pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"])
@pytest.mark.parametrize("good", ["0", "1", "0.25", "", "None"])
def test_callback_config_error_accepts_in_range_arize_sampling_rate(var, good):
    assert callback_config_error("arize", {var: good}) is None


def test_arize_sampling_rate_rejected_on_non_arize_callback():
    error = callback_config_error("langfuse", {"arize_success_sampling_rate": "0.5"})
    assert error is not None
    assert "applies to the arize callback only" in error
    assert callback_config_error("arize", {"arize_success_sampling_rate": "0.5"}) is None


class TestArizeOtlpProtocol:
    def test_protocol_on_a_non_arize_callback_is_rejected(self, monkeypatch):
        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            for callback_name in ["langfuse", "langfuse_otel", "datadog", None]:
                error = callback_config_error(callback_name, {"arize_otlp_protocol": "grpc"})
                assert error is not None and "applies to the arize callback only" in error
        finally:
            is_otel_v2_enabled.cache_clear()

    def test_unknown_protocols_are_rejected(self, monkeypatch):
        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            for bad in ["otlp_http", "HTTP/PROTOBUF", "http_json", "", "None"]:
                error = callback_config_error("arize", {"arize_otlp_protocol": bad})
                assert error is not None and "arize_otlp_protocol" in error and "http/protobuf" in error
        finally:
            is_otel_v2_enabled.cache_clear()

    def test_protocol_is_rejected_while_otel_v2_is_off(self, monkeypatch):
        monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"})
            assert error is not None and "LITELLM_OTEL_V2" in error
        finally:
            is_otel_v2_enabled.cache_clear()

    def test_both_protocols_are_accepted_with_otel_v2_on(self, monkeypatch):
        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            assert callback_config_error("arize", {"arize_otlp_protocol": "grpc"}) is None
            assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None
        finally:
            is_otel_v2_enabled.cache_clear()

    def test_protocol_is_not_a_family_credential(self):
        stored = [{"arize_api_key": "k1"}]
        assert cross_entry_family_error({"arize_otlp_protocol": "grpc"}, stored) is None
        assert cross_entry_family_error({"arize_otlp_protocol": "http/protobuf"}, stored) is None

    def test_protocol_on_a_key_logging_entry_is_validated(self, monkeypatch):
        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            invalid_metadata = {
                "logging": [
                    {
                        "callback_name": "arize",
                        "callback_type": "success",
                        "callback_vars": {"arize_otlp_protocol": "carrier-pigeon"},
                    }
                ]
            }
            error = logging_metadata_config_error(invalid_metadata)
            assert error is not None and "arize_otlp_protocol" in error

            valid_metadata = {
                "logging": [
                    {
                        "callback_name": "arize",
                        "callback_type": "success",
                        "callback_vars": {"arize_otlp_protocol": "http/protobuf"},
                    }
                ]
            }
            assert logging_metadata_config_error(valid_metadata) is None
        finally:
            is_otel_v2_enabled.cache_clear()

    def test_failure_only_arize_callbacks_reject_the_protocol(self, monkeypatch):
        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "failure")
            assert error is not None and "arize_otlp_protocol" in error and "failure" in error
            assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success") is None
            assert (
                callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success_and_failure") is None
            )
            assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None
        finally:
            is_otel_v2_enabled.cache_clear()

    def test_failure_only_key_logging_entry_rejects_the_protocol(self, monkeypatch):
        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
        from litellm.integrations.otel.model.config import is_otel_v2_enabled

        is_otel_v2_enabled.cache_clear()
        try:
            failure_metadata = {
                "logging": [
                    {
                        "callback_name": "arize",
                        "callback_type": "failure",
                        "callback_vars": {"arize_otlp_protocol": "http/protobuf"},
                    }
                ]
            }
            error = logging_metadata_config_error(failure_metadata)
            assert error is not None and "arize_otlp_protocol" in error

            success_metadata = {
                "logging": [
                    {
                        "callback_name": "arize",
                        "callback_type": "success",
                        "callback_vars": {"arize_otlp_protocol": "http/protobuf"},
                    }
                ]
            }
            assert logging_metadata_config_error(success_metadata) is None
        finally:
            is_otel_v2_enabled.cache_clear()


def test_arize_sampling_rates_are_not_family_credentials():
    """The rates choose what the Arize family exports, not where it sends, so an
    entry that repeats or adds a rate next to a stored Arize entry is not the
    credential-redirect shape cross_entry_family_error rejects."""
    stored = [{"arize_api_key": "k1", "arize_success_sampling_rate": "0.5"}]
    assert cross_entry_family_error({"arize_success_sampling_rate": "0.1"}, stored) is None
    assert cross_entry_family_error({"arize_error_sampling_rate": "0.5"}, stored) is None


@pytest.fixture
def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
    from litellm.integrations.otel.model.config import is_otel_v2_enabled

    monkeypatch.setenv("LITELLM_OTEL_V2", "true")
    is_otel_v2_enabled.cache_clear()
    yield
    is_otel_v2_enabled.cache_clear()


@pytest.mark.usefixtures("otel_v2_on")
@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"])
@pytest.mark.parametrize("value", ["no_content", "span_only"])
def test_capture_message_content_is_accepted_on_every_otel_v2_destination(callback_name: str, value: str) -> None:
    assert callback_config_error(callback_name, {"capture_message_content": value}) is None


@pytest.mark.usefixtures("otel_v2_on")
@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"])
@pytest.mark.parametrize("value", ["redis,postgres", "none", " REDIS , postgresql "])
def test_excluded_services_is_accepted_on_every_otel_v2_destination(callback_name: str, value: str) -> None:
    assert callback_config_error(callback_name, {"excluded_services": value}) is None


def test_capture_message_content_is_rejected_while_otel_v2_is_off(monkeypatch: pytest.MonkeyPatch) -> None:
    from litellm.integrations.otel.model.config import is_otel_v2_enabled

    monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
    is_otel_v2_enabled.cache_clear()
    try:
        error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"})
    finally:
        is_otel_v2_enabled.cache_clear()
    assert error is not None and "capture_message_content" in error and "LITELLM_OTEL_V2" in error


def test_excluded_services_is_rejected_while_otel_v2_is_off(monkeypatch: pytest.MonkeyPatch) -> None:
    from litellm.integrations.otel.model.config import is_otel_v2_enabled

    monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
    is_otel_v2_enabled.cache_clear()
    try:
        error: Final = callback_config_error("langfuse_otel", {"excluded_services": "redis"})
    finally:
        is_otel_v2_enabled.cache_clear()
    assert error is not None and "excluded_services" in error and "LITELLM_OTEL_V2" in error


@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None])
def test_capture_message_content_is_rejected_where_it_would_never_take_effect(callback_name: str | None) -> None:
    error: Final = callback_config_error(callback_name, {"capture_message_content": "no_content"})
    assert error is not None and "capture_message_content" in error and "langfuse_otel" in error


@pytest.mark.usefixtures("otel_v2_on")
@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None])
def test_excluded_services_is_rejected_where_it_would_never_take_effect(callback_name: str | None) -> None:
    error: Final = callback_config_error(callback_name, {"excluded_services": "redis"})
    assert error is not None and "excluded_services" in error and "langfuse_otel" in error


@pytest.mark.usefixtures("otel_v2_on")
@pytest.mark.parametrize("value", ["full", "event_only", "span_and_event"])
def test_an_unsupported_capture_message_content_is_rejected_on_key_logging_metadata(value: str) -> None:
    error: Final = logging_metadata_config_error(
        {
            "logging": [
                {
                    "callback_name": "langfuse_otel",
                    "callback_type": "success",
                    "callback_vars": {"capture_message_content": value},
                }
            ]
        }
    )
    assert error is not None and "Invalid capture_message_content" in error


@pytest.mark.usefixtures("otel_v2_on")
@pytest.mark.parametrize("callback_var", ["capture_message_content", "excluded_services"])
def test_key_logging_metadata_rejects_non_string_destination_callback_vars(callback_var: str) -> None:
    error: Final = logging_metadata_config_error(
        {
            "logging": [
                {
                    "callback_name": "newrelic",
                    "callback_type": "success_and_failure",
                    "callback_vars": {callback_var: None},
                }
            ]
        }
    )
    assert error == f"{callback_var} must be a string, got NoneType"


@pytest.mark.usefixtures("otel_v2_on")
def test_key_logging_metadata_with_non_string_callback_var_key_does_not_raise() -> None:
    error: Final = logging_metadata_config_error(
        {
            "logging": [
                {
                    "callback_name": "langfuse_otel",
                    "callback_vars": {1: "x", "excluded_services": "redis"},
                }
            ]
        }
    )
    assert error is None


@pytest.mark.usefixtures("otel_v2_on")
def test_each_entry_keeps_its_own_capture_message_content() -> None:
    error: Final = logging_metadata_config_error(
        {
            "logging": [
                {
                    "callback_name": "langfuse_otel",
                    "callback_type": "success",
                    "callback_vars": {"capture_message_content": "span_only"},
                },
                {
                    "callback_name": "arize",
                    "callback_type": "success",
                    "callback_vars": {"capture_message_content": "no_content"},
                },
            ]
        }
    )
    assert error is None


@pytest.mark.usefixtures("otel_v2_on")
def test_a_failure_only_entry_rejects_capture_message_content() -> None:
    error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}, "failure")
    assert error is not None and "capture_message_content" in error and "success_and_failure" in error


@pytest.mark.parametrize(
    ("stored", "rejected"),
    [
        ([("langfuse_otel", {"capture_message_content": "no_content"})], True),
        ([("langfuse_otel", {"capture_message_content": "span_only"})], False),
        ([("langfuse_otel", {"langfuse_public_key": "pk"})], False),
        ([("newrelic", {"capture_message_content": "no_content"})], False),
    ],
)
def test_entries_for_one_backend_share_one_capture_message_content(
    stored: list[tuple[str, dict[str, str]]], rejected: bool
) -> None:
    """The backend's entries merge into one destination, so a second value would silently win or lose."""
    error: Final = conflicting_callback_var_error(
        "langfuse_otel", "capture_message_content", {"capture_message_content": "span_only"}, stored
    )
    assert (error is not None) is rejected


@pytest.mark.parametrize(
    ("stored", "rejected"),
    [
        ([("langfuse_otel", {"excluded_services": "none"})], True),
        ([("langfuse_otel", {"excluded_services": "redis"})], False),
        ([("langfuse_otel", {"langfuse_public_key": "pk"})], False),
        ([("newrelic", {"excluded_services": "none"})], False),
    ],
)
def test_entries_for_one_backend_share_one_excluded_services_value(
    stored: list[tuple[str, dict[str, str]]], rejected: bool
) -> None:
    error: Final = conflicting_callback_var_error(
        "langfuse_otel", "excluded_services", {"excluded_services": "redis"}, stored
    )
    assert (error is not None) is rejected


@pytest.mark.usefixtures("otel_v2_on")
def test_key_logging_entries_for_one_backend_may_not_disagree_on_capture_message_content() -> None:
    def entry(callback_type: str, capture: str) -> dict[str, str | dict[str, str]]:
        return {
            "callback_name": "langfuse_otel",
            "callback_type": callback_type,
            "callback_vars": {"capture_message_content": capture},
        }

    error: Final = logging_metadata_config_error(
        {"logging": [entry("success", "no_content"), entry("success_and_failure", "span_only")]}
    )
    assert error is not None and "already set to 'no_content' by another langfuse_otel entry" in error


@pytest.mark.usefixtures("otel_v2_on")
def test_key_logging_entries_for_one_backend_may_not_disagree_on_excluded_services() -> None:
    def entry(callback_vars: dict[str, str]) -> dict[str, str | dict[str, str]]:
        return {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": callback_vars}

    error: Final = logging_metadata_config_error(
        {"logging": [entry({"excluded_services": "redis"}), entry({"excluded_services": "none"})]}
    )
    assert error is not None and "already set to 'redis' by another langfuse_otel entry" in error


@pytest.mark.usefixtures("otel_v2_on")
@pytest.mark.parametrize(
    ("stored", "incoming", "conflicts"),
    [
        ("redis,postgres", "postgres,redis", False),
        ("redis", "REDIS", False),
        ("postgres", "postgresql", False),
        ("none", "NONE", False),
        ("redis", "postgres", True),
        ("none", "redis", True),
    ],
)
def test_key_logging_entries_compare_excluded_services_by_parsed_value(
    stored: str, incoming: str, conflicts: bool
) -> None:
    def entry(callback_type: str, excluded_services: str) -> dict[str, str | dict[str, str]]:
        return {
            "callback_name": "newrelic",
            "callback_type": callback_type,
            "callback_vars": {"newrelic_api_key": "license-key", "excluded_services": excluded_services},
        }

    error: Final = logging_metadata_config_error(
        {"logging": [entry("success", stored), entry("success_and_failure", incoming)]}
    )
    assert (error is not None) is conflicts
    if conflicts:
        assert error is not None and f"already set to {stored!r}" in error


@pytest.mark.usefixtures("otel_v2_on")
def test_a_failure_only_entry_rejects_excluded_services() -> None:
    error: Final = callback_config_error("langfuse_otel", {"excluded_services": "redis"}, "failure")
    assert error is not None and "excluded_services" in error and "success_and_failure" in error