Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

15.9 KB
Raw Blame
import json
import os
import unittest
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock, Mock, patch

import httpx
import litellm
import pytest
import respx
from litellm.litellm_core_utils import litellm_logging
from litellm.integrations.braintrust_logging import BraintrustLogger


class TestBraintrustLogger(unittest.TestCase):
    @patch.dict(os.environ, {"BRAINTRUST_API_KEY": "test-env-api-key"})
    @patch.dict(os.environ, {"BRAINTRUST_API_BASE": "https://test-env-api.com/v1"})
    def test_init_with_env_var(self):
        """Test BraintrustLogger initialization with environment variable."""
        logger = BraintrustLogger()
        self.assertEqual(logger.api_key, "test-env-api-key")
        self.assertEqual(logger.api_base, "https://test-env-api.com/v1")
        self.assertEqual(logger.headers["Authorization"], "Bearer test-env-api-key")
        self.assertEqual(logger.headers["Content-Type"], "application/json")

    def test_init_with_explicit_params(self):
        """Test BraintrustLogger initialization with explicit parameters."""
        logger = BraintrustLogger(
            api_key="explicit-key", api_base="https://custom-api.com/v1"
        )
        self.assertEqual(logger.api_key, "explicit-key")
        self.assertEqual(logger.api_base, "https://custom-api.com/v1")
        self.assertEqual(logger.headers["Authorization"], "Bearer explicit-key")

    @patch.dict(os.environ, {}, clear=True)
    def test_init_missing_api_key(self):
        """Test BraintrustLogger initialization fails without API key."""
        with self.assertRaises(Exception) as context:
            BraintrustLogger()
        self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception))

    def test_validate_environment_with_api_key(self):
        """Test validate_environment method with valid API key."""
        logger = BraintrustLogger(api_key="test-key")
        # Should not raise an exception
        logger.validate_environment(api_key="test-key")

    def test_validate_environment_missing_api_key(self):
        """Test validate_environment method with missing API key."""
        with patch.dict(os.environ, {}, clear=True):
            with self.assertRaises(Exception) as context:
                BraintrustLogger(api_key=None)
            self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception))

    @patch("litellm.integrations.braintrust_logging.HTTPHandler")
    def test_log_success_event_with_default_span_name(self, MockHTTPHandler):
        """Test log_success_event uses default span name when not provided."""
        # Mock HTTP response
        mock_response = Mock()
        mock_response.json.return_value = {"id": "test-project-id"}
        mock_http_handler = Mock()
        mock_http_handler.post.return_value = mock_response
        MockHTTPHandler.return_value = mock_http_handler

        # Setup
        logger = BraintrustLogger(api_key="test-key")
        logger.default_project_id = "test-project-id"

        # Create a mock response object
        message_mock = Mock()
        message_mock.json = Mock(return_value={"content": "test"})

        choice_mock = Mock()
        choice_mock.message = message_mock
        choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
        # Mock the __getitem__ to support response_obj["choices"][0]["message"]
        choice_mock.__getitem__ = Mock(return_value=message_mock)

        response_obj = Mock(spec=litellm.ModelResponse)
        response_obj.choices = [choice_mock]
        # Mock the __getitem__ to support response_obj["choices"]
        response_obj.__getitem__ = Mock(return_value=[choice_mock])
        response_obj.usage = litellm.Usage(
            prompt_tokens=10, completion_tokens=20, total_tokens=30
        )

        kwargs = {
            "litellm_call_id": "test-call-id",
            "messages": [{"role": "user", "content": "test"}],
            "litellm_params": {"metadata": {}},
            "model": "gpt-3.5-turbo",
            "response_cost": 0.001,
        }

        # Execute
        logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())

        # Verify
        call_args = mock_http_handler.post.call_args
        self.assertIsNotNone(call_args)
        json_data = call_args.kwargs["json"]
        self.assertEqual(
            json_data["events"][0]["span_attributes"]["name"], "Chat Completion"
        )

    @patch("litellm.integrations.braintrust_logging.HTTPHandler")
    def test_log_success_event_with_custom_span_name(self, MockHTTPHandler):
        """Test log_success_event uses custom span name when provided."""
        # Mock HTTP response
        mock_response = Mock()
        mock_response.json.return_value = {"id": "test-project-id"}
        mock_http_handler = Mock()
        mock_http_handler.post.return_value = mock_response
        MockHTTPHandler.return_value = mock_http_handler

        # Setup
        logger = BraintrustLogger(api_key="test-key")
        logger.default_project_id = "test-project-id"

        # Create a mock response object
        message_mock = Mock()
        message_mock.json = Mock(return_value={"content": "test"})

        choice_mock = Mock()
        choice_mock.message = message_mock
        choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
        choice_mock.__getitem__ = Mock(return_value=message_mock)

        response_obj = Mock(spec=litellm.ModelResponse)
        response_obj.choices = [choice_mock]
        response_obj.__getitem__ = Mock(return_value=[choice_mock])
        response_obj.usage = litellm.Usage(
            prompt_tokens=10, completion_tokens=20, total_tokens=30
        )

        kwargs = {
            "litellm_call_id": "test-call-id",
            "messages": [{"role": "user", "content": "test"}],
            "litellm_params": {"metadata": {"span_name": "Custom Operation"}},
            "model": "gpt-3.5-turbo",
            "response_cost": 0.001,
        }

        # Execute
        logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())

        # Verify
        call_args = mock_http_handler.post.call_args
        self.assertIsNotNone(call_args)
        json_data = call_args.kwargs["json"]
        self.assertEqual(
            json_data["events"][0]["span_attributes"]["name"], "Custom Operation"
        )

    @patch("litellm.integrations.braintrust_logging.get_async_httpx_client")
    async def test_async_log_success_event_with_default_span_name(
        self, mock_get_http_handler
    ):
        """Test async_log_success_event uses default span name when not provided."""
        # Mock async HTTP response
        mock_response = Mock()
        mock_response.json.return_value = {"id": "test-project-id"}
        mock_http_handler = MagicMock()
        mock_http_handler.post = MagicMock(return_value=mock_response)
        mock_get_http_handler.return_value = mock_http_handler

        # Setup
        logger = BraintrustLogger(api_key="test-key")
        logger.default_project_id = "test-project-id"

        # Create a mock response object
        message_mock = Mock()
        message_mock.json = Mock(return_value={"content": "test"})

        choice_mock = Mock()
        choice_mock.message = message_mock
        choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
        choice_mock.__getitem__ = Mock(return_value=message_mock)

        response_obj = Mock(spec=litellm.ModelResponse)
        response_obj.choices = [choice_mock]
        response_obj.__getitem__ = Mock(return_value=[choice_mock])
        response_obj.usage = litellm.Usage(
            prompt_tokens=10, completion_tokens=20, total_tokens=30
        )

        kwargs = {
            "litellm_call_id": "test-call-id",
            "messages": [{"role": "user", "content": "test"}],
            "litellm_params": {"metadata": {}},
            "model": "gpt-3.5-turbo",
            "response_cost": 0.001,
        }

        # Execute
        await logger.async_log_success_event(
            kwargs, response_obj, datetime.now(), datetime.now()
        )

        # Verify
        call_args = mock_http_handler.post.call_args
        self.assertIsNotNone(call_args)
        json_data = call_args.kwargs["json"]
        self.assertEqual(
            json_data["events"][0]["span_attributes"]["name"], "Chat Completion"
        )

    @patch("litellm.integrations.braintrust_logging.get_async_httpx_client")
    async def test_async_log_success_event_with_custom_span_name(
        self, mock_get_http_handler
    ):
        """Test async_log_success_event uses custom span name when provided."""
        # Mock async HTTP response
        mock_response = Mock()
        mock_response.json.return_value = {"id": "test-project-id"}
        mock_http_handler = MagicMock()
        mock_http_handler.post = MagicMock(return_value=mock_response)
        mock_get_http_handler.return_value = mock_http_handler

        # Setup
        logger = BraintrustLogger(api_key="test-key")
        logger.default_project_id = "test-project-id"

        # Create a mock response object
        message_mock = Mock()
        message_mock.json = Mock(return_value={"content": "test"})

        choice_mock = Mock()
        choice_mock.message = message_mock
        choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
        choice_mock.__getitem__ = Mock(return_value=message_mock)

        response_obj = Mock(spec=litellm.ModelResponse)
        response_obj.choices = [choice_mock]
        response_obj.__getitem__ = Mock(return_value=[choice_mock])
        response_obj.usage = litellm.Usage(
            prompt_tokens=10, completion_tokens=20, total_tokens=30
        )

        kwargs = {
            "litellm_call_id": "test-call-id",
            "messages": [{"role": "user", "content": "test"}],
            "litellm_params": {"metadata": {"span_name": "Async Custom Operation"}},
            "model": "gpt-3.5-turbo",
            "response_cost": 0.001,
        }

        # Execute
        await logger.async_log_success_event(
            kwargs, response_obj, datetime.now(), datetime.now()
        )

        # Verify
        call_args = mock_http_handler.post.call_args
        self.assertIsNotNone(call_args)
        json_data = call_args.kwargs["json"]
        self.assertEqual(
            json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation"
        )

    @patch("litellm.integrations.braintrust_logging.HTTPHandler")
    def test_span_name_with_multiple_metadata_fields(self, MockHTTPHandler):
        """Test that span_name works correctly alongside other metadata fields."""
        # Mock HTTP response
        mock_response = Mock()
        mock_response.json.return_value = {"id": "test-project-id"}
        mock_http_handler = Mock()
        mock_http_handler.post.return_value = mock_response
        MockHTTPHandler.return_value = mock_http_handler

        # Setup
        logger = BraintrustLogger(api_key="test-key")
        logger.default_project_id = "test-project-id"

        # Create a mock response object
        message_mock = Mock()
        message_mock.json = Mock(return_value={"content": "test"})

        choice_mock = Mock()
        choice_mock.message = message_mock
        choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
        choice_mock.__getitem__ = Mock(return_value=message_mock)

        response_obj = Mock(spec=litellm.ModelResponse)
        response_obj.choices = [choice_mock]
        response_obj.__getitem__ = Mock(return_value=[choice_mock])
        response_obj.usage = litellm.Usage(
            prompt_tokens=10, completion_tokens=20, total_tokens=30
        )

        kwargs = {
            "litellm_call_id": "test-call-id",
            "messages": [{"role": "user", "content": "test"}],
            "litellm_params": {
                "metadata": {
                    "span_name": "Multi Metadata Test",
                    "project_id": "custom-project",
                    "user_id": "user123",
                    "session_id": "session456",
                }
            },
            "model": "gpt-3.5-turbo",
            "response_cost": 0.001,
        }

        # Execute
        logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())

        # Verify
        call_args = mock_http_handler.post.call_args
        self.assertIsNotNone(call_args)
        json_data = call_args.kwargs["json"]

        # Check span name
        self.assertEqual(
            json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test"
        )

        # Check that other metadata is preserved
        event_metadata = json_data["events"][0]["metadata"]
        self.assertEqual(event_metadata["user_id"], "user123")
        self.assertEqual(event_metadata["session_id"], "session456")


async def _assert_completion_is_logged_by_braintrust(
    project_id: str, metadata: dict[str, object]
) -> None:
    logger: Final = BraintrustLogger(
        api_key="test-key",
        api_base="https://api.braintrustdata.com/v1",
    )
    logger.default_project_id = project_id
    messages: Final = [{"role": "user", "content": "Summarize this request"}]
    response: Final = litellm.ModelResponse(
        id="chatcmpl-test",
        model="gpt-4o-mini",
        choices=[
            {
                "index": 0,
                "message": {"role": "assistant", "content": "The request is summarized."},
                "finish_reason": "stop",
            }
        ],
        usage=litellm.Usage(prompt_tokens=2, completion_tokens=5, total_tokens=7),
    )
    kwargs: Final = {
        "model": "gpt-4o-mini",
        "messages": messages,
        "litellm_params": {"metadata": metadata},
        "litellm_call_id": "call-test",
        "standard_logging_object": {
            "model": "gpt-4o-mini",
            "messages": messages,
            "response": response.model_dump(),
            "metadata": metadata,
        },
    }

    with respx.mock(assert_all_called=True) as router:
        insert: Final = router.post(
            f"https://api.braintrustdata.com/v1/project_logs/{project_id}/insert"
        ).mock(return_value=httpx.Response(200, json={"ok": True}))
        fixed_time: Final = datetime(2024, 1, 1)
        logger.log_success_event(kwargs, response, fixed_time, fixed_time)

    assert insert.call_count == 1
    event: Final = json.loads(insert.calls[0].request.content)["events"][0]
    assert event["input"] == messages
    assert event["output"][0]["message"]["content"] == "The request is summarized."
    assert event["metadata"]["model"] == "gpt-4o-mini"
    assert event["metadata"]["messages"] == messages
    assert event["metadata"]["response"]["choices"][0]["message"]["content"] == "The request is summarized."
    assert event["metrics"]["prompt_tokens"] == response.usage.prompt_tokens
    assert event["metrics"]["completion_tokens"] == response.usage.completion_tokens
    assert event["metrics"]["total_tokens"] == response.usage.total_tokens
    assert event["span_attributes"]["name"] == "Chat Completion"


@pytest.mark.asyncio
async def test_braintrust_logging() -> None:
    await _assert_completion_is_logged_by_braintrust(
        "default-project",
        {},
    )


@pytest.mark.asyncio
async def test_braintrust_logging_specific_project_id() -> None:
    await _assert_completion_is_logged_by_braintrust(
        "123",
        {"project_id": "123", "requester_metadata": {"request_id": "request-123"}},
    )


def test_braintrust_logger_initialization_reuses_instance(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setenv("BRAINTRUST_API_KEY", "test-api-key")
    monkeypatch.setenv("BRAINTRUST_API_BASE", "https://api.braintrustdata.com/v1")
    first: Final = litellm_logging._init_custom_logger_compatible_class(
        "braintrust",
        internal_usage_cache=None,
        llm_router=None,
    )
    second: Final = litellm_logging._init_custom_logger_compatible_class(
        "braintrust",
        internal_usage_cache=None,
        llm_router=None,
    )

    assert isinstance(first, BraintrustLogger)
    assert second is first