The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import json
import os
import unittest
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock, Mock, patch
import httpx
import litellm
import pytest
import respx
from litellm.litellm_core_utils import litellm_logging
from litellm.integrations.braintrust_logging import BraintrustLogger
class TestBraintrustLogger(unittest.TestCase):
@patch.dict(os.environ, {"BRAINTRUST_API_KEY": "test-env-api-key"})
@patch.dict(os.environ, {"BRAINTRUST_API_BASE": "https://test-env-api.com/v1"})
def test_init_with_env_var(self):
"""Test BraintrustLogger initialization with environment variable."""
logger = BraintrustLogger()
self.assertEqual(logger.api_key, "test-env-api-key")
self.assertEqual(logger.api_base, "https://test-env-api.com/v1")
self.assertEqual(logger.headers["Authorization"], "Bearer test-env-api-key")
self.assertEqual(logger.headers["Content-Type"], "application/json")
def test_init_with_explicit_params(self):
"""Test BraintrustLogger initialization with explicit parameters."""
logger = BraintrustLogger(
api_key="explicit-key", api_base="https://custom-api.com/v1"
)
self.assertEqual(logger.api_key, "explicit-key")
self.assertEqual(logger.api_base, "https://custom-api.com/v1")
self.assertEqual(logger.headers["Authorization"], "Bearer explicit-key")
@patch.dict(os.environ, {}, clear=True)
def test_init_missing_api_key(self):
"""Test BraintrustLogger initialization fails without API key."""
with self.assertRaises(Exception) as context:
BraintrustLogger()
self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception))
def test_validate_environment_with_api_key(self):
"""Test validate_environment method with valid API key."""
logger = BraintrustLogger(api_key="test-key")
# Should not raise an exception
logger.validate_environment(api_key="test-key")
def test_validate_environment_missing_api_key(self):
"""Test validate_environment method with missing API key."""
with patch.dict(os.environ, {}, clear=True):
with self.assertRaises(Exception) as context:
BraintrustLogger(api_key=None)
self.assertIn("Missing keys=['BRAINTRUST_API_KEY']", str(context.exception))
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
def test_log_success_event_with_default_span_name(self, MockHTTPHandler):
"""Test log_success_event uses default span name when not provided."""
# Mock HTTP response
mock_response = Mock()
mock_response.json.return_value = {"id": "test-project-id"}
mock_http_handler = Mock()
mock_http_handler.post.return_value = mock_response
MockHTTPHandler.return_value = mock_http_handler
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a mock response object
message_mock = Mock()
message_mock.json = Mock(return_value={"content": "test"})
choice_mock = Mock()
choice_mock.message = message_mock
choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
# Mock the __getitem__ to support response_obj["choices"][0]["message"]
choice_mock.__getitem__ = Mock(return_value=message_mock)
response_obj = Mock(spec=litellm.ModelResponse)
response_obj.choices = [choice_mock]
# Mock the __getitem__ to support response_obj["choices"]
response_obj.__getitem__ = Mock(return_value=[choice_mock])
response_obj.usage = litellm.Usage(
prompt_tokens=10, completion_tokens=20, total_tokens=30
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
}
# Execute
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Chat Completion"
)
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
def test_log_success_event_with_custom_span_name(self, MockHTTPHandler):
"""Test log_success_event uses custom span name when provided."""
# Mock HTTP response
mock_response = Mock()
mock_response.json.return_value = {"id": "test-project-id"}
mock_http_handler = Mock()
mock_http_handler.post.return_value = mock_response
MockHTTPHandler.return_value = mock_http_handler
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a mock response object
message_mock = Mock()
message_mock.json = Mock(return_value={"content": "test"})
choice_mock = Mock()
choice_mock.message = message_mock
choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
choice_mock.__getitem__ = Mock(return_value=message_mock)
response_obj = Mock(spec=litellm.ModelResponse)
response_obj.choices = [choice_mock]
response_obj.__getitem__ = Mock(return_value=[choice_mock])
response_obj.usage = litellm.Usage(
prompt_tokens=10, completion_tokens=20, total_tokens=30
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {"span_name": "Custom Operation"}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
}
# Execute
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Custom Operation"
)
@patch("litellm.integrations.braintrust_logging.get_async_httpx_client")
async def test_async_log_success_event_with_default_span_name(
self, mock_get_http_handler
):
"""Test async_log_success_event uses default span name when not provided."""
# Mock async HTTP response
mock_response = Mock()
mock_response.json.return_value = {"id": "test-project-id"}
mock_http_handler = MagicMock()
mock_http_handler.post = MagicMock(return_value=mock_response)
mock_get_http_handler.return_value = mock_http_handler
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a mock response object
message_mock = Mock()
message_mock.json = Mock(return_value={"content": "test"})
choice_mock = Mock()
choice_mock.message = message_mock
choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
choice_mock.__getitem__ = Mock(return_value=message_mock)
response_obj = Mock(spec=litellm.ModelResponse)
response_obj.choices = [choice_mock]
response_obj.__getitem__ = Mock(return_value=[choice_mock])
response_obj.usage = litellm.Usage(
prompt_tokens=10, completion_tokens=20, total_tokens=30
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
}
# Execute
await logger.async_log_success_event(
kwargs, response_obj, datetime.now(), datetime.now()
)
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Chat Completion"
)
@patch("litellm.integrations.braintrust_logging.get_async_httpx_client")
async def test_async_log_success_event_with_custom_span_name(
self, mock_get_http_handler
):
"""Test async_log_success_event uses custom span name when provided."""
# Mock async HTTP response
mock_response = Mock()
mock_response.json.return_value = {"id": "test-project-id"}
mock_http_handler = MagicMock()
mock_http_handler.post = MagicMock(return_value=mock_response)
mock_get_http_handler.return_value = mock_http_handler
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a mock response object
message_mock = Mock()
message_mock.json = Mock(return_value={"content": "test"})
choice_mock = Mock()
choice_mock.message = message_mock
choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
choice_mock.__getitem__ = Mock(return_value=message_mock)
response_obj = Mock(spec=litellm.ModelResponse)
response_obj.choices = [choice_mock]
response_obj.__getitem__ = Mock(return_value=[choice_mock])
response_obj.usage = litellm.Usage(
prompt_tokens=10, completion_tokens=20, total_tokens=30
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {"metadata": {"span_name": "Async Custom Operation"}},
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
}
# Execute
await logger.async_log_success_event(
kwargs, response_obj, datetime.now(), datetime.now()
)
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs["json"]
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Async Custom Operation"
)
@patch("litellm.integrations.braintrust_logging.HTTPHandler")
def test_span_name_with_multiple_metadata_fields(self, MockHTTPHandler):
"""Test that span_name works correctly alongside other metadata fields."""
# Mock HTTP response
mock_response = Mock()
mock_response.json.return_value = {"id": "test-project-id"}
mock_http_handler = Mock()
mock_http_handler.post.return_value = mock_response
MockHTTPHandler.return_value = mock_http_handler
# Setup
logger = BraintrustLogger(api_key="test-key")
logger.default_project_id = "test-project-id"
# Create a mock response object
message_mock = Mock()
message_mock.json = Mock(return_value={"content": "test"})
choice_mock = Mock()
choice_mock.message = message_mock
choice_mock.dict = Mock(return_value={"message": {"content": "test"}})
choice_mock.__getitem__ = Mock(return_value=message_mock)
response_obj = Mock(spec=litellm.ModelResponse)
response_obj.choices = [choice_mock]
response_obj.__getitem__ = Mock(return_value=[choice_mock])
response_obj.usage = litellm.Usage(
prompt_tokens=10, completion_tokens=20, total_tokens=30
)
kwargs = {
"litellm_call_id": "test-call-id",
"messages": [{"role": "user", "content": "test"}],
"litellm_params": {
"metadata": {
"span_name": "Multi Metadata Test",
"project_id": "custom-project",
"user_id": "user123",
"session_id": "session456",
}
},
"model": "gpt-3.5-turbo",
"response_cost": 0.001,
}
# Execute
logger.log_success_event(kwargs, response_obj, datetime.now(), datetime.now())
# Verify
call_args = mock_http_handler.post.call_args
self.assertIsNotNone(call_args)
json_data = call_args.kwargs["json"]
# Check span name
self.assertEqual(
json_data["events"][0]["span_attributes"]["name"], "Multi Metadata Test"
)
# Check that other metadata is preserved
event_metadata = json_data["events"][0]["metadata"]
self.assertEqual(event_metadata["user_id"], "user123")
self.assertEqual(event_metadata["session_id"], "session456")
async def _assert_completion_is_logged_by_braintrust(
project_id: str, metadata: dict[str, object]
) -> None:
logger: Final = BraintrustLogger(
api_key="test-key",
api_base="https://api.braintrustdata.com/v1",
)
logger.default_project_id = project_id
messages: Final = [{"role": "user", "content": "Summarize this request"}]
response: Final = litellm.ModelResponse(
id="chatcmpl-test",
model="gpt-4o-mini",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "The request is summarized."},
"finish_reason": "stop",
}
],
usage=litellm.Usage(prompt_tokens=2, completion_tokens=5, total_tokens=7),
)
kwargs: Final = {
"model": "gpt-4o-mini",
"messages": messages,
"litellm_params": {"metadata": metadata},
"litellm_call_id": "call-test",
"standard_logging_object": {
"model": "gpt-4o-mini",
"messages": messages,
"response": response.model_dump(),
"metadata": metadata,
},
}
with respx.mock(assert_all_called=True) as router:
insert: Final = router.post(
f"https://api.braintrustdata.com/v1/project_logs/{project_id}/insert"
).mock(return_value=httpx.Response(200, json={"ok": True}))
fixed_time: Final = datetime(2024, 1, 1)
logger.log_success_event(kwargs, response, fixed_time, fixed_time)
assert insert.call_count == 1
event: Final = json.loads(insert.calls[0].request.content)["events"][0]
assert event["input"] == messages
assert event["output"][0]["message"]["content"] == "The request is summarized."
assert event["metadata"]["model"] == "gpt-4o-mini"
assert event["metadata"]["messages"] == messages
assert event["metadata"]["response"]["choices"][0]["message"]["content"] == "The request is summarized."
assert event["metrics"]["prompt_tokens"] == response.usage.prompt_tokens
assert event["metrics"]["completion_tokens"] == response.usage.completion_tokens
assert event["metrics"]["total_tokens"] == response.usage.total_tokens
assert event["span_attributes"]["name"] == "Chat Completion"
@pytest.mark.asyncio
async def test_braintrust_logging() -> None:
await _assert_completion_is_logged_by_braintrust(
"default-project",
{},
)
@pytest.mark.asyncio
async def test_braintrust_logging_specific_project_id() -> None:
await _assert_completion_is_logged_by_braintrust(
"123",
{"project_id": "123", "requester_metadata": {"request_id": "request-123"}},
)
def test_braintrust_logger_initialization_reuses_instance(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("BRAINTRUST_API_KEY", "test-api-key")
monkeypatch.setenv("BRAINTRUST_API_BASE", "https://api.braintrustdata.com/v1")
first: Final = litellm_logging._init_custom_logger_compatible_class(
"braintrust",
internal_usage_cache=None,
llm_router=None,
)
second: Final = litellm_logging._init_custom_logger_compatible_class(
"braintrust",
internal_usage_cache=None,
llm_router=None,
)
assert isinstance(first, BraintrustLogger)
assert second is first