:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""
Integration tests for LiteLLM callbacks.
Verifies that callbacks are properly invoked during actual LLM requests.
Skipped unless ANTHROPIC_API_KEY is set.
"""
import os
from unittest.mock import patch
import pytest
from anthropic import Anthropic
from openai import OpenAI
from llm_gateway.metrics.prometheus import CALLBACK_SUCCESS, TOKENS_INPUT, TOKENS_OUTPUT
ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY")
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
ANTHROPIC_TEST_MODEL = "claude-haiku-4-5-20251001"
pytestmark = pytest.mark.xfail(strict=False, reason="Upstream provider may be rate-limited or temporarily unavailable")
class TestCallbacksFireOnAnthropicRequest:
pytestmark = pytest.mark.skipif(not ANTHROPIC_API_KEY, reason="ANTHROPIC_API_KEY not set")
def test_callbacks_fire_on_non_streaming_request(self, anthropic_client: Anthropic) -> None:
initial_rate_limit = CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get()
initial_prometheus = CALLBACK_SUCCESS.labels(callback="prometheus")._value.get()
anthropic_client.messages.create(
model=ANTHROPIC_TEST_MODEL,
messages=[{"role": "user", "content": "Say 'test'"}],
max_tokens=5,
)
assert CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get() > initial_rate_limit
assert CALLBACK_SUCCESS.labels(callback="prometheus")._value.get() > initial_prometheus
def test_callbacks_fire_on_streaming_request(self, anthropic_client: Anthropic) -> None:
initial_rate_limit = CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get()
initial_prometheus = CALLBACK_SUCCESS.labels(callback="prometheus")._value.get()
with anthropic_client.messages.stream(
model=ANTHROPIC_TEST_MODEL,
messages=[{"role": "user", "content": "Say 'hi'"}],
max_tokens=5,
) as stream:
stream.get_final_text()
assert CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get() > initial_rate_limit
assert CALLBACK_SUCCESS.labels(callback="prometheus")._value.get() > initial_prometheus
def test_prometheus_callback_records_tokens(self, anthropic_client: Anthropic) -> None:
initial_input = TOKENS_INPUT.labels(
provider="anthropic", model=ANTHROPIC_TEST_MODEL, product="llm_gateway"
)._value.get()
initial_output = TOKENS_OUTPUT.labels(
provider="anthropic", model=ANTHROPIC_TEST_MODEL, product="llm_gateway"
)._value.get()
anthropic_client.messages.create(
model=ANTHROPIC_TEST_MODEL,
messages=[{"role": "user", "content": "Say 'a'"}],
max_tokens=5,
)
assert (
TOKENS_INPUT.labels(provider="anthropic", model=ANTHROPIC_TEST_MODEL, product="llm_gateway")._value.get()
> initial_input
)
assert (
TOKENS_OUTPUT.labels(provider="anthropic", model=ANTHROPIC_TEST_MODEL, product="llm_gateway")._value.get()
> initial_output
)
class TestCallbacksFireOnOpenAIRequest:
pytestmark = pytest.mark.skipif(not OPENAI_API_KEY, reason="OPENAI_API_KEY not set")
def test_callbacks_fire_on_non_streaming_request(self, openai_client: OpenAI) -> None:
initial_rate_limit = CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get()
initial_prometheus = CALLBACK_SUCCESS.labels(callback="prometheus")._value.get()
openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Say 'test'"}],
max_tokens=5,
)
assert CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get() > initial_rate_limit
assert CALLBACK_SUCCESS.labels(callback="prometheus")._value.get() > initial_prometheus
def test_callbacks_fire_on_streaming_request(self, openai_client: OpenAI) -> None:
initial_rate_limit = CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get()
initial_prometheus = CALLBACK_SUCCESS.labels(callback="prometheus")._value.get()
stream = openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Say 'hi'"}],
max_tokens=5,
stream=True,
)
for _ in stream:
pass
assert CALLBACK_SUCCESS.labels(callback="rate_limit")._value.get() > initial_rate_limit
assert CALLBACK_SUCCESS.labels(callback="prometheus")._value.get() > initial_prometheus
class TestCallbacksReceiveCorrectData:
pytestmark = pytest.mark.skipif(not ANTHROPIC_API_KEY, reason="ANTHROPIC_API_KEY not set")
def test_callback_receives_token_counts(self, anthropic_client: Anthropic) -> None:
received_data = {}
async def capture_on_success(self, kwargs, response_obj, start_time, end_time, end_user_id):
standard_logging_object = kwargs.get("standard_logging_object", {})
received_data["prompt_tokens"] = standard_logging_object.get("prompt_tokens")
received_data["completion_tokens"] = standard_logging_object.get("completion_tokens")
received_data["model"] = standard_logging_object.get("model")
received_data["provider"] = standard_logging_object.get("custom_llm_provider")
from llm_gateway.callbacks.prometheus import PrometheusCallback
with patch.object(PrometheusCallback, "_on_success", capture_on_success):
anthropic_client.messages.create(
model=ANTHROPIC_TEST_MODEL,
messages=[{"role": "user", "content": "Say 'x'"}],
max_tokens=5,
)
prompt_tokens = received_data.get("prompt_tokens")
completion_tokens = received_data.get("completion_tokens")
assert prompt_tokens is not None
assert prompt_tokens > 0
assert completion_tokens is not None
assert completion_tokens > 0
assert "claude" in received_data.get("model", "").lower()
assert received_data.get("provider") == "anthropic"
def test_callback_receives_streaming_token_counts(self, anthropic_client: Anthropic) -> None:
received_data = {}
async def capture_on_success(self, kwargs, response_obj, start_time, end_time, end_user_id):
standard_logging_object = kwargs.get("standard_logging_object", {})
received_data["prompt_tokens"] = standard_logging_object.get("prompt_tokens")
received_data["completion_tokens"] = standard_logging_object.get("completion_tokens")
from llm_gateway.callbacks.prometheus import PrometheusCallback
with patch.object(PrometheusCallback, "_on_success", capture_on_success):
with anthropic_client.messages.stream(
model=ANTHROPIC_TEST_MODEL,
messages=[{"role": "user", "content": "Say 'y'"}],
max_tokens=5,
) as stream:
stream.get_final_text()
assert received_data.get("prompt_tokens") is not None
assert received_data.get("completion_tokens") is not None
class TestEndUserIdExtraction:
pytestmark = pytest.mark.skipif(not OPENAI_API_KEY, reason="OPENAI_API_KEY not set")
def test_openai_user_param_is_not_used_as_end_user(self, openai_client: OpenAI) -> None:
"""Verify that the 'user' parameter in OpenAI requests is NOT used as end_user_id.
The end_user_id is always the authenticated user's ID to prevent rate limit poisoning.
"""
received_data: dict[str, object] = {}
async def capture_on_success(self, kwargs, response_obj, start_time, end_time, end_user_id):
received_data["end_user_id"] = end_user_id
received_data["user_in_kwargs"] = kwargs.get("user")
from llm_gateway.callbacks.rate_limiting import RateLimitCallback
with patch.object(RateLimitCallback, "_on_success", capture_on_success):
openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Say 'test'"}],
max_tokens=5,
user="test-end-user-123",
)
# end_user_id is always the authenticated user's ID (123 from conftest fixture),
# never the client-provided 'user' param
assert received_data.get("end_user_id") == "123"
assert received_data.get("user_in_kwargs") == "test-end-user-123"
def test_openai_request_without_user_param(self, openai_client: OpenAI) -> None:
"""Verify that requests without 'user' parameter still have end_user_id from auth."""
received_data: dict[str, str | None] = {}
async def capture_on_success(self, kwargs, response_obj, start_time, end_time, end_user_id):
received_data["end_user_id"] = end_user_id
from llm_gateway.callbacks.rate_limiting import RateLimitCallback
with patch.object(RateLimitCallback, "_on_success", capture_on_success):
openai_client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "Say 'test'"}],
max_tokens=5,
)
# end_user_id is always the authenticated user's ID (123 from conftest fixture)
assert received_data.get("end_user_id") == "123"