Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.6 KB
Raw Blame
from unittest.mock import patch

import pytest

from llm_gateway.auth.models import AuthenticatedUser
from llm_gateway.callbacks.base import InstrumentedCallback
from llm_gateway.metrics.prometheus import CALLBACK_ERRORS, CALLBACK_SUCCESS


class MockCallback(InstrumentedCallback):
    callback_name = "test"

    def __init__(self):
        super().__init__()
        self.on_success_called = False
        self.on_failure_called = False
        self.raise_on_success = False
        self.raise_on_failure = False
        self.received_end_user_id: str | None = None

    async def _on_success(self, kwargs, response_obj, start_time, end_time, end_user_id) -> None:
        if self.raise_on_success:
            raise ValueError("Test error in success")
        self.on_success_called = True
        self.received_end_user_id = end_user_id

    async def _on_failure(self, kwargs, response_obj, start_time, end_time, end_user_id) -> None:
        if self.raise_on_failure:
            raise ValueError("Test error in failure")
        self.on_failure_called = True
        self.received_end_user_id = end_user_id


class TestInstrumentedCallback:
    @pytest.fixture
    def callback(self):
        return MockCallback()

    @pytest.mark.asyncio
    async def test_success_event_calls_on_success(self, callback: MockCallback) -> None:
        await callback.async_log_success_event({}, None, 0.0, 1.0)

        assert callback.on_success_called is True
        assert callback.on_failure_called is False

    @pytest.mark.asyncio
    async def test_failure_event_calls_on_failure(self, callback: MockCallback) -> None:
        await callback.async_log_failure_event({}, None, 0.0, 1.0)

        assert callback.on_failure_called is True
        assert callback.on_success_called is False

    @pytest.mark.asyncio
    async def test_success_event_increments_success_metric(self, callback: MockCallback) -> None:
        initial = CALLBACK_SUCCESS.labels(callback="test")._value.get()

        await callback.async_log_success_event({}, None, 0.0, 1.0)

        assert CALLBACK_SUCCESS.labels(callback="test")._value.get() == initial + 1

    @pytest.mark.asyncio
    async def test_failure_event_increments_success_metric(self, callback: MockCallback) -> None:
        initial = CALLBACK_SUCCESS.labels(callback="test")._value.get()

        await callback.async_log_failure_event({}, None, 0.0, 1.0)

        assert CALLBACK_SUCCESS.labels(callback="test")._value.get() == initial + 1

    @pytest.mark.asyncio
    async def test_error_in_success_increments_error_metric(self, callback: MockCallback) -> None:
        callback.raise_on_success = True
        initial = CALLBACK_ERRORS.labels(callback="test", error_type="ValueError")._value.get()

        await callback.async_log_success_event({}, None, 0.0, 1.0)

        assert CALLBACK_ERRORS.labels(callback="test", error_type="ValueError")._value.get() == initial + 1

    @pytest.mark.asyncio
    async def test_error_in_failure_increments_error_metric(self, callback: MockCallback) -> None:
        callback.raise_on_failure = True
        initial = CALLBACK_ERRORS.labels(callback="test", error_type="ValueError")._value.get()

        await callback.async_log_failure_event({}, None, 0.0, 1.0)

        assert CALLBACK_ERRORS.labels(callback="test", error_type="ValueError")._value.get() == initial + 1

    @pytest.mark.asyncio
    async def test_error_in_success_captures_exception(self, callback: MockCallback) -> None:
        callback.raise_on_success = True

        with patch("llm_gateway.callbacks.base.capture_exception") as mock_capture:
            await callback.async_log_success_event({}, None, 0.0, 1.0)

            mock_capture.assert_called_once()
            args = mock_capture.call_args
            assert isinstance(args[0][0], ValueError)
            assert args[0][1] == {"callback": "test", "event": "success"}

    @pytest.mark.asyncio
    async def test_error_in_failure_captures_exception(self, callback: MockCallback) -> None:
        callback.raise_on_failure = True

        with patch("llm_gateway.callbacks.base.capture_exception") as mock_capture:
            await callback.async_log_failure_event({}, None, 0.0, 1.0)

            mock_capture.assert_called_once()
            args = mock_capture.call_args
            assert isinstance(args[0][0], ValueError)
            assert args[0][1] == {"callback": "test", "event": "failure"}

    @pytest.mark.asyncio
    async def test_error_does_not_propagate(self, callback: MockCallback) -> None:
        callback.raise_on_success = True
        callback.raise_on_failure = True

        # Should not raise
        await callback.async_log_success_event({}, None, 0.0, 1.0)
        await callback.async_log_failure_event({}, None, 0.0, 1.0)

    @pytest.mark.asyncio
    async def test_kwargs_passed_to_on_success(self, callback: MockCallback) -> None:
        received_kwargs: dict[str, object] = {}

        async def capture_kwargs(self, kwargs, response_obj, start_time, end_time, end_user_id):
            received_kwargs.update(kwargs)

        test_kwargs = {"standard_logging_object": {"model": "test-model"}}

        with patch.object(MockCallback, "_on_success", capture_kwargs):
            await callback.async_log_success_event(test_kwargs, None, 0.0, 1.0)

        assert received_kwargs == test_kwargs


class TestExtractEndUserId:
    @pytest.fixture
    def callback(self) -> InstrumentedCallback:
        return MockCallback()

    @pytest.fixture
    def oauth_user(self) -> AuthenticatedUser:
        return AuthenticatedUser(
            user_id=123,
            team_id=456,
            auth_method="oauth_access_token",
            distinct_id="user-distinct-id-123",
        )

    @pytest.fixture
    def api_key_user(self) -> AuthenticatedUser:
        return AuthenticatedUser(
            user_id=123,
            team_id=456,
            auth_method="personal_api_key",
            distinct_id="user-distinct-id-123",
        )

    def test_oauth_returns_user_id(self, callback: InstrumentedCallback, oauth_user: AuthenticatedUser) -> None:
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=oauth_user):
            result = callback._extract_end_user_id({})
        assert result == "123"

    def test_oauth_ignores_end_user_in_request(
        self, callback: InstrumentedCallback, oauth_user: AuthenticatedUser
    ) -> None:
        kwargs = {"standard_logging_object": {"end_user": "request-user"}}
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=oauth_user):
            result = callback._extract_end_user_id(kwargs)
        assert result == "123"

    def test_api_key_returns_auth_user_id(
        self, callback: InstrumentedCallback, api_key_user: AuthenticatedUser
    ) -> None:
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=api_key_user):
            result = callback._extract_end_user_id({})
        assert result == "123"

    def test_api_key_ignores_client_provided_user(
        self, callback: InstrumentedCallback, api_key_user: AuthenticatedUser
    ) -> None:
        kwargs = {"user": "attacker-supplied-id", "standard_logging_object": {"end_user": "request-user"}}
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=api_key_user):
            result = callback._extract_end_user_id(kwargs)
        assert result == "123"

    def test_api_key_ignores_metadata_user_id(
        self, callback: InstrumentedCallback, api_key_user: AuthenticatedUser
    ) -> None:
        kwargs = {"litellm_params": {"metadata": {"user_id": "anthropic-user"}}}
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=api_key_user):
            result = callback._extract_end_user_id(kwargs)
        assert result == "123"

    def test_no_auth_user_returns_none(self, callback: InstrumentedCallback) -> None:
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=None):
            result = callback._extract_end_user_id({})
        assert result is None

    def test_no_auth_user_returns_none_even_with_kwargs(self, callback: InstrumentedCallback) -> None:
        kwargs = {"user": "some-user", "standard_logging_object": {"end_user": "request-user"}}
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=None):
            result = callback._extract_end_user_id(kwargs)
        assert result is None


class TestExtractEndUserIdPoisoningPrevention:
    """Verify that client-provided user fields cannot poison rate limit buckets.

    An attacker could supply another user's ID in the 'user' request param,
    'standard_logging_object.end_user', or 'litellm_params.metadata.user_id'
    to bill costs against the victim's quota.
    """

    @pytest.fixture
    def callback(self) -> InstrumentedCallback:
        return MockCallback()

    @pytest.fixture
    def attacker_user(self) -> AuthenticatedUser:
        return AuthenticatedUser(
            user_id=999,
            team_id=1,
            auth_method="personal_api_key",
            distinct_id="attacker-distinct-id",
        )

    @pytest.mark.parametrize(
        "kwargs",
        [
            pytest.param({"user": "42"}, id="openai_user_param_with_victim_id"),
            pytest.param({"user": "victim-user-id"}, id="openai_user_param_with_victim_string"),
            pytest.param(
                {"standard_logging_object": {"end_user": "42"}},
                id="standard_logging_object_end_user",
            ),
            pytest.param(
                {"litellm_params": {"metadata": {"user_id": "42"}}},
                id="anthropic_metadata_user_id",
            ),
            pytest.param(
                {
                    "user": "42",
                    "standard_logging_object": {"end_user": "42"},
                    "litellm_params": {"metadata": {"user_id": "42"}},
                },
                id="all_injection_vectors_combined",
            ),
        ],
    )
    def test_attacker_cannot_poison_victim_rate_limit_bucket(
        self, callback: InstrumentedCallback, attacker_user: AuthenticatedUser, kwargs: dict
    ) -> None:
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=attacker_user):
            result = callback._extract_end_user_id(kwargs)
        assert result == "999", "Must return attacker's own authenticated ID, not the injected victim ID"

    def test_attacker_cannot_evade_rate_limiting_by_omitting_user(
        self, callback: InstrumentedCallback, attacker_user: AuthenticatedUser
    ) -> None:
        with patch("llm_gateway.callbacks.base.get_auth_user", return_value=attacker_user):
            result = callback._extract_end_user_id({})
        assert result == "999", "Must return authenticated ID even when no user param is provided"