Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

29.1 KB
Raw Blame
import asyncio
import importlib
import json
import logging
import os
from collections.abc import AsyncIterator, Iterator, Sequence
from typing import Awaitable, Final, TypedDict, TypeVar
from unittest.mock import AsyncMock

import httpx
import pytest
import pytest_asyncio
from pydantic import TypeAdapter
from typing_extensions import ReadOnly

import litellm
from litellm._logging import verbose_logger
from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE
from litellm.integrations.gcs_pubsub.pub_sub import *
from litellm.integrations.generic_api.generic_api_callback import LOG_FORMAT_TYPES, GenericAPILogger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import StandardLoggingPayload
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome


_T = TypeVar("_T")


async def _await_with_spawned_tasks(call: Awaitable[_T]) -> _T:
    before: Final = asyncio.all_tasks()
    result: Final = await call
    spawned: Final = asyncio.all_tasks() - before - {asyncio.current_task()}
    await asyncio.wait_for(asyncio.gather(*spawned), timeout=5)
    return result


@pytest.fixture(autouse=True)
def set_verbose_logger_level() -> Iterator[None]:
    original_level = verbose_logger.level
    verbose_logger.setLevel(logging.DEBUG)
    yield
    verbose_logger.setLevel(original_level)


@pytest.mark.asyncio
async def test_generic_api_callback():
    """
    Test the GenericAPILogger callback with a standard logging payload.
    This test mocks the HTTP client and validates that the logger properly
    formats and sends the expected payload.
    """

    # Create a mock for the async_httpx_client's post method
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    # Set up an endpoint for testing
    test_endpoint = "https://example.com/api/logs"
    test_headers = {"Authorization": "Bearer test_token"}
    os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint

    # Initialize the GenericAPILogger and set the mock
    generic_logger = GenericAPILogger(endpoint=test_endpoint, headers=test_headers, flush_interval=3600)
    generic_logger.async_httpx_client.post = mock_post
    litellm.callbacks = [generic_logger]

    # Make the completion call
    response = await _await_with_spawned_tasks(
        litellm.acompletion(
            model="gpt-5.5",
            messages=[{"role": "user", "content": "Hello, world!"}],
            mock_response="hi",
            user="test_user",
        )
    )

    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
    await generic_logger.flush_queue()

    # Assert httpx post was called
    mock_post.assert_called_once()

    # Get the actual request body from the mock
    actual_url = mock_post.call_args[1]["url"]
    print(
        "logs were flushed to URL",
        actual_url,
        "with the following headers",
        mock_post.call_args[1]["headers"],
    )
    assert actual_url == test_endpoint, f"Expected URL {test_endpoint}, got {actual_url}"

    # Validate headers
    assert mock_post.call_args[1]["headers"]["Content-Type"] == "application/json", (
        "Content-Type should be application/json"
    )

    # For the GenericAPILogger, it sends the payload directly as JSON in the data field
    json_data = mock_post.call_args[1]["data"]
    # Parse the JSON string

    print("##########\n")
    print("json_data", json_data)
    actual_request = json.loads(json_data)

    # The payload is a list of StandardLoggingPayload objects in the log queue
    assert isinstance(actual_request, list), "Request body should be a list"
    assert len(actual_request) > 0, "Request body list should not be empty"

    this_test_messages: Final = [{"role": "user", "content": "Hello, world!"}]
    mine: Final = [item for item in actual_request if item.get("messages") == this_test_messages]
    assert len(mine) == 1, f"Expected this test's single call in the batch, got {len(mine)} of {len(actual_request)}"

    payload_item: StandardLoggingPayload = StandardLoggingPayload(**mine[0])
    print("##########\n")
    print(json.dumps(payload_item, indent=4))
    print("##########\n")

    # Basic assertions for standard logging payload
    assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
    assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"
    assert payload_item["model_parameters"]["user"] == "test_user", "User should be test_user"
    assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"
    assert payload_item["messages"] == [{"role": "user", "content": "Hello, world!"}], "Messages should be the same"
    assert payload_item["response"]["choices"][0]["message"]["content"] == "hi", "Response should be hi"


@pytest.mark.asyncio
async def test_generic_api_callback_multiple_logs():
    """
    Test the GenericAPILogger callback with multiple chat completions
    """
    # Create a mock for the async_httpx_client's post method
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    # Set up an endpoint for testing
    test_endpoint = "https://example.com/api/logs"
    test_headers = {"Authorization": "Bearer test_token"}
    os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint

    # Initialize the GenericAPILogger and set the mock
    generic_logger = GenericAPILogger(endpoint=test_endpoint, headers=test_headers, flush_interval=3600)
    generic_logger.async_httpx_client.post = mock_post
    litellm.callbacks = [generic_logger]

    # Make the completion call
    for _ in range(10):
        response = await _await_with_spawned_tasks(
            litellm.acompletion(
                model="gpt-5.5",
                messages=[{"role": "user", "content": "Hello, world!"}],
                mock_response="hi",
                user="test_user",
            )
        )

    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
    await generic_logger.flush_queue()

    # Assert httpx post was called
    mock_post.assert_called_once()

    # Get the actual request body from the mock
    actual_url = mock_post.call_args[1]["url"]
    print("##########\n")
    print(
        "logs were flushed to URL",
        actual_url,
        "with the following headers",
        mock_post.call_args[1]["headers"],
    )
    assert actual_url == test_endpoint, f"Expected URL {test_endpoint}, got {actual_url}"

    # For the GenericAPILogger, it sends the payload directly as JSON in the data field
    json_data = mock_post.call_args[1]["data"]
    # Parse the JSON string

    print("##########\n")
    print("json_data", json_data)
    actual_request = json.loads(json_data)

    # The payload is a list of StandardLoggingPayload objects in the log queue
    assert isinstance(actual_request, list), "Request body should be a list"
    assert len(actual_request) > 0, "Request body list should not be empty"
    assert len(actual_request) == 10, "Request body list should be 10 items, since we made 10 calls"

    # Validate all payload items
    for payload_item in actual_request:
        payload_item: StandardLoggingPayload = StandardLoggingPayload(**payload_item)
        print("##########\n")
        print(json.dumps(payload_item, indent=4))
        print("##########\n")

        assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
        assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"
        assert payload_item["model_parameters"]["user"] == "test_user", "User should be test_user"
        assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"
        assert payload_item["messages"] == [{"role": "user", "content": "Hello, world!"}], "Messages should be the same"
        assert payload_item["response"]["choices"][0]["message"]["content"] == "hi", "Response should be hi"


@pytest.mark.asyncio
async def test_generic_api_callback_ndjson_format():
    """
    Test the GenericAPILogger callback with ndjson log format.
    Validates that logs are sent as newline-delimited JSON.
    """
    # Create a mock for the async_httpx_client's post method
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    # Set up an endpoint for testing
    test_endpoint = "https://example.com/api/logs"
    test_headers = {"Authorization": "Bearer test_token"}
    os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint

    # Initialize the GenericAPILogger with ndjson format
    generic_logger = GenericAPILogger(
        endpoint=test_endpoint,
        headers=test_headers,
        flush_interval=3600,
        log_format="ndjson",  # Set NDJSON format
    )
    generic_logger.async_httpx_client.post = mock_post
    litellm.callbacks = [generic_logger]

    # Make multiple completion calls to generate multiple logs
    for i in range(3):
        response = await _await_with_spawned_tasks(
            litellm.acompletion(
                model="gpt-5.5",
                messages=[{"role": "user", "content": f"Hello, world! {i}"}],
                mock_response="hi",
                user="test_user",
            )
        )

    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
    await generic_logger.flush_queue()

    # Assert httpx post was called
    mock_post.assert_called_once()

    # Get the actual request body from the mock
    actual_url = mock_post.call_args[1]["url"]
    assert actual_url == test_endpoint, f"Expected URL {test_endpoint}, got {actual_url}"

    # Get the data sent
    ndjson_data = mock_post.call_args[1]["data"]
    print("##########\n")
    print("ndjson_data:", ndjson_data)
    print("##########\n")

    # Validate it's NDJSON format (newline-delimited)
    assert isinstance(ndjson_data, str), "Data should be a string for NDJSON"

    # Split by newlines and parse each line
    lines = ndjson_data.strip().split("\n")
    assert len(lines) == 3, f"Expected 3 lines of NDJSON, got {len(lines)}"

    # Validate each line is valid JSON
    for i, line in enumerate(lines):
        payload_item = json.loads(line)
        payload_item = StandardLoggingPayload(**payload_item)

        # Basic assertions
        assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
        assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"
        assert payload_item["model_parameters"]["user"] == "test_user", "User should be test_user"


@pytest.mark.asyncio
async def test_generic_api_callback_single_format():
    """
    Test the GenericAPILogger callback with single log format.
    Validates that each log is sent as an individual request in parallel.
    """
    # Create a mock for the async_httpx_client's post method
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    # Set up an endpoint for testing
    test_endpoint = "https://example.com/api/logs"
    test_headers = {"Authorization": "Bearer test_token"}
    os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint

    # Initialize the GenericAPILogger with single format
    generic_logger = GenericAPILogger(
        endpoint=test_endpoint,
        headers=test_headers,
        flush_interval=3600,
        log_format="single",  # Set single format
    )
    generic_logger.async_httpx_client.post = mock_post
    litellm.callbacks = [generic_logger]

    # Make 3 completion calls
    for i in range(3):
        response = await _await_with_spawned_tasks(
            litellm.acompletion(
                model="gpt-5.5",
                messages=[{"role": "user", "content": f"Hello, world! {i}"}],
                mock_response="hi",
                user="test_user",
            )
        )

    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
    await generic_logger.flush_queue()

    # Assert httpx post was called 3 times (once per log in batch)
    assert mock_post.call_count == 3, f"Expected 3 calls, got {mock_post.call_count}"

    # Validate each call sent a single log object (not an array)
    for call_idx in range(3):
        call_args = mock_post.call_args_list[call_idx]
        json_data = call_args[1]["data"]

        print(f"########## Call {call_idx} ##########")
        print("json_data:", json_data)

        # Parse and validate - should be a single object, not an array
        actual_request = json.loads(json_data)
        assert isinstance(actual_request, dict), f"Call {call_idx}: Expected dict, got {type(actual_request)}"

        # Validate it's a valid StandardLoggingPayload
        payload_item = StandardLoggingPayload(**actual_request)
        assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
        assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"


@pytest.mark.asyncio
async def test_generic_api_callback_json_array_format_explicit():
    """
    Test the GenericAPILogger callback with explicit json_array format.
    Validates backward compatibility when explicitly set to json_array.
    """
    # Create a mock for the async_httpx_client's post method
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    # Set up an endpoint for testing
    test_endpoint = "https://example.com/api/logs"
    test_headers = {"Authorization": "Bearer test_token"}
    os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint

    # Initialize the GenericAPILogger with explicit json_array format
    generic_logger = GenericAPILogger(
        endpoint=test_endpoint,
        headers=test_headers,
        flush_interval=3600,
        log_format="json_array",  # Explicitly set json_array
    )
    generic_logger.async_httpx_client.post = mock_post
    litellm.callbacks = [generic_logger]

    # Make multiple completion calls
    for i in range(5):
        response = await _await_with_spawned_tasks(
            litellm.acompletion(
                model="gpt-5.5",
                messages=[{"role": "user", "content": f"Hello, world! {i}"}],
                mock_response="hi",
                user="test_user",
            )
        )

    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
    await generic_logger.flush_queue()

    # Assert httpx post was called once (batched)
    mock_post.assert_called_once()

    # Get the data and validate it's a JSON array
    json_data = mock_post.call_args[1]["data"]
    actual_request = json.loads(json_data)

    assert isinstance(actual_request, list), "Request body should be a list (JSON array)"
    assert len(actual_request) == 5, f"Expected 5 items, got {len(actual_request)}"

    # Validate each item
    for payload_item in actual_request:
        payload_item = StandardLoggingPayload(**payload_item)
        assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
        assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5"


@pytest.mark.asyncio
async def test_generic_api_callback_sumologic_uses_ndjson():
    """
    Test that the sumologic callback uses ndjson format by default
    when loaded from generic_api_compatible_callbacks.json
    """
    # Create a mock for the async_httpx_client's post method
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    # Set environment variable for sumologic
    os.environ["SUMOLOGIC_WEBHOOK_URL"] = "https://collectors.sumologic.com/receiver/v1/http/test123"

    # Initialize using callback_name (loads from JSON config)
    generic_logger = GenericAPILogger(callback_name="sumologic", flush_interval=3600)
    generic_logger.async_httpx_client.post = mock_post
    litellm.callbacks = [generic_logger]

    # Verify the logger has ndjson format
    assert generic_logger.log_format == "ndjson", "Sumologic should use ndjson format"

    # Make completion calls
    for i in range(2):
        await _await_with_spawned_tasks(
            litellm.acompletion(
                model="gpt-5.5",
                messages=[{"role": "user", "content": f"Test {i}"}],
                mock_response="response",
                user="test_user",
            )
        )

    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
    await generic_logger.flush_queue()

    # Assert httpx post was called
    mock_post.assert_called_once()

    # Verify NDJSON format
    ndjson_data = mock_post.call_args[1]["data"]
    assert isinstance(ndjson_data, str), "Data should be a string for NDJSON"

    lines = ndjson_data.strip().split("\n")
    records: Final = [json.loads(line) for line in lines]

    this_test_messages: Final = [[{"role": "user", "content": f"Test {i}"}] for i in range(2)]
    mine: Final = [record for record in records if record.get("messages") in this_test_messages]
    assert len(mine) == 2, f"Expected this test's 2 calls as NDJSON lines, got {len(mine)} of {len(records)}"


@pytest.mark.asyncio
async def test_generic_api_callback_sends_queued_events_on_its_own(monkeypatch: pytest.MonkeyPatch):
    mock_post = AsyncMock()
    mock_post.return_value.status_code = 200
    mock_post.return_value.text = "OK"

    test_endpoint = "https://example.com/api/logs"
    test_headers = {"Authorization": "Bearer test_token"}
    monkeypatch.setenv("GENERIC_LOGGER_ENDPOINT", test_endpoint)

    generic_logger = GenericAPILogger(endpoint=test_endpoint, headers=test_headers, flush_interval=0.05)
    generic_logger.async_httpx_client.post = mock_post
    monkeypatch.setattr(litellm, "callbacks", [generic_logger])

    for i in range(3):
        await litellm.acompletion(
            model="gpt-5.5",
            messages=[{"role": "user", "content": f"Hello, world! {i}"}],
            mock_response="hi",
            user="test_user",
        )

    async def _poll() -> None:
        while sum(len(json.loads(call.kwargs["data"])) for call in mock_post.call_args_list) < 3:
            await asyncio.sleep(0.01)

    await asyncio.wait_for(_poll(), timeout=10)

    posted_bodies: Final = [json.loads(call.kwargs["data"]) for call in mock_post.call_args_list]
    for body in posted_bodies:
        assert isinstance(body, list), "Each posted body should be a JSON array"
    posted_items: Final = sum(posted_bodies, [])
    contents: Final = sorted(item["messages"][0]["content"] for item in posted_items)
    assert contents == [f"Hello, world! {i}" for i in range(3)]


@pytest.mark.asyncio
async def test_generic_api_callback_invalid_log_format():
    """
    Test that invalid log_format values raise a ValueError
    """
    test_endpoint = "https://example.com/api/logs"
    os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint

    with pytest.raises(ValueError, match="Invalid log_format"):
        GenericAPILogger(
            endpoint=test_endpoint,
            log_format="invalid_format",  # type: ignore  # Intentionally invalid for testing
        )


@pytest.mark.asyncio
async def test_generic_api_callback_retries_timeout_then_succeeds():
    """
    Test that GenericAPILogger retries LiteLLM timeout errors when configured.
    """
    test_endpoint = "https://example.com/api/logs"
    generic_logger = GenericAPILogger(
        endpoint=test_endpoint,
        max_retries=1,
        retry_delay=0,
        timeout=0.2,
    )

    mock_post = AsyncMock()
    mock_post.side_effect = [
        litellm.Timeout(
            message="Connection timed out",
            model="default-model-name",
            llm_provider="litellm-httpx-handler",
        ),
        type("Response", (), {"status_code": 200})(),
    ]
    generic_logger.async_httpx_client.post = mock_post
    generic_logger.log_queue = [{"event": "timeout-retry"}]

    await generic_logger.async_send_batch()

    assert mock_post.call_count == 2
    first_call = mock_post.call_args_list[0][1]
    assert first_call["url"] == test_endpoint
    assert first_call["timeout"] == 0.2
    assert json.loads(first_call["data"]) == [{"event": "timeout-retry"}]


@pytest.mark.asyncio
async def test_generic_api_callback_retries_5xx_then_succeeds():
    """
    Test that GenericAPILogger retries transient HTTP 5xx errors when configured.
    """
    test_endpoint = "https://example.com/api/logs"
    generic_logger = GenericAPILogger(
        endpoint=test_endpoint,
        max_retries=1,
        retry_delay=0,
    )

    request = httpx.Request("POST", test_endpoint)
    response = httpx.Response(status_code=503, request=request)
    mock_post = AsyncMock()
    mock_post.side_effect = [
        httpx.HTTPStatusError(
            "Server error",
            request=request,
            response=response,
        ),
        type("Response", (), {"status_code": 200})(),
    ]
    generic_logger.async_httpx_client.post = mock_post
    generic_logger.log_queue = [{"event": "5xx-retry"}]

    await generic_logger.async_send_batch()

    assert mock_post.call_count == 2


@pytest.mark.asyncio
async def test_generic_api_callback_does_not_retry_4xx():
    """
    Test that GenericAPILogger does not retry non-transient HTTP 4xx errors.
    """
    test_endpoint = "https://example.com/api/logs"
    generic_logger = GenericAPILogger(
        endpoint=test_endpoint,
        max_retries=2,
        retry_delay=0,
    )

    request = httpx.Request("POST", test_endpoint)
    response = httpx.Response(status_code=401, request=request)
    mock_post = AsyncMock()
    mock_post.side_effect = httpx.HTTPStatusError(
        "Unauthorized",
        request=request,
        response=response,
    )
    generic_logger.async_httpx_client.post = mock_post
    generic_logger.log_queue = [{"event": "4xx-no-retry"}]

    await generic_logger.async_send_batch()

    mock_post.assert_called_once()


@pytest.fixture(autouse=True)
def _vcr_outcome_gate(request, vcr):
    install_live_call_probe(request, vcr)
    yield
    record_vcr_outcome(request, vcr)


@pytest_asyncio.fixture(loop_scope="function", autouse=True)
async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]:
    yield
    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)


LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0


@pytest.fixture(scope="function", autouse=True)
def isolate_litellm_state():
    """
    Per-function isolation fixture.

    Resets litellm state to the true defaults captured at conftest import time,
    then restores after the test. This prevents module-level mutations (e.g.
    `litellm.num_retries = 3` at the top of test_langfuse_e2e_test.py) from
    leaking across tests within the same xdist worker.
    """
    from litellm.litellm_core_utils import litellm_logging as ll_logging
    from litellm.proxy.management_helpers import audit_logs as ll_audit_logs

    if hasattr(litellm, "in_memory_llm_clients_cache"):
        litellm.in_memory_llm_clients_cache.flush_cache()
    ll_logging._in_memory_loggers.clear()
    ll_audit_logs._audit_log_callback_cache.clear()
    for attr in _LIST_ATTRS:
        if attr in _DEFAULTS:
            default = _DEFAULTS[attr]
            setattr(litellm, attr, default.copy() if isinstance(default, list) else default)
    for attr in _SCALAR_ATTRS:
        if attr in _DEFAULTS:
            setattr(litellm, attr, _DEFAULTS[attr])
    yield
    if hasattr(litellm, "in_memory_llm_clients_cache"):
        litellm.in_memory_llm_clients_cache.flush_cache()
    ll_logging._in_memory_loggers.clear()
    ll_audit_logs._audit_log_callback_cache.clear()
    for attr in _LIST_ATTRS:
        if attr in _DEFAULTS:
            default = _DEFAULTS[attr]
            setattr(litellm, attr, default.copy() if isinstance(default, list) else default)
    for attr in _SCALAR_ATTRS:
        if attr in _DEFAULTS:
            setattr(litellm, attr, _DEFAULTS[attr])


_LIST_ATTRS = (
    "callbacks",
    "success_callback",
    "failure_callback",
    "_async_success_callback",
    "_async_failure_callback",
    "service_callback",
    "pre_call_rules",
    "post_call_rules",
)

_SCALAR_ATTRS = (
    "set_verbose",
    "cache",
    "num_retries",
    "num_retries_per_request",
    "turn_off_message_logging",
    "redact_messages_in_exceptions",
    "redact_user_api_key_info",
    "s3_callback_params",
    "s3_audit_callback_params",
    "datadog_params",
    "vector_store_registry",
)

_DEFAULTS: dict = {}


@pytest.fixture(scope="module", autouse=True)
def setup_and_teardown():
    """
    Module-scoped setup. Reloads litellm only in single-process mode
    (skipped under xdist to avoid cross-worker interference).
    """
    import litellm

    worker_id = os.environ.get("PYTEST_XDIST_WORKER", None)
    if worker_id is None:
        importlib.reload(litellm)
        try:
            if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
                import litellm.proxy.proxy_server

                importlib.reload(litellm.proxy.proxy_server)
        except Exception as e:
            print(f"Error reloading litellm.proxy.proxy_server: {e}")
        if hasattr(litellm, "in_memory_llm_clients_cache"):
            litellm.in_memory_llm_clients_cache.flush_cache()
    yield


class _QueuedLog(TypedDict):
    id: ReadOnly[str]


_QUEUED_LOG: Final = TypeAdapter(_QueuedLog)
_LOG_BATCH: Final = TypeAdapter(tuple[_QueuedLog, ...])


def _batch_events(body: bytes, log_format: LOG_FORMAT_TYPES) -> tuple[_QueuedLog, ...]:
    match log_format:
        case "json_array":
            return _LOG_BATCH.validate_json(body)
        case "ndjson":
            return tuple(_QUEUED_LOG.validate_json(line) for line in body.splitlines())
        case "single":
            return (_QUEUED_LOG.validate_json(body),)


def _received_log_ids(requests: Sequence[httpx.Request], log_format: LOG_FORMAT_TYPES) -> Iterator[str]:
    for request in requests:
        for event in _batch_events(request.content, log_format):
            yield event["id"]


class _InProcessLogSink:
    def __init__(self, *, hold_first: bool = False) -> None:
        self.hold_first: Final = hold_first
        self.arrived: Final = asyncio.Event()
        self.release: Final = asyncio.Event()
        self.requests: tuple[httpx.Request, ...] = ()

    async def respond(self, request: httpx.Request) -> httpx.Response:
        self.requests = (*self.requests, request)
        if self.hold_first and len(self.requests) == 1:
            self.arrived.set()
            await self.release.wait()
        return httpx.Response(200)


@pytest.mark.asyncio
@pytest.mark.parametrize("log_format", ("json_array", "ndjson", "single"))
async def test_generic_api_keeps_failure_arriving_during_success_batch(log_format: LOG_FORMAT_TYPES) -> None:
    sink: Final = _InProcessLogSink(hold_first=True)
    handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(sink.respond))
    before_tasks: Final = frozenset(asyncio.all_tasks())
    logger: Final = GenericAPILogger(
        endpoint="https://owned-logger.invalid",
        async_httpx_client=handler,
        log_format=log_format,
        batch_size=1000,
        flush_interval=3600,
    )
    owned_tasks: Final = frozenset(asyncio.all_tasks()) - before_tasks
    try:
        async with asyncio.TaskGroup() as flushes:
            await logger.async_log_success_event({"standard_logging_object": {"id": "before"}}, None, None, None)
            first_flush: Final = flushes.create_task(logger.flush_queue())
            try:
                await sink.arrived.wait()
                await logger.async_log_failure_event({"standard_logging_object": {"id": "during"}}, None, None, None)
            finally:
                sink.release.set()
            await first_flush
            await logger.flush_queue()
            assert tuple(_received_log_ids(sink.requests, log_format)) == ("before", "during")
    finally:
        sink.release.set()
        for task in owned_tasks:
            task.cancel()
        await asyncio.gather(*owned_tasks, return_exceptions=True)
        await handler.close()


@pytest.mark.asyncio
@pytest.mark.parametrize("log_format", ("json_array", "ndjson", "single"))
async def test_generic_api_concurrent_flushes_deliver_each_event_once(log_format: LOG_FORMAT_TYPES) -> None:
    sink: Final = _InProcessLogSink(hold_first=True)
    handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(sink.respond))
    before_tasks: Final = frozenset(asyncio.all_tasks())
    logger: Final = GenericAPILogger(
        endpoint="https://owned-logger.invalid",
        async_httpx_client=handler,
        log_format=log_format,
        batch_size=1000,
        flush_interval=3600,
    )
    owned_tasks: Final = frozenset(asyncio.all_tasks()) - before_tasks
    try:
        await logger.async_log_success_event({"standard_logging_object": {"id": "once"}}, None, None, None)
        async with asyncio.TaskGroup() as flushes:
            flushes.create_task(logger.flush_queue())
            flushes.create_task(logger.async_send_batch())
            try:
                await sink.arrived.wait()
            finally:
                sink.release.set()
        assert tuple(_received_log_ids(sink.requests, log_format)) == ("once",)
    finally:
        sink.release.set()
        for task in owned_tasks:
            task.cancel()
        await asyncio.gather(*owned_tasks, return_exceptions=True)
        await handler.close()