Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

3.3 KB
Raw Blame
import uuid
from collections.abc import Callable, Iterator, Mapping
from pathlib import Path
from typing import Final

import pytest
from integration._support.client import Gateway, eventually, gateway_from_environment, object_value
from integration._support.otlp_sink import Span, SpanSinks, recorded_spans
from integration._support.process import owned_proxy
from pydantic import JsonValue

pytestmark: Final = pytest.mark.timeout(180)

AuditConfigWriter = Callable[[Path, Mapping[str, JsonValue]], Path]


@pytest.fixture(scope="module")
def gateway(audit_sinks: SpanSinks) -> Iterator[Gateway]:
    with gateway_from_environment() as base:
        yield base


def _traces(spans: tuple[Span, ...]) -> dict[str, frozenset[str]]:
    trace_ids: Final = {span["trace_id"] for span in spans}
    return {trace: frozenset(span["name"] for span in spans if span["trace_id"] == trace) for trace in trace_ids}


def test_default_otel_logger_keeps_spend_flush_outside_the_request_trace(
    gateway: Gateway, audit_sinks: SpanSinks, otel_audit_config: AuditConfigWriter, tmp_path: Path
) -> None:
    config: Final = otel_audit_config(tmp_path, {})
    overrides: Final = {"OTEL_EXPORTER": "http/json", "OTEL_ENDPOINT": audit_sinks.operator}
    with owned_proxy(gateway, tmp_path, overrides, config=config) as candidate, candidate.scenario() as scenario:
        model: Final = scenario.model()
        key: Final = scenario.key(models=[model])
        start, _ = recorded_spans(audit_sinks.operator)
        response: Final = candidate.request(
            "POST",
            "/v1/chat/completions",
            {"model": model, "messages": [{"role": "user", "content": f"otel v1 {uuid.uuid4().hex}"}]},
            key=key,
        )
        assert response.status_code == 200, response.text
        expected: Final = frozenset({"postgres", "redis", "raw_gen_ai_request"})
        traces: Final = eventually(
            lambda: _traces(recorded_spans(audit_sinks.operator, start)[1]),
            lambda grouped: any(expected <= names for names in grouped.values()),
            seconds=60,
            return_last_on_timeout=True,
        )
        assert any(expected <= names for names in traces.values()), {
            trace: sorted(names) for trace, names in traces.items()
        }
        request_trace: Final = next(trace for trace, names in traces.items() if expected <= names)
        key_info: Final = eventually(
            lambda: candidate.request("GET", "/key/info", key=key, params={"key": key}),
            lambda response: response.status_code == 200
            and float(str(object_value(response.json()["info"])["spend"])) > 0,
            seconds=60,
        )
        assert key_info.status_code == 200, key_info.text
        assert float(str(object_value(key_info.json()["info"])["spend"])) > 0
        request_spans: Final = tuple(
            span for span in recorded_spans(audit_sinks.operator, start)[1] if span["trace_id"] == request_trace
        )
        assert not any(span["name"] == "batch_write_to_db" for span in request_spans), request_spans
        assert not any(
            span["name"] == "postgres"
            and span["attributes"].get("call_type") == "commit_spend_updates"
            and span["attributes"].get("table_name") == "LiteLLM_VerificationToken"
            for span in request_spans
        ), request_spans