Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

18.8 KB
Raw Blame
import asyncio
import time
from collections.abc import AsyncGenerator, Awaitable, Callable
from dataclasses import dataclass
from typing import Any

import structlog
from fastapi import HTTPException
from fastapi.responses import StreamingResponse

from llm_gateway.auth.models import AuthenticatedUser
from llm_gateway.config import get_settings
from llm_gateway.metrics.prometheus import (
    ACTIVE_STREAMS,
    CONCURRENT_REQUESTS,
    LLM_TIME_TO_FIRST_TOKEN,
    PROVIDER_ERRORS,
    REQUEST_COUNT,
    REQUEST_LATENCY,
    STREAMING_CLIENT_DISCONNECT,
)
from llm_gateway.observability import capture_exception
from llm_gateway.request_context import (
    rebuild_request_context,
    set_auth_user,
    set_effort,
    set_time_to_first_token,
)
from llm_gateway.streaming.sse import format_sse_stream

logger = structlog.get_logger(__name__)


def _clean_effort(value: Any) -> str | None:
    if isinstance(value, str) and (stripped := value.strip()):
        return stripped
    return None


def effort_from_output_config(request_data: dict[str, Any]) -> str | None:
    """Anthropic Messages: ``output_config: {"effort": "..."}``."""
    output_config = request_data.get("output_config")
    return _clean_effort(output_config.get("effort")) if isinstance(output_config, dict) else None


def effort_from_reasoning_effort(request_data: dict[str, Any]) -> str | None:
    """OpenAI chat completions: top-level ``reasoning_effort``."""
    return _clean_effort(request_data.get("reasoning_effort"))


def effort_from_reasoning(request_data: dict[str, Any]) -> str | None:
    """OpenAI Responses: ``reasoning: {"effort": "..."}``."""
    reasoning = request_data.get("reasoning")
    return _clean_effort(reasoning.get("effort")) if isinstance(reasoning, dict) else None


def no_effort(request_data: dict[str, Any]) -> str | None:
    """Endpoints with no reasoning-effort parameter (e.g. transcription)."""
    return None


@dataclass
class ProviderConfig:
    name: str
    endpoint_name: str
    # Where reasoning effort lives varies per API surface. Required (no default) so a new
    # provider can't be added without deciding; see the effort_from_* functions above.
    extract_effort: Callable[[dict[str, Any]], str | None]


ANTHROPIC_CONFIG = ProviderConfig(
    name="anthropic",
    endpoint_name="anthropic_messages",
    extract_effort=effort_from_output_config,
)
BEDROCK_CONFIG = ProviderConfig(
    name="bedrock",
    endpoint_name="bedrock_messages",
    extract_effort=effort_from_output_config,
)
OPENAI_CONFIG = ProviderConfig(
    name="openai",
    endpoint_name="chat_completions",
    extract_effort=effort_from_reasoning_effort,
)
OPENAI_RESPONSES_CONFIG = ProviderConfig(
    name="openai",
    endpoint_name="responses",
    extract_effort=effort_from_reasoning,
)
OPENAI_TRANSCRIPTION_CONFIG = ProviderConfig(
    name="openai",
    endpoint_name="audio_transcriptions",
    extract_effort=no_effort,
)
# Split endpoint labels so an adapter-specific regression is distinguishable in metrics.
CLOUDFLARE_ANTHROPIC_CONFIG = ProviderConfig(
    name="cloudflare",
    endpoint_name="cloudflare_anthropic_messages",
    extract_effort=effort_from_output_config,
)
CLOUDFLARE_OPENAI_CONFIG = ProviderConfig(
    name="cloudflare",
    endpoint_name="cloudflare_chat_completions",
    extract_effort=effort_from_reasoning_effort,
)
CLOUDFLARE_OPENAI_RESPONSES_CONFIG = ProviderConfig(
    name="cloudflare",
    endpoint_name="cloudflare_responses",
    extract_effort=effort_from_reasoning,
)
# Modal-served GLM (OpenAI-compatible vLLM endpoint). Same per-surface split as Cloudflare so a
# Modal-specific regression is distinguishable in metrics during the migration.
MODAL_ANTHROPIC_CONFIG = ProviderConfig(
    name="modal",
    endpoint_name="modal_anthropic_messages",
    extract_effort=effort_from_output_config,
)
MODAL_OPENAI_CONFIG = ProviderConfig(
    name="modal",
    endpoint_name="modal_chat_completions",
    extract_effort=effort_from_reasoning_effort,
)
MODAL_OPENAI_RESPONSES_CONFIG = ProviderConfig(
    name="modal",
    endpoint_name="modal_responses",
    extract_effort=effort_from_reasoning,
)
BASETEN_ANTHROPIC_CONFIG = ProviderConfig(
    name="baseten",
    endpoint_name="baseten_anthropic_messages",
    extract_effort=effort_from_output_config,
)
BASETEN_OPENAI_CONFIG = ProviderConfig(
    name="baseten",
    endpoint_name="baseten_chat_completions",
    extract_effort=effort_from_reasoning_effort,
)
BASETEN_OPENAI_RESPONSES_CONFIG = ProviderConfig(
    name="baseten",
    endpoint_name="baseten_responses",
    extract_effort=effort_from_reasoning,
)

_KNOWN_LITELLM_PROVIDER_PREFIXES = (
    "anthropic/",
    "bedrock/",
    "fireworks_ai/",
    "openai/",
    "openrouter/",
)


def normalize_litellm_model_name(model: str, provider: str) -> str:
    """Add an explicit LiteLLM provider prefix when a provider endpoint receives a bare model ID."""
    if model.startswith(_KNOWN_LITELLM_PROVIDER_PREFIXES):
        return model
    return f"{provider}/{model}"


# Block model prefixes that would route to a provider we don't call via the generic path:
# - Google needs litellm[google] (not installed) — would crash in vertex_llm_base with ImportError.
# - Native `cloudflare/...` bypasses per-call credential injection and the CLOUDFLARE_ALLOWED_MODELS
#   allowlist the `@cf/...` path enforces — reject so callers can't smuggle models onto gateway creds.
# Matches explicit prefixes (gemini/, vertex_ai/) and bare gemini- names litellm routes to vertex/gemini,
# which may not be in the cost registry yet when brand new.
_UNSUPPORTED_PROVIDERS = frozenset({"vertex_ai", "vertex_ai-language-models", "gemini", "cloudflare"})
_UNSUPPORTED_MODEL_PREFIXES = (
    *(f"{p}/" for p in _UNSUPPORTED_PROVIDERS),
    "gemini-",
)


class ProviderError(HTTPException):
    """An HTTPException raised from the upstream provider call itself, as opposed to gateway-local
    validation (unsupported model, bad headers, timeouts). Lets downstream handlers tell a genuine
    provider failure apart from a gateway 400 that merely echoes caller-controlled input — e.g. the
    Anthropic billing-block detector must not key off an unsupported-model message containing the
    caller's model name. Subclasses HTTPException so it serializes to the client identically.
    """


def _raise_unsupported_model(model: str) -> None:
    raise HTTPException(
        status_code=400,
        detail={
            "error": {
                "message": f"Model '{model}' is not supported by this gateway",
                "type": "invalid_request_error",
                "code": "model_not_supported",
            }
        },
    )


def _raise_if_unsupported_model(model: str) -> None:
    from llm_gateway.services.model_registry import ModelRegistryService

    if model.lower().startswith(_UNSUPPORTED_MODEL_PREFIXES):
        _raise_unsupported_model(model)
    info = ModelRegistryService.get_instance().get_model(model)
    if info is not None and info.provider in _UNSUPPORTED_PROVIDERS:
        _raise_unsupported_model(model)


# LLM routing/auth params — never accept from user input (request redirection, key exfiltration).
FORBIDDEN_REQUEST_PARAMS = frozenset(
    {"api_key", "api_base", "base_url", "api_version", "organization", "model_list", "fallbacks", "custom_llm_provider"}
)


def _sanitize_request_value(value: Any) -> Any:
    # Strip recursively: litellm forwards nested params (e.g. model_list[*].litellm_params.api_key)
    # to the provider, so a shallow filter is insufficient.
    if isinstance(value, dict):
        return {k: _sanitize_request_value(v) for k, v in value.items() if k not in FORBIDDEN_REQUEST_PARAMS}
    if isinstance(value, list):
        return [_sanitize_request_value(item) for item in value]
    return value


def _sanitize_request_data(data: dict[str, Any]) -> dict[str, Any]:
    return {k: _sanitize_request_value(v) for k, v in data.items() if k not in FORBIDDEN_REQUEST_PARAMS}


async def handle_llm_request(
    request_data: dict[str, Any],
    user: AuthenticatedUser,
    model: str,
    is_streaming: bool,
    provider_config: ProviderConfig,
    llm_call: Callable[..., Awaitable[Any]],
    product: str = "llm_gateway",
) -> dict[str, Any] | StreamingResponse:
    _raise_if_unsupported_model(model)
    request_data = _sanitize_request_data(request_data)
    settings = get_settings()
    start_time = time.monotonic()

    rebuild_request_context(product)
    set_auth_user(user)

    # Stash effort for the PostHog callback to stamp on the $ai_generation event (mirrors
    # time_to_first_token). Set unconditionally so a stale value can't leak if the context
    # is reused.
    set_effort(provider_config.extract_effort(request_data))

    structlog.contextvars.bind_contextvars(
        user_id=user.user_id,
        team_id=user.team_id,
        provider=provider_config.name,
        model=model,
    )

    if is_streaming:
        return await _handle_streaming_request(
            request_data=request_data,
            user=user,
            model=model,
            provider_config=provider_config,
            llm_call=llm_call,
            start_time=start_time,
            timeout=settings.streaming_timeout,
            product=product,
        )

    CONCURRENT_REQUESTS.labels(provider=provider_config.name, model=model, product=product).inc()
    try:
        return await _handle_non_streaming_request(
            request_data=request_data,
            user=user,
            model=model,
            provider_config=provider_config,
            llm_call=llm_call,
            start_time=start_time,
            timeout=settings.request_timeout,
            product=product,
        )

    except TimeoutError:
        PROVIDER_ERRORS.labels(provider=provider_config.name, error_type="timeout", product=product).inc()
        logger.error(
            "llm_request_timeout",
            endpoint=provider_config.endpoint_name,
            streaming=False,
        )
        raise HTTPException(
            status_code=504,
            detail={"error": {"message": "Request timed out", "type": "timeout_error", "code": None}},
        ) from None
    except HTTPException:
        raise
    except Exception as e:
        PROVIDER_ERRORS.labels(provider=provider_config.name, error_type=type(e).__name__, product=product).inc()
        capture_exception(e, {"provider": provider_config.name, "model": model, "user_id": user.user_id})
        status_code = getattr(e, "status_code", 500)
        logger.exception(
            "llm_request_failed",
            endpoint=provider_config.endpoint_name,
            streaming=False,
            status_code=status_code,
            error_type=type(e).__name__,
            error_message=getattr(e, "message", str(e)),
            provider_error_type=getattr(e, "type", None),
            provider_error_code=getattr(e, "code", None),
        )
        raise ProviderError(
            status_code=status_code,
            detail={
                "error": {
                    "message": getattr(e, "message", str(e)),
                    "type": getattr(e, "type", "internal_error"),
                    "code": getattr(e, "code", None),
                }
            },
        ) from e
    finally:
        CONCURRENT_REQUESTS.labels(provider=provider_config.name, model=model, product=product).dec()


async def _handle_streaming_request(
    request_data: dict[str, Any],
    user: AuthenticatedUser,
    model: str,
    provider_config: ProviderConfig,
    llm_call: Callable[..., Awaitable[Any]],
    start_time: float,
    timeout: float,
    product: str = "llm_gateway",
) -> StreamingResponse:
    CONCURRENT_REQUESTS.labels(provider=provider_config.name, model=model, product=product).inc()
    try:
        llm_response = await asyncio.wait_for(llm_call(**request_data), timeout=timeout)
    except TimeoutError:
        CONCURRENT_REQUESTS.labels(provider=provider_config.name, model=model, product=product).dec()
        PROVIDER_ERRORS.labels(provider=provider_config.name, error_type="timeout", product=product).inc()
        REQUEST_COUNT.labels(
            endpoint=provider_config.endpoint_name,
            provider=provider_config.name,
            model=model,
            status_code="504",
            auth_method=user.auth_method,
            product=product,
        ).inc()
        REQUEST_LATENCY.labels(
            endpoint=provider_config.endpoint_name,
            provider=provider_config.name,
            streaming="true",
            product=product,
        ).observe(time.monotonic() - start_time)
        logger.error(
            "llm_request_timeout",
            endpoint=provider_config.endpoint_name,
            streaming=True,
        )
        raise HTTPException(
            status_code=504,
            detail={"error": {"message": "Request timed out", "type": "timeout_error", "code": None}},
        ) from None
    except Exception as e:
        CONCURRENT_REQUESTS.labels(provider=provider_config.name, model=model, product=product).dec()
        PROVIDER_ERRORS.labels(provider=provider_config.name, error_type=type(e).__name__, product=product).inc()
        capture_exception(e, {"provider": provider_config.name, "model": model, "streaming": True})
        status_code = getattr(e, "status_code", 500)
        logger.exception(
            "llm_request_failed",
            endpoint=provider_config.endpoint_name,
            streaming=True,
            status_code=status_code,
            error_type=type(e).__name__,
            error_message=getattr(e, "message", str(e)),
            provider_error_type=getattr(e, "type", None),
            provider_error_code=getattr(e, "code", None),
        )
        REQUEST_COUNT.labels(
            endpoint=provider_config.endpoint_name,
            provider=provider_config.name,
            model=model,
            status_code=str(status_code),
            auth_method=user.auth_method,
            product=product,
        ).inc()
        REQUEST_LATENCY.labels(
            endpoint=provider_config.endpoint_name,
            provider=provider_config.name,
            streaming="true",
            product=product,
        ).observe(time.monotonic() - start_time)
        raise ProviderError(
            status_code=status_code,
            detail={
                "error": {
                    "message": getattr(e, "message", str(e)),
                    "type": getattr(e, "type", "internal_error"),
                    "code": getattr(e, "code", None),
                }
            },
        ) from e

    async def stream_generator() -> AsyncGenerator[bytes]:
        ACTIVE_STREAMS.labels(provider=provider_config.name, model=model, product=product).inc()
        status_code = "200"
        provider_start = time.monotonic()
        first_chunk_received = False

        try:
            async for chunk in format_sse_stream(llm_response):
                if not first_chunk_received:
                    first_chunk_received = True
                    time_to_first = time.monotonic() - provider_start
                    LLM_TIME_TO_FIRST_TOKEN.labels(provider=provider_config.name, model=model, product=product).observe(
                        time_to_first
                    )
                    set_time_to_first_token(time_to_first)
                yield chunk

        except asyncio.CancelledError:
            STREAMING_CLIENT_DISCONNECT.labels(provider=provider_config.name, model=model, product=product).inc()
            raise
        except TimeoutError:
            status_code = "504"
            PROVIDER_ERRORS.labels(provider=provider_config.name, error_type="timeout", product=product).inc()
            logger.error(
                "stream_chunk_timeout",
                endpoint=provider_config.endpoint_name,
            )
            raise
        except Exception as e:
            status_code = str(getattr(e, "status_code", 500))
            PROVIDER_ERRORS.labels(provider=provider_config.name, error_type=type(e).__name__, product=product).inc()
            capture_exception(e, {"provider": provider_config.name, "model": model, "streaming": True})
            logger.exception(
                "stream_chunk_failed",
                endpoint=provider_config.endpoint_name,
                status_code=status_code,
                error_type=type(e).__name__,
                error_message=str(e),
            )
            raise
        finally:
            duration_ms = round((time.monotonic() - start_time) * 1000, 2)
            if status_code == "200":
                logger.info(
                    "llm_request_completed",
                    endpoint=provider_config.endpoint_name,
                    streaming=True,
                    duration_ms=duration_ms,
                )
            ACTIVE_STREAMS.labels(provider=provider_config.name, model=model, product=product).dec()
            CONCURRENT_REQUESTS.labels(provider=provider_config.name, model=model, product=product).dec()
            REQUEST_COUNT.labels(
                endpoint=provider_config.endpoint_name,
                provider=provider_config.name,
                model=model,
                status_code=status_code,
                auth_method=user.auth_method,
                product=product,
            ).inc()
            REQUEST_LATENCY.labels(
                endpoint=provider_config.endpoint_name,
                provider=provider_config.name,
                streaming="true",
                product=product,
            ).observe(time.monotonic() - start_time)

    return StreamingResponse(
        stream_generator(),
        media_type="text/event-stream",
        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
    )


async def _handle_non_streaming_request(
    request_data: dict[str, Any],
    user: AuthenticatedUser,
    model: str,
    provider_config: ProviderConfig,
    llm_call: Callable[..., Awaitable[Any]],
    start_time: float,
    timeout: float,
    product: str = "llm_gateway",
) -> dict[str, Any]:
    status_code = "200"
    try:
        response = await asyncio.wait_for(llm_call(**request_data), timeout=timeout)
        response_dict = response.model_dump() if hasattr(response, "model_dump") else response
        duration_ms = round((time.monotonic() - start_time) * 1000, 2)
        logger.info(
            "llm_request_completed",
            endpoint=provider_config.endpoint_name,
            streaming=False,
            duration_ms=duration_ms,
        )
        return response_dict
    except TimeoutError:
        status_code = "504"
        raise
    except Exception as e:
        status_code = str(getattr(e, "status_code", 500))
        raise
    finally:
        REQUEST_COUNT.labels(
            endpoint=provider_config.endpoint_name,
            provider=provider_config.name,
            model=model,
            status_code=status_code,
            auth_method=user.auth_method,
            product=product,
        ).inc()
        REQUEST_LATENCY.labels(
            endpoint=provider_config.endpoint_name,
            provider=provider_config.name,
            streaming="false",
            product=product,
        ).observe(time.monotonic() - start_time)