Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
6.9 KB
Raw Blame
"""OpenAI-compatible LLM proxy for Langflow and other OpenAI clients.

Auth is a short-lived Langflow hop token (same family as ingest tokens),
the user JWT, or an `orag_` API key. Provider secrets stay in OpenRAG.
"""

from typing import Any

from fastapi import Depends, Request
from fastapi.responses import JSONResponse, StreamingResponse

from dependencies import require_llm_proxy_any_permission, require_llm_proxy_permission
from observability.tracing import trace_context
from services.langflow_llm_token_service import HOP_PURPOSE_CHAT
from services.llm_gateway import LlmGatewayError, chat_completions, embeddings
from services.model_catalog import (
    CATALOG_UNAVAILABLE_MESSAGE,
    CatalogUnavailableError,
    catalog,
    openai_models_list,
    refresh_live_models,
)
from session_manager import User
from utils.logging_config import get_logger

logger = get_logger(__name__)

_catalog_read = require_llm_proxy_any_permission(("providers:read", "chat:use", "knowledge:upload"))
_embeddings_use = require_llm_proxy_any_permission(("chat:use", "knowledge:upload"))
_chat_use = require_llm_proxy_permission("chat:use")


def _openai_error(
    message: str,
    status_code: int,
    error_type: str = "invalid_request_error",
    code: str | None = None,
):
    """Build an OpenAI error response, including a code only when supplied."""
    error = {"message": message, "type": error_type}
    if code:
        error["code"] = code
    return JSONResponse({"error": error}, status_code=status_code)


def _gateway_error(exc: LlmGatewayError):
    """Log internal gateway details and return the caller-safe error and code."""
    error_type = "invalid_request_error" if exc.status_code < 500 else "api_error"
    if exc.detail != exc.message:
        logger.error("LLM gateway error", status_code=exc.status_code, error=exc.detail)
    return _openai_error(exc.message, exc.status_code, error_type, exc.code)


async def list_openai_models_endpoint(
    user: User = Depends(_catalog_read),
):
    """OpenAI-compatible model inventory. GET /v1/models"""
    try:
        # A cluster-hosted provider is the only source for its own model list.
        await refresh_live_models()
        return JSONResponse(openai_models_list())
    except CatalogUnavailableError as exc:
        # Log the cause, but never echo exception text back to the caller
        # (CodeQL py/stack-trace-exposure).
        logger.error("Model catalogue unavailable", error=str(exc))
        return _openai_error(CATALOG_UNAVAILABLE_MESSAGE, 503, "api_error")


async def model_catalog_endpoint(
    user: User = Depends(_catalog_read),
):
    """LiteLLM picker payload. GET /v1/model-catalog"""
    # `no-store` for the same reason as /models/providers: the catalogue is
    # filtered by that config file, so a cached copy survives the restart that
    # was supposed to apply the edit. LiteLLM's table is cached in-process, so
    # rebuilding this response is cheap.
    try:
        await refresh_live_models()
        return JSONResponse(catalog(), headers={"Cache-Control": "no-store"})
    except CatalogUnavailableError as exc:
        logger.error("Model catalogue unavailable", error=str(exc))
        return JSONResponse({"error": CATALOG_UNAVAILABLE_MESSAGE}, status_code=503)


async def model_providers_endpoint(
    user: User = Depends(_catalog_read),
):
    """Providers this run mode exposes. GET /v1/model-providers

    Same list the console reads from /models/providers, so an SDK client and the
    UI never disagree about which providers a deployment offers.
    """
    from config.model_providers import provider_visibility_payload

    # `no-store`, not a max-age: the list is derived from
    # `config/model_providers.yaml`, which an operator edits and then restarts
    # the backend for. A cached response outlives that restart in the browser,
    # so the console keeps drawing the old provider cards/tabs and the change
    # looks like it did nothing. The payload is a few hundred bytes and React
    # Query already holds it for the session, so the round trip costs nothing.
    return JSONResponse(
        provider_visibility_payload(),
        headers={"Cache-Control": "no-store"},
    )


async def _read_json_body(request: Request) -> dict[str, Any] | JSONResponse:
    try:
        body = await request.json()
    except Exception:
        return _openai_error("Request body must be valid JSON", 400)
    if not isinstance(body, dict):
        return _openai_error("Request body must be a JSON object", 400)
    return body


def _trace_caller(request: Request, user: User, *, api_purpose: str | None = None):
    """Label the gateway calls this request makes: Langflow run or API caller.

    ``api_purpose`` overrides the gateway's purpose for API callers only;
    Langflow runs keep the gateway's (their hop token says chat or ingest).
    """
    run_id = getattr(request.state, "llm_hop_run_id", None)
    if not run_id:
        return trace_context(source="api", user_id=user.user_id, purpose=api_purpose)
    hop_purpose = getattr(request.state, "llm_hop_purpose", None)
    return trace_context(
        source="langflow",
        user_id=user.user_id,
        session_id=run_id,
        attributes={"openrag.langflow_run_purpose": hop_purpose} if hop_purpose else None,
    )


async def chat_completions_endpoint(
    request: Request,
    user: User = Depends(_chat_use),
):
    """OpenAI-compatible chat completions. POST /v1/chat/completions"""
    body = await _read_json_body(request)
    if isinstance(body, JSONResponse):
        return body
    try:
        # The provider call starts here even when streaming, so the label
        # covers it; the stream is only read after the block exits.
        with _trace_caller(request, user):
            result = await chat_completions(body)
    except LlmGatewayError as exc:
        return _gateway_error(exc)
    if body.get("stream"):
        return StreamingResponse(
            result,
            media_type="text/event-stream",
            headers={
                "Cache-Control": "no-cache",
                "Connection": "keep-alive",
                "X-Accel-Buffering": "no",
            },
        )
    return JSONResponse(result)


async def embeddings_endpoint(
    request: Request,
    user: User = Depends(_embeddings_use),
):
    """OpenAI-compatible embeddings. POST /v1/embeddings"""
    body = await _read_json_body(request)
    if isinstance(body, JSONResponse):
        return body
    # A chat run's hop token marks query embeddings, which must not queue
    # behind a bulk ingest at a concurrency-limited provider.
    interactive = getattr(request.state, "llm_hop_purpose", None) == HOP_PURPOSE_CHAT
    try:
        # A non-interactive call here isn't necessarily ingestion: the gateway
        # would label an API caller's embeddings `ingest`.
        with _trace_caller(request, user, api_purpose="api"):
            result = await embeddings(body, interactive=interactive)
    except LlmGatewayError as exc:
        return _gateway_error(exc)
    return JSONResponse(result)