Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
9.5 KB
Raw Blame
import os
from typing import Annotated, Any
from urllib.parse import urlparse

import boto3
from fastapi import Depends, File, Form, Request, UploadFile
from fastapi.responses import JSONResponse
from pydantic import BaseModel

from dependencies import (
    get_chat_service,
    get_current_user,
    get_docling_service,
    get_document_service,
    get_models_service,
    get_rbac_service,
    get_session_manager,
    get_task_service,
    has_effective_permission,
    require_all_permissions,
    require_permission,
)
from session_manager import User
from utils.logging_config import get_logger

logger = get_logger(__name__)


class UploadPathBody(BaseModel):
    path: str


class UploadBucketBody(BaseModel):
    s3_url: str
    # When True, objects whose filename (S3 key) already exists in the index
    # replace the indexed copy; when False (default) they are skipped.
    replace_duplicates: bool = False


async def upload(
    file: Annotated[UploadFile, File(...)],
    document_service: Annotated[Any, Depends(get_document_service)],
    session_manager: Annotated[Any, Depends(get_session_manager)],
    user: Annotated[User, Depends(require_permission("knowledge:upload"))],
):
    """Upload a single file"""
    try:
        from config.settings import is_no_auth_mode

        is_no_auth = is_no_auth_mode()
        owner_user_id = user.user_id if (user and not is_no_auth) else None
        owner_name = user.name if user else None
        owner_email = user.email if user else None

        result = await document_service.process_upload_file(
            file,
            owner_user_id=owner_user_id,
            jwt_token=user.jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
        )
        return JSONResponse(result, status_code=201)
    except Exception as e:
        error_msg = str(e)
        if "AuthenticationException" in error_msg or "access denied" in error_msg.lower():
            logger.warning("[INGEST] Upload rejected — access denied", error=error_msg)
            return JSONResponse({"error": error_msg}, status_code=403)
        else:
            logger.exception("[INGEST] Upload failed")
            return JSONResponse({"error": error_msg}, status_code=500)


async def upload_path(
    body: UploadPathBody,
    task_service: Annotated[Any, Depends(get_task_service)],
    session_manager: Annotated[Any, Depends(get_session_manager)],
    user: Annotated[User, Depends(require_permission("knowledge:upload"))],
):
    """Upload all files from a directory path"""
    if not body.path or not os.path.isdir(body.path):
        return JSONResponse({"error": "Invalid path"}, status_code=400)

    file_paths = [os.path.join(root, fn) for root, _, files in os.walk(body.path) for fn in files]

    if not file_paths:
        return JSONResponse({"error": "No files found in directory"}, status_code=400)

    jwt_token = user.jwt_token

    from config.settings import is_no_auth_mode

    is_no_auth = is_no_auth_mode()
    owner_user_id = user.user_id if (user and not is_no_auth) else None
    owner_name = user.name if user else None
    owner_email = user.email if user else None

    from api.documents import _ensure_index_exists

    try:
        await _ensure_index_exists(jwt_token)
    except RuntimeError:
        logger.exception("[INGEST] Index preflight check failed for upload_path")
        return JSONResponse({"error": "Index preflight check failed"}, status_code=500)

    task_id = await task_service.create_upload_task(
        owner_user_id,
        file_paths,
        jwt_token=jwt_token,
        owner_name=owner_name,
        owner_email=owner_email,
        # This endpoint never replaces (replace_duplicates stays False), so no
        # delete is attempted; False keeps it that way.
        allow_anonymous_delete=False,
    )

    return JSONResponse(
        {"task_id": task_id, "total_files": len(file_paths), "status": "accepted"},
        status_code=201,
    )


async def upload_context(
    file: Annotated[UploadFile, File(...)],
    document_service: Annotated[Any, Depends(get_document_service)],
    chat_service: Annotated[Any, Depends(get_chat_service)],
    session_manager: Annotated[Any, Depends(get_session_manager)],
    user: Annotated[User, Depends(require_all_permissions(("knowledge:upload", "chat:use")))],
    previous_response_id: Annotated[str | None, Form()] = None,
    endpoint: Annotated[str, Form()] = "langflow",
):
    """Upload a file and add its content as context to the current conversation"""
    filename = file.filename or "uploaded_document"
    user_id = user.user_id if user else None
    storage_user_id = (getattr(user, "db_user_id", None) or user.user_id) if user else None

    if previous_response_id and storage_user_id:
        from api.chat import _assert_owns

        await _assert_owns(previous_response_id, storage_user_id)

    jwt_token = user.jwt_token

    from services.docling_service import DoclingPayloadTooLarge
    from utils.upload_limits import UploadTooLarge

    try:
        doc_result = await document_service.process_upload_context(
            file, filename, user_id=user_id, jwt_token=jwt_token
        )
    except (UploadTooLarge, DoclingPayloadTooLarge) as exc:
        return JSONResponse({"error": str(exc)}, status_code=413)

    from config.settings import is_no_auth_mode

    is_no_auth = is_no_auth_mode()
    owner_user_id = user.user_id if (user and not is_no_auth) else None
    owner_name = user.name if user else None
    owner_email = user.email if user else None

    response_text, response_id = await chat_service.upload_context_chat(
        doc_result["content"],
        filename,
        user_id=user_id,
        jwt_token=jwt_token,
        previous_response_id=previous_response_id,
        endpoint=endpoint,
        owner=owner_user_id,
        owner_name=owner_name,
        owner_email=owner_email,
        storage_user_id=storage_user_id,
    )

    response_data = {
        "status": "context_added",
        "filename": doc_result["filename"],
        "pages": doc_result["pages"],
        "content_length": doc_result["content_length"],
        "response_id": response_id,
        "confirmation": response_text,
    }

    return JSONResponse(response_data)


async def upload_options(
    user: Annotated[User, Depends(get_current_user)],
):
    """Return availability of upload features"""
    aws_enabled = bool(os.getenv("AWS_ACCESS_KEY_ID") and os.getenv("AWS_SECRET_ACCESS_KEY"))
    from config.settings import UPLOAD_BATCH_SIZE
    from utils.upload_limits import max_upload_mb

    return JSONResponse(
        {
            "aws": aws_enabled,
            "upload_batch_size": UPLOAD_BATCH_SIZE,
            "max_upload_mb": max_upload_mb(),
        }
    )


async def upload_bucket(
    body: UploadBucketBody,
    request: Request,
    task_service: Annotated[Any, Depends(get_task_service)],
    models_service: Annotated[Any, Depends(get_models_service)],
    docling_service: Annotated[Any, Depends(get_docling_service)],
    session_manager: Annotated[Any, Depends(get_session_manager)],
    user: Annotated[User, Depends(require_permission("knowledge:upload"))],
    rbac: Annotated[Any, Depends(get_rbac_service)],
):
    """Process all files from an S3 bucket URL"""
    if not os.getenv("AWS_ACCESS_KEY_ID") or not os.getenv("AWS_SECRET_ACCESS_KEY"):
        return JSONResponse({"error": "AWS credentials not configured"}, status_code=400)

    if not body.s3_url or not body.s3_url.startswith("s3://"):
        return JSONResponse({"error": "Invalid S3 URL"}, status_code=400)

    parsed = urlparse(body.s3_url)
    bucket = parsed.netloc
    prefix = parsed.path.lstrip("/")

    s3_client = boto3.client("s3")
    keys = []
    paginator = s3_client.get_paginator("list_objects_v2")
    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
        for obj in page.get("Contents", []):
            key = obj["Key"]
            if not key.endswith("/"):
                keys.append(key)

    if not keys:
        return JSONResponse({"error": "No files found in bucket"}, status_code=400)

    jwt_token = user.jwt_token

    from config.settings import is_no_auth_mode
    from models.processors import S3FileProcessor

    is_no_auth = is_no_auth_mode()
    owner_user_id = user.user_id if (user and not is_no_auth) else None
    owner_name = user.name if user else None
    owner_email = user.email if user else None
    task_user_id = user.user_id if (user and not is_no_auth) else None

    from api.documents import _ensure_index_exists

    try:
        await _ensure_index_exists(jwt_token)
    except RuntimeError:
        logger.exception("[INGEST] Index preflight check failed for upload_bucket")
        return JSONResponse({"error": "Index preflight check failed"}, status_code=500)

    processor = S3FileProcessor(
        task_service.document_service,
        bucket,
        models_service=models_service,
        docling_service=docling_service,
        s3_client=s3_client,
        owner_user_id=owner_user_id,
        jwt_token=jwt_token,
        owner_name=owner_name,
        owner_email=owner_email,
        replace_duplicates=body.replace_duplicates,
        # An S3 key that matches a shared (ownerless) document replaces it for
        # the whole instance, so that takes the permission to delete one.
        allow_anonymous_delete=await has_effective_permission(
            request, user, rbac, "knowledge:delete:anonymous"
        ),
    )

    task_id = await task_service.create_custom_task(task_user_id, keys, processor)

    return JSONResponse(
        {"task_id": task_id, "total_files": len(keys), "status": "accepted"},
        status_code=201,
    )