Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
17.8 KB
Raw Blame
import json
import re
import uuid
from datetime import UTC, datetime
from typing import Any
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError

from fastapi import Depends, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel

from api.files import _validate_date_range
from dependencies import (
    get_knowledge_filter_service,
    get_monitor_service,
    get_session_manager,
    require_permission,
)
from session_manager import User
from utils.logging_config import get_logger

logger = get_logger(__name__)


def _error_response(result: dict) -> JSONResponse:
    """Map a failed knowledge-filter service result to the right HTTP status.

    Distinguishes an OpenSearch authentication failure (credential rejected ->
    401, re-authenticate) from authorization ("access denied"/owner check ->
    403). Conflating the two as 403 "insufficient permissions" hides the real
    cause. ``not found`` stays 404; everything else is 500.
    """
    from utils.opensearch_utils import is_opensearch_auth_error

    error_msg = result.get("error", "") or ""
    low = error_msg.lower()
    if "not found" in low or "already deleted" in low:
        return JSONResponse(result, status_code=404)
    if is_opensearch_auth_error(error_msg):
        return JSONResponse(result, status_code=401)
    if "access denied" in low or "insufficient permissions" in low:
        return JSONResponse(result, status_code=403)
    return JSONResponse(result, status_code=500)


_CALENDAR_DAY_RE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", re.ASCII)
_DATE_BOUND_KEYS = ("created_after", "created_before")
_DATE_CONTEXT_KEYS = ("from", "to", "timezone")


def _validate_calendar_day(value: Any, name: str) -> str:
    if not isinstance(value, str):
        raise ValueError(f"{name} must be a YYYY-MM-DD string")
    # Exact zero-padded form only: the range check compares these as strings.
    if not _CALENDAR_DAY_RE.fullmatch(value):
        raise ValueError(f"{name} must be a valid YYYY-MM-DD date")
    try:
        datetime.strptime(value, "%Y-%m-%d")
    except ValueError as err:
        raise ValueError(f"{name} must be a valid YYYY-MM-DD date") from err
    return value


def _normalize_date_range(data: dict, filters: dict) -> tuple[dict, dict | None]:
    """Validate the optional saved date bounds and their display context.

    Returns ``(bounds, context)``. ``bounds`` holds only the UTC bounds that are
    set; ``context`` is the calendar days + IANA timezone used to restore the
    picker, kept outside ``filters`` so it never reaches retrieval.
    Raises ValueError for any malformed value.
    """
    bounds: dict[str, str] = {}
    for key in _DATE_BOUND_KEYS:
        value = filters.get(key)
        if value is None or value == "":
            continue
        if not isinstance(value, str):
            raise ValueError(f"{key} must be an ISO 8601 string")
        bounds[key] = value
    try:
        _validate_date_range(bounds.get("created_after"), bounds.get("created_before"))
    except HTTPException as err:
        raise ValueError(str(err.detail)) from err

    raw_context = data.get("date_range_context")
    if raw_context is None:
        return bounds, None
    if not bounds:
        # Context without bounds would describe a range that is not applied.
        return bounds, None
    if not isinstance(raw_context, dict):
        raise ValueError("date_range_context must be an object")
    context: dict[str, str] = {}
    for key in ("from", "to"):
        if raw_context.get(key) is not None:
            context[key] = _validate_calendar_day(raw_context[key], f"date_range_context.{key}")
    if "from" in context and "to" in context and context["from"] > context["to"]:
        raise ValueError("date_range_context.from must be on or before date_range_context.to")
    tz = raw_context.get("timezone")
    if tz is not None:
        try:
            if not isinstance(tz, str) or not tz:
                raise ValueError
            ZoneInfo(tz)
        except (ValueError, ZoneInfoNotFoundError, OSError) as err:
            raise ValueError("date_range_context.timezone must be a valid IANA timezone") from err
        context["timezone"] = tz
    return bounds, context or None


def normalize_query_data(query_data: str | dict) -> str:
    """
    Normalize query_data to ensure all required fields exist with defaults.
    This prevents frontend crashes when API-created filters have incomplete data.

    Optional ``created_after``/``created_before`` bounds are validated and kept
    inside ``filters``; filters saved without dates stay unrestricted.
    Raises ValueError for malformed date data.
    """
    if isinstance(query_data, str):
        try:
            data = json.loads(query_data)
        except json.JSONDecodeError:
            data = {}
    else:
        data = query_data or {}

    filters = data.get("filters") or {}
    normalized_filters = {
        "data_sources": filters.get("data_sources", ["*"]),
        "document_types": filters.get("document_types", ["*"]),
        "owners": filters.get("owners", ["*"]),
        "connector_types": filters.get("connector_types", ["*"]),
    }
    bounds, date_context = _normalize_date_range(data, filters)
    normalized_filters.update(bounds)

    normalized = {
        "query": data.get("query", ""),
        "filters": normalized_filters,
        "limit": data.get("limit", 10),
        "scoreThreshold": data.get("scoreThreshold", 0),
        "color": data.get("color", "zinc"),
        "icon": data.get("icon", "filter"),
    }
    if date_context:
        normalized["date_range_context"] = date_context

    return json.dumps(normalized)


class CreateFilterBody(BaseModel):
    name: str
    description: str = ""
    queryData: Any | None = None
    allowedUsers: list[str] = []
    allowedGroups: list[str] = []


class SearchFiltersBody(BaseModel):
    query: str = ""
    limit: int = 20


class UpdateFilterBody(BaseModel):
    name: str | None = None
    description: str | None = None
    queryData: Any | None = None
    allowedUsers: list[str] | None = None
    allowedGroups: list[str] | None = None


class SubscribeBody(BaseModel):
    notification_config: dict | None = None


async def create_knowledge_filter(
    body: CreateFilterBody,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:create")),
):
    """Create a new knowledge filter"""
    if not body.name:
        return JSONResponse({"error": "Knowledge filter name is required"}, status_code=400)

    if not body.queryData:
        return JSONResponse({"error": "Query data is required"}, status_code=400)

    try:
        normalized_query_data = normalize_query_data(body.queryData)
    except Exception:
        logger.error("Failed to normalize query_data", exc_info=True)
        return JSONResponse({"error": "Invalid queryData format"}, status_code=400)

    jwt_token = user.jwt_token

    filter_id = str(uuid.uuid4())
    filter_doc = {
        "id": filter_id,
        "name": body.name,
        "description": body.description,
        "query_data": normalized_query_data,
        "owner": user.user_id,
        "allowed_users": body.allowedUsers,
        "allowed_groups": body.allowedGroups,
        "created_at": datetime.now(UTC).isoformat(),
        "updated_at": datetime.now(UTC).isoformat(),
    }

    result = await knowledge_filter_service.create_knowledge_filter(
        filter_doc, user_id=user.user_id, jwt_token=jwt_token
    )

    if result.get("success"):
        return JSONResponse(result, status_code=201)
    return _error_response(result)


async def search_knowledge_filters(
    body: SearchFiltersBody,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:read")),
):
    """Search for knowledge filters by name, description, or query content"""
    jwt_token = user.jwt_token
    result = await knowledge_filter_service.search_knowledge_filters(
        body.query, user_id=user.user_id, jwt_token=jwt_token, limit=body.limit
    )

    if result.get("success"):
        return JSONResponse(result, status_code=200)
    return _error_response(result)


async def get_knowledge_filter(
    filter_id: str,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:read")),
):
    """Get a specific knowledge filter by ID"""
    jwt_token = user.jwt_token
    result = await knowledge_filter_service.get_knowledge_filter(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )

    if result.get("success"):
        return JSONResponse(result, status_code=200)
    return _error_response(result)


async def update_knowledge_filter(
    filter_id: str,
    body: UpdateFilterBody,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:edit:own")),
):
    """Update an existing knowledge filter by delete + recreate"""
    jwt_token = user.jwt_token

    existing_result = await knowledge_filter_service.get_knowledge_filter(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )
    if not existing_result.get("success"):
        return JSONResponse(
            {"error": "Knowledge filter not found or access denied"}, status_code=404
        )

    existing_filter = existing_result["filter"]

    query_data = body.queryData if body.queryData is not None else existing_filter["query_data"]
    try:
        normalized_query_data = normalize_query_data(query_data)
    except Exception:
        logger.error("Failed to normalize query_data", exc_info=True)
        return JSONResponse({"error": "Invalid queryData format"}, status_code=400)

    delete_result = await knowledge_filter_service.delete_knowledge_filter(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )
    if not delete_result.get("success"):
        return JSONResponse(
            {"error": "Failed to delete existing knowledge filter"}, status_code=500
        )

    updated_filter = {
        "id": filter_id,
        "name": body.name if body.name is not None else existing_filter["name"],
        "description": body.description
        if body.description is not None
        else existing_filter["description"],
        "query_data": normalized_query_data,
        "owner": existing_filter["owner"],
        "allowed_users": body.allowedUsers
        if body.allowedUsers is not None
        else existing_filter.get("allowed_users", []),
        "allowed_groups": body.allowedGroups
        if body.allowedGroups is not None
        else existing_filter.get("allowed_groups", []),
        "created_at": existing_filter["created_at"],
        "updated_at": datetime.now(UTC).isoformat(),
    }

    result = await knowledge_filter_service.create_knowledge_filter(
        updated_filter, user_id=user.user_id, jwt_token=jwt_token
    )

    if result.get("success"):
        return JSONResponse(result, status_code=200)
    return _error_response(result)


async def delete_knowledge_filter(
    filter_id: str,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:edit:own")),
):
    """Delete a knowledge filter"""
    jwt_token = user.jwt_token
    result = await knowledge_filter_service.delete_knowledge_filter(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )

    if result.get("success"):
        return JSONResponse(result, status_code=200)
    return _error_response(result)


async def subscribe_to_knowledge_filter(
    filter_id: str,
    body: SubscribeBody,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    monitor_service=Depends(get_monitor_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:read")),
):
    """Create a subscription to a knowledge filter"""
    jwt_token = user.jwt_token

    filter_result = await knowledge_filter_service.get_knowledge_filter(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )
    if not filter_result.get("success"):
        return JSONResponse(
            {"error": "Knowledge filter not found or access denied"}, status_code=404
        )

    filter_doc = filter_result["filter"]

    monitor_result = await monitor_service.create_knowledge_filter_monitor(
        filter_id=filter_id,
        filter_name=filter_doc["name"],
        query_data=filter_doc["query_data"],
        user_id=user.user_id,
        jwt_token=jwt_token,
        notification_config=body.notification_config,
    )

    if not monitor_result.get("success"):
        return JSONResponse(monitor_result, status_code=500)

    subscription_data = {
        "subscription_id": monitor_result["subscription_id"],
        "monitor_id": monitor_result["monitor_id"],
        "webhook_url": monitor_result["webhook_url"],
        "created_at": datetime.now(UTC).isoformat(),
        "notification_config": body.notification_config or {},
    }

    update_result = await knowledge_filter_service.add_subscription(
        filter_id, subscription_data, user_id=user.user_id, jwt_token=jwt_token
    )

    if update_result.get("success"):
        return JSONResponse(
            {
                "success": True,
                "subscription_id": monitor_result["subscription_id"],
                "monitor_id": monitor_result["monitor_id"],
                "webhook_url": monitor_result["webhook_url"],
                "message": f"Successfully subscribed to knowledge filter: {filter_doc['name']}",
            },
            status_code=201,
        )
    else:
        await monitor_service.delete_monitor(monitor_result["monitor_id"], user.user_id, jwt_token)
        return JSONResponse({"error": "Failed to create subscription"}, status_code=500)


async def list_knowledge_filter_subscriptions(
    filter_id: str,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:read")),
):
    """List subscriptions for a knowledge filter"""
    jwt_token = user.jwt_token
    result = await knowledge_filter_service.get_filter_subscriptions(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )

    if result.get("success"):
        return JSONResponse(result, status_code=200)
    return _error_response(result)


async def cancel_knowledge_filter_subscription(
    filter_id: str,
    subscription_id: str,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    monitor_service=Depends(get_monitor_service),
    session_manager=Depends(get_session_manager),
    user: User = Depends(require_permission("kf:read")),
):
    """Cancel a subscription to a knowledge filter"""
    jwt_token = user.jwt_token

    subscriptions_result = await knowledge_filter_service.get_filter_subscriptions(
        filter_id, user_id=user.user_id, jwt_token=jwt_token
    )
    if not subscriptions_result.get("success"):
        return JSONResponse(
            {"error": "Knowledge filter not found or access denied"}, status_code=404
        )

    subscription = None
    for sub in subscriptions_result.get("subscriptions", []):
        if sub.get("subscription_id") == subscription_id:
            subscription = sub
            break

    if not subscription:
        return JSONResponse({"error": "Subscription not found"}, status_code=404)

    await monitor_service.delete_monitor(subscription["monitor_id"], user.user_id, jwt_token)

    remove_result = await knowledge_filter_service.remove_subscription(
        filter_id, subscription_id, user_id=user.user_id, jwt_token=jwt_token
    )

    if remove_result.get("success"):
        return JSONResponse(
            {"success": True, "message": "Subscription cancelled successfully"}, status_code=200
        )
    else:
        return JSONResponse({"error": "Failed to cancel subscription"}, status_code=500)


async def knowledge_filter_webhook(
    filter_id: str,
    subscription_id: str,
    request: Request,
    knowledge_filter_service=Depends(get_knowledge_filter_service),
    session_manager=Depends(get_session_manager),
):
    """Handle webhook notifications from OpenSearch monitors"""
    try:
        payload = await request.json()

        logger.info(
            "Knowledge filter webhook received",
            filter_id=filter_id,
            subscription_id=subscription_id,
            payload_size=len(str(payload)),
        )

        findings = payload.get("findings", [])
        if not findings:
            logger.info("No findings in webhook payload", subscription_id=subscription_id)
            return JSONResponse({"status": "no_findings"})

        matched_documents = []
        for finding in findings:
            matched_documents.append(
                {
                    "document_id": finding.get("_id"),
                    "index": finding.get("_index"),
                    "source": finding.get("_source", {}),
                    "score": finding.get("_score"),
                }
            )

        logger.info(
            "Knowledge filter matched documents",
            filter_id=filter_id,
            matched_count=len(matched_documents),
        )

        return JSONResponse(
            {
                "status": "processed",
                "filter_id": filter_id,
                "subscription_id": subscription_id,
                "matched_documents": len(matched_documents),
                "timestamp": datetime.now(UTC).isoformat(),
            }
        )

    except Exception as e:
        logger.exception(
            "[API] Knowledge filter webhook failed",
            filter_id=filter_id,
            subscription_id=subscription_id,
        )
        return JSONResponse({"error": f"Webhook processing failed: {str(e)}"}, status_code=500)