OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import json
import re
import uuid
from datetime import UTC, datetime
from typing import Any
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from fastapi import Depends, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from api.files import _validate_date_range
from dependencies import (
get_knowledge_filter_service,
get_monitor_service,
get_session_manager,
require_permission,
)
from session_manager import User
from utils.logging_config import get_logger
logger = get_logger(__name__)
def _error_response(result: dict) -> JSONResponse:
"""Map a failed knowledge-filter service result to the right HTTP status.
Distinguishes an OpenSearch authentication failure (credential rejected ->
401, re-authenticate) from authorization ("access denied"/owner check ->
403). Conflating the two as 403 "insufficient permissions" hides the real
cause. ``not found`` stays 404; everything else is 500.
"""
from utils.opensearch_utils import is_opensearch_auth_error
error_msg = result.get("error", "") or ""
low = error_msg.lower()
if "not found" in low or "already deleted" in low:
return JSONResponse(result, status_code=404)
if is_opensearch_auth_error(error_msg):
return JSONResponse(result, status_code=401)
if "access denied" in low or "insufficient permissions" in low:
return JSONResponse(result, status_code=403)
return JSONResponse(result, status_code=500)
_CALENDAR_DAY_RE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", re.ASCII)
_DATE_BOUND_KEYS = ("created_after", "created_before")
_DATE_CONTEXT_KEYS = ("from", "to", "timezone")
def _validate_calendar_day(value: Any, name: str) -> str:
if not isinstance(value, str):
raise ValueError(f"{name} must be a YYYY-MM-DD string")
# Exact zero-padded form only: the range check compares these as strings.
if not _CALENDAR_DAY_RE.fullmatch(value):
raise ValueError(f"{name} must be a valid YYYY-MM-DD date")
try:
datetime.strptime(value, "%Y-%m-%d")
except ValueError as err:
raise ValueError(f"{name} must be a valid YYYY-MM-DD date") from err
return value
def _normalize_date_range(data: dict, filters: dict) -> tuple[dict, dict | None]:
"""Validate the optional saved date bounds and their display context.
Returns ``(bounds, context)``. ``bounds`` holds only the UTC bounds that are
set; ``context`` is the calendar days + IANA timezone used to restore the
picker, kept outside ``filters`` so it never reaches retrieval.
Raises ValueError for any malformed value.
"""
bounds: dict[str, str] = {}
for key in _DATE_BOUND_KEYS:
value = filters.get(key)
if value is None or value == "":
continue
if not isinstance(value, str):
raise ValueError(f"{key} must be an ISO 8601 string")
bounds[key] = value
try:
_validate_date_range(bounds.get("created_after"), bounds.get("created_before"))
except HTTPException as err:
raise ValueError(str(err.detail)) from err
raw_context = data.get("date_range_context")
if raw_context is None:
return bounds, None
if not bounds:
# Context without bounds would describe a range that is not applied.
return bounds, None
if not isinstance(raw_context, dict):
raise ValueError("date_range_context must be an object")
context: dict[str, str] = {}
for key in ("from", "to"):
if raw_context.get(key) is not None:
context[key] = _validate_calendar_day(raw_context[key], f"date_range_context.{key}")
if "from" in context and "to" in context and context["from"] > context["to"]:
raise ValueError("date_range_context.from must be on or before date_range_context.to")
tz = raw_context.get("timezone")
if tz is not None:
try:
if not isinstance(tz, str) or not tz:
raise ValueError
ZoneInfo(tz)
except (ValueError, ZoneInfoNotFoundError, OSError) as err:
raise ValueError("date_range_context.timezone must be a valid IANA timezone") from err
context["timezone"] = tz
return bounds, context or None
def normalize_query_data(query_data: str | dict) -> str:
"""
Normalize query_data to ensure all required fields exist with defaults.
This prevents frontend crashes when API-created filters have incomplete data.
Optional ``created_after``/``created_before`` bounds are validated and kept
inside ``filters``; filters saved without dates stay unrestricted.
Raises ValueError for malformed date data.
"""
if isinstance(query_data, str):
try:
data = json.loads(query_data)
except json.JSONDecodeError:
data = {}
else:
data = query_data or {}
filters = data.get("filters") or {}
normalized_filters = {
"data_sources": filters.get("data_sources", ["*"]),
"document_types": filters.get("document_types", ["*"]),
"owners": filters.get("owners", ["*"]),
"connector_types": filters.get("connector_types", ["*"]),
}
bounds, date_context = _normalize_date_range(data, filters)
normalized_filters.update(bounds)
normalized = {
"query": data.get("query", ""),
"filters": normalized_filters,
"limit": data.get("limit", 10),
"scoreThreshold": data.get("scoreThreshold", 0),
"color": data.get("color", "zinc"),
"icon": data.get("icon", "filter"),
}
if date_context:
normalized["date_range_context"] = date_context
return json.dumps(normalized)
class CreateFilterBody(BaseModel):
name: str
description: str = ""
queryData: Any | None = None
allowedUsers: list[str] = []
allowedGroups: list[str] = []
class SearchFiltersBody(BaseModel):
query: str = ""
limit: int = 20
class UpdateFilterBody(BaseModel):
name: str | None = None
description: str | None = None
queryData: Any | None = None
allowedUsers: list[str] | None = None
allowedGroups: list[str] | None = None
class SubscribeBody(BaseModel):
notification_config: dict | None = None
async def create_knowledge_filter(
body: CreateFilterBody,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:create")),
):
"""Create a new knowledge filter"""
if not body.name:
return JSONResponse({"error": "Knowledge filter name is required"}, status_code=400)
if not body.queryData:
return JSONResponse({"error": "Query data is required"}, status_code=400)
try:
normalized_query_data = normalize_query_data(body.queryData)
except Exception:
logger.error("Failed to normalize query_data", exc_info=True)
return JSONResponse({"error": "Invalid queryData format"}, status_code=400)
jwt_token = user.jwt_token
filter_id = str(uuid.uuid4())
filter_doc = {
"id": filter_id,
"name": body.name,
"description": body.description,
"query_data": normalized_query_data,
"owner": user.user_id,
"allowed_users": body.allowedUsers,
"allowed_groups": body.allowedGroups,
"created_at": datetime.now(UTC).isoformat(),
"updated_at": datetime.now(UTC).isoformat(),
}
result = await knowledge_filter_service.create_knowledge_filter(
filter_doc, user_id=user.user_id, jwt_token=jwt_token
)
if result.get("success"):
return JSONResponse(result, status_code=201)
return _error_response(result)
async def search_knowledge_filters(
body: SearchFiltersBody,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:read")),
):
"""Search for knowledge filters by name, description, or query content"""
jwt_token = user.jwt_token
result = await knowledge_filter_service.search_knowledge_filters(
body.query, user_id=user.user_id, jwt_token=jwt_token, limit=body.limit
)
if result.get("success"):
return JSONResponse(result, status_code=200)
return _error_response(result)
async def get_knowledge_filter(
filter_id: str,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:read")),
):
"""Get a specific knowledge filter by ID"""
jwt_token = user.jwt_token
result = await knowledge_filter_service.get_knowledge_filter(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if result.get("success"):
return JSONResponse(result, status_code=200)
return _error_response(result)
async def update_knowledge_filter(
filter_id: str,
body: UpdateFilterBody,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:edit:own")),
):
"""Update an existing knowledge filter by delete + recreate"""
jwt_token = user.jwt_token
existing_result = await knowledge_filter_service.get_knowledge_filter(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if not existing_result.get("success"):
return JSONResponse(
{"error": "Knowledge filter not found or access denied"}, status_code=404
)
existing_filter = existing_result["filter"]
query_data = body.queryData if body.queryData is not None else existing_filter["query_data"]
try:
normalized_query_data = normalize_query_data(query_data)
except Exception:
logger.error("Failed to normalize query_data", exc_info=True)
return JSONResponse({"error": "Invalid queryData format"}, status_code=400)
delete_result = await knowledge_filter_service.delete_knowledge_filter(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if not delete_result.get("success"):
return JSONResponse(
{"error": "Failed to delete existing knowledge filter"}, status_code=500
)
updated_filter = {
"id": filter_id,
"name": body.name if body.name is not None else existing_filter["name"],
"description": body.description
if body.description is not None
else existing_filter["description"],
"query_data": normalized_query_data,
"owner": existing_filter["owner"],
"allowed_users": body.allowedUsers
if body.allowedUsers is not None
else existing_filter.get("allowed_users", []),
"allowed_groups": body.allowedGroups
if body.allowedGroups is not None
else existing_filter.get("allowed_groups", []),
"created_at": existing_filter["created_at"],
"updated_at": datetime.now(UTC).isoformat(),
}
result = await knowledge_filter_service.create_knowledge_filter(
updated_filter, user_id=user.user_id, jwt_token=jwt_token
)
if result.get("success"):
return JSONResponse(result, status_code=200)
return _error_response(result)
async def delete_knowledge_filter(
filter_id: str,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:edit:own")),
):
"""Delete a knowledge filter"""
jwt_token = user.jwt_token
result = await knowledge_filter_service.delete_knowledge_filter(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if result.get("success"):
return JSONResponse(result, status_code=200)
return _error_response(result)
async def subscribe_to_knowledge_filter(
filter_id: str,
body: SubscribeBody,
knowledge_filter_service=Depends(get_knowledge_filter_service),
monitor_service=Depends(get_monitor_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:read")),
):
"""Create a subscription to a knowledge filter"""
jwt_token = user.jwt_token
filter_result = await knowledge_filter_service.get_knowledge_filter(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if not filter_result.get("success"):
return JSONResponse(
{"error": "Knowledge filter not found or access denied"}, status_code=404
)
filter_doc = filter_result["filter"]
monitor_result = await monitor_service.create_knowledge_filter_monitor(
filter_id=filter_id,
filter_name=filter_doc["name"],
query_data=filter_doc["query_data"],
user_id=user.user_id,
jwt_token=jwt_token,
notification_config=body.notification_config,
)
if not monitor_result.get("success"):
return JSONResponse(monitor_result, status_code=500)
subscription_data = {
"subscription_id": monitor_result["subscription_id"],
"monitor_id": monitor_result["monitor_id"],
"webhook_url": monitor_result["webhook_url"],
"created_at": datetime.now(UTC).isoformat(),
"notification_config": body.notification_config or {},
}
update_result = await knowledge_filter_service.add_subscription(
filter_id, subscription_data, user_id=user.user_id, jwt_token=jwt_token
)
if update_result.get("success"):
return JSONResponse(
{
"success": True,
"subscription_id": monitor_result["subscription_id"],
"monitor_id": monitor_result["monitor_id"],
"webhook_url": monitor_result["webhook_url"],
"message": f"Successfully subscribed to knowledge filter: {filter_doc['name']}",
},
status_code=201,
)
else:
await monitor_service.delete_monitor(monitor_result["monitor_id"], user.user_id, jwt_token)
return JSONResponse({"error": "Failed to create subscription"}, status_code=500)
async def list_knowledge_filter_subscriptions(
filter_id: str,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:read")),
):
"""List subscriptions for a knowledge filter"""
jwt_token = user.jwt_token
result = await knowledge_filter_service.get_filter_subscriptions(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if result.get("success"):
return JSONResponse(result, status_code=200)
return _error_response(result)
async def cancel_knowledge_filter_subscription(
filter_id: str,
subscription_id: str,
knowledge_filter_service=Depends(get_knowledge_filter_service),
monitor_service=Depends(get_monitor_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_permission("kf:read")),
):
"""Cancel a subscription to a knowledge filter"""
jwt_token = user.jwt_token
subscriptions_result = await knowledge_filter_service.get_filter_subscriptions(
filter_id, user_id=user.user_id, jwt_token=jwt_token
)
if not subscriptions_result.get("success"):
return JSONResponse(
{"error": "Knowledge filter not found or access denied"}, status_code=404
)
subscription = None
for sub in subscriptions_result.get("subscriptions", []):
if sub.get("subscription_id") == subscription_id:
subscription = sub
break
if not subscription:
return JSONResponse({"error": "Subscription not found"}, status_code=404)
await monitor_service.delete_monitor(subscription["monitor_id"], user.user_id, jwt_token)
remove_result = await knowledge_filter_service.remove_subscription(
filter_id, subscription_id, user_id=user.user_id, jwt_token=jwt_token
)
if remove_result.get("success"):
return JSONResponse(
{"success": True, "message": "Subscription cancelled successfully"}, status_code=200
)
else:
return JSONResponse({"error": "Failed to cancel subscription"}, status_code=500)
async def knowledge_filter_webhook(
filter_id: str,
subscription_id: str,
request: Request,
knowledge_filter_service=Depends(get_knowledge_filter_service),
session_manager=Depends(get_session_manager),
):
"""Handle webhook notifications from OpenSearch monitors"""
try:
payload = await request.json()
logger.info(
"Knowledge filter webhook received",
filter_id=filter_id,
subscription_id=subscription_id,
payload_size=len(str(payload)),
)
findings = payload.get("findings", [])
if not findings:
logger.info("No findings in webhook payload", subscription_id=subscription_id)
return JSONResponse({"status": "no_findings"})
matched_documents = []
for finding in findings:
matched_documents.append(
{
"document_id": finding.get("_id"),
"index": finding.get("_index"),
"source": finding.get("_source", {}),
"score": finding.get("_score"),
}
)
logger.info(
"Knowledge filter matched documents",
filter_id=filter_id,
matched_count=len(matched_documents),
)
return JSONResponse(
{
"status": "processed",
"filter_id": filter_id,
"subscription_id": subscription_id,
"matched_documents": len(matched_documents),
"timestamp": datetime.now(UTC).isoformat(),
}
)
except Exception as e:
logger.exception(
"[API] Knowledge filter webhook failed",
filter_id=filter_id,
subscription_id=subscription_id,
)
return JSONResponse({"error": f"Webhook processing failed: {str(e)}"}, status_code=500)