Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
13.9 KB
Raw Blame
from typing import Any, Literal

from fastapi import Depends, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field, field_validator

from api.files import validate_date_filter_bounds
from config.settings import is_url_connector_enabled
from connectors.url.projection import get_source_manifest, list_page_manifests
from dependencies import (
    get_search_service,
    require_permission,
)
from session_manager import User
from utils.logging_config import get_logger
from utils.opensearch_utils import DISK_SPACE_ERROR_MESSAGE, OpenSearchDiskSpaceError

logger = get_logger(__name__)

SUGGEST_MAX_FILTER_KEYS = 10
SUGGEST_MAX_FILTER_VALUES = 100

# OpenSearch's default result window caps a single ranked search at 10,000
# hits. Knowledge-source results are grouped after the ranked search, so use
# progressively larger candidate sets (up to that window) before applying the
# caller's result limit.
_MAX_KNOWLEDGE_SOURCE_CANDIDATES = 10_000


class SearchBody(BaseModel):
    query: str
    filters: dict[str, Any] = Field(default_factory=dict)
    limit: int = 10
    scoreThreshold: float = Field(default=0, alias="scoreThreshold")
    created_after: str | None = None
    created_before: str | None = None
    resultMode: Literal["chunks", "knowledge_sources", "website_pages"] = "chunks"

    model_config = {"populate_by_name": True}


class SuggestBody(BaseModel):
    """Search-as-you-type request. Bounds are enforced server-side."""

    query: str = Field(max_length=200)
    filters: dict[str, Any] = Field(default_factory=dict)
    # False: cheap keyword phase for every debounced keystroke.
    # True: also run the semantic (embedding) search.
    semantic: bool = False
    scoreThreshold: float = Field(default=0, ge=0, le=100)

    model_config = {"populate_by_name": True}

    @field_validator("filters")
    @classmethod
    def _bound_filters(cls, filters: dict[str, Any]) -> dict[str, Any]:
        if len(filters) > SUGGEST_MAX_FILTER_KEYS:
            raise ValueError("too many filters")
        for values in filters.values():
            if isinstance(values, list) and len(values) > SUGGEST_MAX_FILTER_VALUES:
                raise ValueError("too many filter values")
        return filters


def _website_page_chunk(page: dict[str, Any]) -> dict[str, Any]:
    """Adapt one OpenSearch page manifest to the child-page table shape."""
    return {
        "filename": page.get("filename") or page.get("canonical_url") or "Untitled page",
        "mimetype": page.get("mimetype") or "text/html",
        "page": 1,
        "text": "",
        "score": 0,
        "source_url": page.get("source_url") or page.get("canonical_url") or "",
        "file_size": page.get("file_size") or 0,
        "connector_type": "url",
        "document_id": page.get("content_document_id") or page.get("document_id"),
        "indexed_time": page.get("indexed_time"),
        "web_source_id": page.get("web_source_id"),
        "web_page_id": page.get("web_page_id"),
        "web_page_depth": page.get("web_page_depth") or 0,
        "canonical_url": page.get("canonical_url"),
        "status": page.get("status") or "active",
        "error": page.get("last_error"),
        "chunk_count": page.get("chunk_count") or 0,
        "embedding_model": page.get("embedding_model"),
        "embedding_dimensions": page.get("embedding_dimensions"),
    }


def _website_source_chunk(source: dict[str, Any], matching_page: dict[str, Any]) -> dict[str, Any]:
    """Represent a content-matching website page as its parent Knowledge row."""
    source_id = str(source["id"])
    return {
        "filename": source.get("filename") or source.get("name") or "Untitled website",
        "mimetype": source.get("mimetype") or "text/html",
        "page": matching_page.get("page"),
        "text": "",
        "highlights": matching_page.get("highlights", []),
        "score": matching_page.get("score", 0),
        "source_url": source.get("source_url") or source.get("starting_url") or "",
        "owner": source.get("owner") or source.get("owner_id"),
        "owner_name": source.get("owner_name", ""),
        "owner_email": source.get("owner_email", ""),
        "file_size": source.get("file_size", 0),
        "connector_type": "url",
        "embedding_model": source.get("embedding_model"),
        "embedding_provider": source.get("embedding_provider"),
        "embedding_space_id": source.get("embedding_space_id"),
        "embedding_dimensions": source.get("embedding_dimensions"),
        "allowed_users": source.get("allowed_users", []),
        "allowed_groups": source.get("allowed_groups", []),
        "allowed_principal_labels": source.get("allowed_principal_labels", []),
        "document_id": source.get("document_id"),
        "indexed_time": source.get("indexed_time"),
        "web_source_id": source_id,
        "web_child_count": source.get("web_child_count", 0),
        "chunk_count": source.get("chunk_count", 0),
        "canonical_url": source.get("starting_url"),
        "status": source.get("status", "active"),
        "error": source.get("last_error") or source.get("error"),
    }


async def _aggregate_website_page_matches(
    results: list[dict[str, Any]], user: User
) -> list[dict[str, Any]]:
    """Replace matching website-page chunks with their parent source manifests.

    A website source manifest has no body text or embedding. Its indexed child
    pages do, so excluding child chunks before ranking means content searches
    can never find their parent source. Keep the page hit's rank and score,
    but return the parent projection expected by the Knowledge table.
    """
    ranked_results: list[tuple[int, dict[str, Any]]] = []
    page_matches: dict[str, tuple[int, dict[str, Any]]] = {}
    direct_source_hits: dict[str, tuple[int, dict[str, Any]]] = {}

    for rank, result in enumerate(results):
        source_id = result.get("web_source_id")
        if result.get("web_page_id") and source_id:
            page_matches.setdefault(str(source_id), (rank, result))
        elif result.get("connector_type") == "url" and source_id:
            direct_source_hits[str(source_id)] = (rank, result)
        else:
            ranked_results.append((rank, result))

    for source_id, (page_rank, page_match) in page_matches.items():
        source = await get_source_manifest(source_id)
        if source is None or source.get("owner_id") != user.user_id or source.get("deleting"):
            continue
        direct_hit = direct_source_hits.pop(source_id, None)
        rank = min(page_rank, direct_hit[0]) if direct_hit else page_rank
        source_result = _website_source_chunk(source, page_match)
        if direct_hit and (direct_score := direct_hit[1].get("score")) is not None:
            source_result["score"] = max(source_result["score"] or 0, direct_score)
        ranked_results.append((rank, source_result))

    ranked_results.extend(direct_source_hits.values())
    return [result for _, result in sorted(ranked_results, key=lambda item: item[0])]


async def _website_page_search_result(
    body: SearchBody,
    user: User,
    search_service=None,
) -> dict[str, Any]:
    """Search durable child-page manifests, including zero-chunk pages.

    Manifest matches cover page URL and title, while indexed chunks cover page
    content. Both are projected from the manifest so page-table rows always
    carry durable page metadata such as status and page ID.
    """
    source_ids = body.filters.get("web_source_ids")
    if not isinstance(source_ids, list) or len(source_ids) != 1 or not source_ids[0]:
        raise HTTPException(422, "website_pages search requires one web_source_ids filter")
    source_id = source_ids[0]
    source = await get_source_manifest(source_id)
    if source is None or source.get("owner_id") != user.user_id:
        raise HTTPException(404, "Website source not found")

    query = body.query.strip()
    bounds = {
        "created_after": body.filters.get("created_after"),
        "created_before": body.filters.get("created_before"),
    }
    all_pages = await list_page_manifests(source_id, **bounds)
    matching_pages = (
        await list_page_manifests(source_id, query, **bounds)
        if query and query != "*"
        else all_pages
    )
    pages_by_id = {str(page["web_page_id"]): page for page in all_pages if page.get("web_page_id")}
    results_by_page_id = {
        str(page["web_page_id"]): _website_page_chunk(page)
        for page in matching_pages
        if page.get("web_page_id")
    }

    if query and query != "*" and search_service is not None:
        content_matches = await search_service.search(
            body.query,
            user_id=user.user_id,
            jwt_token=user.jwt_token,
            filters=body.filters,
            limit=body.limit,
            score_threshold=body.scoreThreshold,
            result_mode="chunks",
        )
        for match in content_matches.get("results", []):
            page_id = match.get("web_page_id")
            page = pages_by_id.get(str(page_id)) if page_id else None
            if page is None:
                continue
            result = results_by_page_id.setdefault(str(page_id), _website_page_chunk(page))
            if (score := match.get("score")) is not None and score > result["score"]:
                result["score"] = score
                result["highlights"] = match.get("highlights", [])

    results = list(results_by_page_id.values())
    return {"results": results, "total": len(results)}


async def _search_knowledge_sources(
    body: SearchBody,
    search_service: Any,
    user: User,
) -> dict[str, Any]:
    """Gather and group enough ranked hits before enforcing ``body.limit``."""
    requested_limit = max(body.limit, 0)
    candidate_limit = min(max(requested_limit * 10, 100), _MAX_KNOWLEDGE_SOURCE_CANDIDATES)

    while True:
        result = await search_service.search(
            body.query,
            user_id=user.user_id,
            jwt_token=user.jwt_token,
            filters=body.filters,
            limit=candidate_limit,
            score_threshold=body.scoreThreshold,
            result_mode=body.resultMode,
        )
        raw_results = result.get("results", [])
        grouped_results = await _aggregate_website_page_matches(raw_results, user)

        if (
            len(grouped_results) >= requested_limit
            or len(raw_results) < candidate_limit
            or candidate_limit == _MAX_KNOWLEDGE_SOURCE_CANDIDATES
        ):
            result["results"] = grouped_results[:requested_limit]
            result["total"] = len(result["results"])
            return result

        candidate_limit = min(candidate_limit * 2, _MAX_KNOWLEDGE_SOURCE_CANDIDATES)


async def search(
    body: SearchBody,
    search_service=Depends(get_search_service),
    user: User = Depends(require_permission("search:use")),
):
    """Search for documents"""
    merged_filters: dict[str, Any] = dict(body.filters)
    if body.created_after:
        merged_filters["created_after"] = body.created_after
    if body.created_before:
        merged_filters["created_before"] = body.created_before
    # Validate after merging: the bounds can also arrive inside `filters`.
    validate_date_filter_bounds(merged_filters)
    # Downstream result-mode helpers read body.filters, so make them see the merged bounds.
    body.filters = merged_filters

    try:
        if body.resultMode == "website_pages" and not is_url_connector_enabled():
            raise HTTPException(status_code=404, detail="Website connector is not enabled")

        jwt_token = user.jwt_token

        logger.debug(
            "Search API request",
            user_id=user.user_id,
            has_jwt_token=jwt_token is not None,
            query=body.query,
            filters=body.filters,
            limit=body.limit,
            score_threshold=body.scoreThreshold,
        )

        if body.resultMode == "website_pages":
            result = await _website_page_search_result(body, user, search_service)
        elif body.resultMode == "knowledge_sources":
            result = await _search_knowledge_sources(body, search_service, user)
        else:
            result = await search_service.search(
                body.query,
                user_id=user.user_id,
                jwt_token=jwt_token,
                filters=body.filters,
                limit=body.limit,
                score_threshold=body.scoreThreshold,
                result_mode=body.resultMode,
            )
        return JSONResponse(result, status_code=200)
    except HTTPException:
        raise
    except OpenSearchDiskSpaceError:
        return JSONResponse({"error": DISK_SPACE_ERROR_MESSAGE}, status_code=507)
    except Exception as e:
        error_msg = str(e)
        if "AuthenticationException" in error_msg or "access denied" in error_msg.lower():
            return JSONResponse({"error": error_msg}, status_code=403)
        else:
            return JSONResponse({"error": error_msg}, status_code=500)


async def suggest(
    body: SuggestBody,
    search_service=Depends(get_search_service),
    user: User = Depends(require_permission("search:use")),
):
    """Query completions, matching files and per-file previews."""
    try:
        result = await search_service.suggest(
            body.query,
            user_id=user.user_id,
            jwt_token=user.jwt_token,
            filters=body.filters,
            semantic=body.semantic,
            score_threshold=body.scoreThreshold,
        )
        if result.get("error"):
            logger.error("Search suggest failed", user_id=user.user_id, error=result["error"])
            return JSONResponse({"error": "Suggestions unavailable"}, status_code=502)
        return JSONResponse(result, status_code=200)
    except OpenSearchDiskSpaceError:
        return JSONResponse({"error": DISK_SPACE_ERROR_MESSAGE}, status_code=507)
    except Exception as e:
        # Detailed error stays server-side; clients get a generic message.
        logger.error("Search suggest failed", user_id=user.user_id, error=str(e))
        return JSONResponse({"error": "Suggestions unavailable"}, status_code=500)