Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
8.6 KB
Raw Blame
"""
Service for file-level listing and search via OpenSearch aggregations.

Aggregates document chunks by filename to produce file-level views,
with support for pagination, filtering, sorting, and fuzzy search.
"""

from typing import Any

from config.settings import get_index_name
from utils.logging_config import get_logger

logger = get_logger(__name__)


class FileService:
    """Provides file-level views over the chunk-based OpenSearch index."""

    def __init__(self, session_manager=None):
        self.session_manager = session_manager

    async def list_files(
        self,
        user_id: str,
        jwt_token: str = None,
        page: int = 1,
        page_size: int = 25,
        sort_by: str = "filename",
        sort_order: str = "asc",
        connector_type: str | None = None,
        mimetype: str | None = None,
        owner: str | None = None,
        search: str | None = None,
    ) -> dict[str, Any]:
        """
        List ingested files with server-side pagination, filtering, and sorting.

        Aggregates chunks by filename using OpenSearch terms aggregation,
        then paginates and sorts the resulting file list in-memory.
        """
        opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)

        query = self._build_filter_query(user_id, connector_type, mimetype, owner, search)
        agg_body = self._build_file_aggregation(query)

        try:
            result = await opensearch_client.search(
                index=get_index_name(),
                body=agg_body,
            )
        except Exception as e:
            logger.error("Failed to list files", error=str(e))
            raise

        files = self._parse_aggregation_buckets(result)
        files = self._sort_files(files, sort_by, sort_order)

        total = len(files)
        start = (page - 1) * page_size
        end = start + page_size
        paginated = files[start:end]

        return {
            "files": paginated,
            "total": total,
            "page": page,
            "page_size": page_size,
        }

    async def search_files(
        self,
        user_id: str,
        jwt_token: str = None,
        query: str = "",
        page: int = 1,
        page_size: int = 25,
        connector_type: str | None = None,
        mimetype: str | None = None,
        owner: str | None = None,
    ) -> dict[str, Any]:
        """
        Search files by name with fuzzy/prefix matching.

        Uses wildcard, match_phrase_prefix, and fuzzy queries on the
        filename field, then aggregates matching chunks into file-level results.
        """
        return await self.list_files(
            user_id=user_id,
            jwt_token=jwt_token,
            page=page,
            page_size=page_size,
            sort_by="filename",
            sort_order="asc",
            connector_type=connector_type,
            mimetype=mimetype,
            owner=owner,
            search=query,
        )

    def _build_filter_query(
        self,
        user_id: str,
        connector_type: str | None = None,
        mimetype: str | None = None,
        owner: str | None = None,
        search: str | None = None,
    ) -> dict[str, Any]:
        """Build the bool query with optional filters + filename search."""
        must = []
        filter_clauses = []

        if connector_type:
            filter_clauses.append({"term": {"connector_type": connector_type}})
        if mimetype:
            filter_clauses.append({"term": {"mimetype": mimetype}})
        if owner:
            filter_clauses.append({"term": {"owner": owner}})

        if search:
            must.append(
                {
                    "bool": {
                        "should": [
                            {
                                "wildcard": {
                                    "filename": {
                                        "value": f"*{search.lower()}*",
                                        "case_insensitive": True,
                                    }
                                }
                            },
                            {
                                "prefix": {
                                    "filename": {"value": search.lower(), "case_insensitive": True}
                                }
                            },
                        ],
                        "minimum_should_match": 1,
                    }
                }
            )

        query: dict[str, Any] = {"bool": {"filter": filter_clauses}}
        if must:
            query["bool"]["must"] = must

        return query

    def _build_file_aggregation(self, query: dict[str, Any]) -> dict[str, Any]:
        """Build the OpenSearch aggregation body for file-level grouping."""
        return {
            "size": 0,
            "query": query,
            "aggs": {
                "files": {
                    "terms": {
                        "field": "filename",
                        "size": 10000,
                    },
                    "aggs": {
                        "file_metadata": {
                            "top_hits": {
                                "size": 1,
                                "_source": [
                                    "document_id",
                                    "filename",
                                    "mimetype",
                                    "file_size",
                                    "source_url",
                                    "owner",
                                    "owner_name",
                                    "owner_email",
                                    "connector_type",
                                    "embedding_model",
                                    "embedding_dimensions",
                                    "indexed_time",
                                    "allowed_users",
                                    "allowed_groups",
                                    "allowed_principal_labels",
                                ],
                                "sort": [{"indexed_time": {"order": "desc"}}],
                            }
                        },
                        "chunk_count": {"value_count": {"field": "_id"}},
                    },
                }
            },
        }

    def _parse_aggregation_buckets(self, result: dict[str, Any]) -> list[dict[str, Any]]:
        """Parse OpenSearch aggregation buckets into file dicts."""
        buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])

        files = []
        for bucket in buckets:
            hits = bucket.get("file_metadata", {}).get("hits", {}).get("hits", [])
            if not hits:
                continue

            source = hits[0].get("_source", {})
            files.append(
                {
                    "filename": bucket["key"],
                    "document_id": source.get("document_id", ""),
                    "mimetype": source.get("mimetype", ""),
                    "file_size": source.get("file_size", 0),
                    "source_url": source.get("source_url", ""),
                    "owner": source.get("owner", ""),
                    "owner_name": source.get("owner_name", ""),
                    "owner_email": source.get("owner_email", ""),
                    "connector_type": source.get("connector_type", ""),
                    "embedding_model": source.get("embedding_model", ""),
                    "embedding_dimensions": source.get("embedding_dimensions"),
                    "indexed_time": source.get("indexed_time", ""),
                    "chunk_count": bucket.get("chunk_count", {}).get("value", 0),
                    "allowed_users": source.get("allowed_users", []),
                    "allowed_groups": source.get("allowed_groups", []),
                    "allowed_principal_labels": source.get("allowed_principal_labels", []),
                }
            )

        return files

    def _sort_files(
        self,
        files: list[dict[str, Any]],
        sort_by: str,
        sort_order: str,
    ) -> list[dict[str, Any]]:
        """Sort file list by the given field."""
        valid_sort_fields = {
            "filename",
            "file_size",
            "mimetype",
            "indexed_time",
            "connector_type",
            "chunk_count",
            "owner",
        }
        if sort_by not in valid_sort_fields:
            sort_by = "filename"

        reverse = sort_order.lower() == "desc"

        _numeric_sort_fields = {"file_size", "chunk_count"}

        def _sort_key(f):
            return f.get(sort_by) or (0 if sort_by in _numeric_sort_fields else "")

        return sorted(files, key=_sort_key, reverse=reverse)