Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
27.7 KB
Raw Blame
"""
File service v2 — composite aggregation pagination.

Uses OpenSearch composite aggregation for O(page_size) server-side
pagination. Only page_size buckets are processed per request regardless of
total file count.

Sort-field notes
----------------
* Fields in _COMPOSITE_SORT_FIELDS are handled entirely by the composite agg —
  ordering is globally correct and cursor-paginated.
* "owner" is a real keyword field so it lives in _COMPOSITE_SORT_FIELDS.
* "chunk_count" is a derived metric (value_count sub-agg).  Composite aggs
  cannot order by sub-aggregation metrics, so sorting by chunk_count falls back
  to a classic terms aggregation sorted by the sub-agg, with offset-based
  pagination (_build_terms_aggregation_for_chunk_count).  This is O(from+size)
  at the shard level but gives a globally correct sort.
"""

from typing import Any

from opensearchpy.exceptions import NotFoundError

from config.settings import get_index_name
from utils.logging_config import get_logger
from utils.opensearch_queries import build_date_range_query

logger = get_logger(__name__)


def _is_index_not_found(exc: Exception) -> bool:
    """True for a missing index (fresh install, nothing ingested yet)."""
    return (
        isinstance(exc, NotFoundError)
        and exc.status_code == 404
        and exc.error == "index_not_found_exception"
    )


# A file row is a (filename, document_id) pair: document_id is a content hash,
# so identical content under two names is two rows. Missing values count as "".
_FILE_IDENTITY_SCRIPT = (
    "(doc['filename'].size() == 0 ? '' : doc['filename'].value) + params.separator + "
    "(doc['document_id'].size() == 0 ? '' : doc['document_id'].value)"
)


def _file_identity_script() -> dict[str, Any]:
    """Painless script building the (filename, document_id) identity key."""
    return {
        "lang": "painless",
        "source": _FILE_IDENTITY_SCRIPT,
        # Painless has no "\\n" string escape, so the separator is a param.
        "params": {"separator": "\n"},
    }


def _distinct_files_agg() -> dict[str, Any]:
    """Cardinality aggregation counting distinct files by identity."""
    return {"cardinality": {"script": _file_identity_script(), "precision_threshold": 3000}}


_COMPOSITE_SORT_FIELDS: dict[str, str] = {
    "filename": "filename",
    "file_size": "file_size",
    "mimetype": "mimetype",
    "indexed_time": "indexed_time",
    "connector_type": "connector_type",
    "embedding_model": "embedding_model",
    "owner": "owner",
}


class FileServiceV2:
    """File-level views via composite aggregation (v2 — cursor pagination)."""

    def __init__(self, session_manager=None):
        self.session_manager = session_manager

    @staticmethod
    def _exact_metadata_filter(field: str, value: str) -> dict[str, Any]:
        """Match metadata written before and after its keyword mapping existed."""
        return {
            "bool": {
                "should": [
                    {"term": {field: value}},
                    {"term": {f"{field}.keyword": value}},
                ],
                "minimum_should_match": 1,
            }
        }

    async def list_files(
        self,
        user_id: str,
        jwt_token: str = None,
        page: int = 1,
        page_size: int = 25,
        sort_by: str = "filename",
        sort_order: str = "asc",
        connector_type: list[str] | None = None,
        mimetype: list[str] | None = None,
        owner: list[str] | None = None,
        search: str | None = None,
        after_key: dict | None = None,
        data_sources: list[str] | None = None,
        created_after: str | None = None,
        created_before: str | None = None,
    ) -> dict[str, Any]:
        """
        List files with server-side pagination via composite aggregation.

        Cost is O(page_size) per request (as opposed to returning all unique
        file sizes from OpenSearch).  Returns after_key for the next page
        (None when on the last page), plus an approximate total from a
        cardinality aggregation.

        Exception: when sort_by="chunk_count" a terms aggregation is used
        instead (composite aggs cannot order by sub-agg metrics).  Pagination
        is offset-based in that path; after_key is always None.
        """
        opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)

        query = self._build_filter_query(
            user_id,
            connector_type,
            mimetype,
            owner,
            search,
            data_sources,
            created_after,
            created_before,
        )
        total, is_approximate = await self._get_file_count(opensearch_client, query)

        if sort_by == "chunk_count":
            return await self._list_files_by_chunk_count(
                opensearch_client=opensearch_client,
                query=query,
                page=page,
                page_size=page_size,
                sort_order=sort_order,
                total=total,
                is_approximate=is_approximate,
            )

        if page > 1 and after_key is None:
            return {
                "files": [],
                "total": total,
                "is_approximate": is_approximate,
                "page": page,
                "page_size": page_size,
                "after_key": None,
            }

        composite_sort_field = _COMPOSITE_SORT_FIELDS.get(sort_by, "filename")

        agg_body = self._build_composite_aggregation(
            query=query,
            page_size=page_size,
            sort_field=composite_sort_field,
            sort_order=sort_order,
            after_key=after_key,
        )

        try:
            result = await opensearch_client.search(
                index=get_index_name(),
                body=agg_body,
            )
        except Exception as e:
            logger.error("Failed to list files (v2)", error=str(e))
            from utils.opensearch_utils import is_opensearch_auth_error

            if is_opensearch_auth_error(e):
                raise
            if not _is_index_not_found(e):
                raise
            return {
                "files": [],
                "total": 0,
                "is_approximate": False,
                "page": page,
                "page_size": page_size,
                "after_key": None,
            }

        raw_bucket_count = len(result.get("aggregations", {}).get("files", {}).get("buckets", []))
        files, next_after_key = self._parse_composite_buckets(result)

        if raw_bucket_count < page_size:  # final page, no after_key
            next_after_key = None

        return {
            "files": files,
            "total": total,
            "is_approximate": is_approximate,
            "page": page,
            "page_size": page_size,
            "after_key": next_after_key,
        }

    async def upload_histogram(
        self,
        user_id: str,
        jwt_token: str = None,
        start: str = "",
        end: str = "",
        time_zone: str = "UTC",
        connector_type: list[str] | None = None,
        mimetype: list[str] | None = None,
        owner: list[str] | None = None,
        data_sources: list[str] | None = None,
    ) -> dict[str, Any]:
        """Distinct files (filename + document_id, like the file list) per calendar day (in *time_zone*) within [start, end], plus the all-time total."""
        opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
        query = self._build_filter_query(
            user_id, connector_type, mimetype, owner, None, data_sources
        )
        # The window is a filter sub-agg rather than part of the query so the
        # same request also yields the all-time total.
        body = {
            "size": 0,
            "query": query,
            "aggs": {
                "total": _distinct_files_agg(),
                "window": {
                    "filter": build_date_range_query("indexed_time", gte=start, lte=end),
                    "aggs": {
                        "days": {
                            "date_histogram": {
                                "field": "indexed_time",
                                "calendar_interval": "day",
                                "time_zone": time_zone,
                                "format": "yyyy-MM-dd",
                                "min_doc_count": 1,
                            },
                            "aggs": {"files": _distinct_files_agg()},
                        }
                    },
                },
            },
        }
        result = await opensearch_client.search(index=get_index_name(), body=body)
        aggs = result.get("aggregations", {})
        buckets = aggs.get("window", {}).get("days", {}).get("buckets", [])
        return {
            "total": aggs.get("total", {}).get("value", 0),
            "days": {
                b["key_as_string"]: b.get("files", {}).get("value", 0)
                for b in buckets
                if b.get("key_as_string")
            },
        }

    async def _list_files_by_chunk_count(
        self,
        opensearch_client: Any,
        query: dict[str, Any],
        page: int,
        page_size: int,
        sort_order: str,
        total: int,
        is_approximate: bool,
    ) -> dict[str, Any]:
        """
        List files sorted globally by chunk_count using a terms aggregation.

        Terms aggs support ordering by sub-aggregation metrics, which composite
        aggs do not.  Pagination is offset-based (from = (page-1)*page_size).
        """
        offset = (page - 1) * page_size
        agg_body = self._build_terms_aggregation_for_chunk_count(
            query=query,
            page_size=page_size,
            offset=offset,
            sort_order=sort_order,
        )

        try:
            result = await opensearch_client.search(
                index=get_index_name(),
                body=agg_body,
            )
        except Exception as e:
            logger.error("Failed to list files by chunk_count (v2)", error=str(e))
            from utils.opensearch_utils import is_opensearch_auth_error

            if is_opensearch_auth_error(e):
                raise
            if not _is_index_not_found(e):
                raise
            return {
                "files": [],
                "total": 0,
                "page": page,
                "page_size": page_size,
                "after_key": None,
            }

        files = self._parse_terms_buckets(result, offset=offset)

        return {
            "files": files,
            "total": total,
            "is_approximate": is_approximate,
            "page": page,
            "page_size": page_size,
            "after_key": None,  # offset pagination; no cursor
        }

    async def search_files(
        self,
        user_id: str,
        jwt_token: str = None,
        query: str = "",
        page: int = 1,
        page_size: int = 25,
        connector_type: list[str] | None = None,
        mimetype: list[str] | None = None,
        owner: list[str] | None = None,
        after_key: dict | None = None,
        data_sources: list[str] | None = None,
        created_after: str | None = None,
        created_before: str | None = None,
    ) -> dict[str, Any]:
        """Search files by name with fuzzy/prefix matching."""
        return await self.list_files(
            user_id=user_id,
            jwt_token=jwt_token,
            page=page,
            page_size=page_size,
            sort_by="filename",
            sort_order="asc",
            connector_type=connector_type,
            mimetype=mimetype,
            owner=owner,
            search=query,
            after_key=after_key,
            data_sources=data_sources,
            created_after=created_after,
            created_before=created_before,
        )

    def _build_filter_query(
        self,
        user_id: str,
        connector_type: list[str] | None = None,
        mimetype: list[str] | None = None,
        owner: list[str] | None = None,
        search: str | None = None,
        data_sources: list[str] | None = None,
        created_after: str | None = None,
        created_before: str | None = None,
    ) -> dict[str, Any]:
        """Build the bool query (filters, filename search, date range) for file views."""

        def _effective_values(values: str | list[str] | None) -> list[str]:
            if values is None:
                return []
            if isinstance(values, str):
                values = [values]
            return [value for value in values if value != "*"]

        must = []
        filter_clauses = []

        effective_connector_types = _effective_values(connector_type)
        if effective_connector_types:
            clause = (
                {"term": {"connector_type": effective_connector_types[0]}}
                if len(effective_connector_types) == 1
                else {"terms": {"connector_type": effective_connector_types}}
            )
            filter_clauses.append(clause)

        effective_mimetypes = _effective_values(mimetype)
        if effective_mimetypes:
            clause = (
                {"term": {"mimetype": effective_mimetypes[0]}}
                if len(effective_mimetypes) == 1
                else {"terms": {"mimetype": effective_mimetypes}}
            )
            filter_clauses.append(clause)

        effective_owners = _effective_values(owner)
        if effective_owners:
            clause = (
                {"term": {"owner": effective_owners[0]}}
                if len(effective_owners) == 1
                else {"terms": {"owner": effective_owners}}
            )
            filter_clauses.append(clause)

        effective_sources = _effective_values(data_sources)
        if effective_sources:
            clause = (
                {"term": {"filename": effective_sources[0]}}
                if len(effective_sources) == 1
                else {"terms": {"filename": effective_sources}}
            )
            filter_clauses.append(clause)

        date_range_clause = build_date_range_query(
            "indexed_time", gte=created_after, lte=created_before
        )
        if date_range_clause is not None:
            filter_clauses.append(date_range_clause)

        if search:
            must.append(
                {
                    "bool": {
                        "should": [
                            {
                                "wildcard": {
                                    "filename": {
                                        "value": f"*{search.lower()}*",
                                        "case_insensitive": True,
                                    }
                                }
                            },
                            {
                                "prefix": {
                                    "filename": {"value": search.lower(), "case_insensitive": True}
                                }
                            },
                        ],
                        "minimum_should_match": 1,
                    }
                }
            )

        query: dict[str, Any] = {
            "bool": {
                "filter": filter_clauses,
            }
        }
        # The root Knowledge view exposes URL source projections only.
        # Legacy documents with no record_kind continue to match. Failed
        # first-time URL crawls mirror regular failed uploads: they do not
        # become persistent Knowledge rows.
        query["bool"]["must_not"] = [
            self._exact_metadata_filter("record_kind", "web_page"),
            self._exact_metadata_filter("record_kind", "web_page_manifest"),
            {
                "bool": {
                    "filter": [
                        self._exact_metadata_filter("record_kind", "web_source"),
                        self._exact_metadata_filter("status", "failed"),
                    ]
                }
            },
        ]
        if must:
            query["bool"]["must"] = must
        return query

    def _build_composite_aggregation(
        self,
        query: dict[str, Any],
        page_size: int,
        sort_field: str,
        sort_order: str,
        after_key: dict | None,
    ) -> dict[str, Any]:
        """Build the composite aggregation request, with filename/document_id tiebreakers."""
        composite: dict[str, Any] = {
            "size": page_size,
            "sources": [
                {
                    sort_field: {
                        "terms": {
                            "field": sort_field,
                            "order": sort_order,
                            "missing_bucket": True,
                        }
                    }
                },
                *(
                    [
                        {
                            "filename_tiebreak": {
                                "terms": {
                                    "field": "filename",
                                    "order": sort_order,
                                    "missing_bucket": True,
                                }
                            }
                        }
                    ]
                    if sort_field != "filename"
                    else []
                ),
                {
                    "document_id_tiebreak": {
                        "terms": {
                            "field": "document_id",
                            "order": sort_order,
                            "missing_bucket": True,
                        }
                    }
                },
            ],
        }

        if after_key:
            composite["after"] = after_key

        return {
            "size": 0,
            "query": query,
            "aggs": {
                "files": {
                    "composite": composite,
                    "aggs": {
                        "file_metadata": {
                            "top_hits": {
                                "size": 1,
                                "_source": [
                                    "document_id",
                                    "filename",
                                    "mimetype",
                                    "file_size",
                                    "source_url",
                                    "owner",
                                    "owner_name",
                                    "owner_email",
                                    "connector_type",
                                    "embedding_model",
                                    "embedding_provider",
                                    "embedding_space_id",
                                    "embedding_dimensions",
                                    "indexed_time",
                                    "allowed_users",
                                    "allowed_groups",
                                    "allowed_principal_labels",
                                    "web_source_id",
                                    "web_page_id",
                                    "web_page_depth",
                                    "web_child_count",
                                    "chunk_count",
                                    "record_kind",
                                    "status",
                                    "error",
                                ],
                                "sort": [{"indexed_time": {"order": "desc"}}],
                            }
                        },
                        "chunk_count": {"value_count": {"field": "_id"}},
                    },
                }
            },
        }

    def _build_terms_aggregation_for_chunk_count(
        self,
        query: dict[str, Any],
        page_size: int,
        offset: int,
        sort_order: str,
    ) -> dict[str, Any]:
        """
        Build a terms aggregation sorted by chunk_count sub-agg.

        Terms agg supports `order: { sub_agg: asc/desc }`.  To implement
        offset-based pagination we request (offset + page_size) buckets and
        slice in Python — OpenSearch has no native offset for aggs.
        """
        return {
            "size": 0,
            "query": query,
            "aggs": {
                "files": {
                    "terms": {
                        # Same identity as _get_file_count, so totals and buckets agree.
                        "script": _file_identity_script(),
                        "size": offset + page_size,
                        "order": {"sort_chunk_count": sort_order},
                    },
                    "aggs": {
                        "file_metadata": {
                            "top_hits": {
                                "size": 1,
                                "_source": [
                                    "document_id",
                                    "filename",
                                    "mimetype",
                                    "file_size",
                                    "source_url",
                                    "owner",
                                    "owner_name",
                                    "owner_email",
                                    "connector_type",
                                    "embedding_model",
                                    "embedding_provider",
                                    "embedding_space_id",
                                    "embedding_dimensions",
                                    "indexed_time",
                                    "allowed_users",
                                    "allowed_groups",
                                    "allowed_principal_labels",
                                    "web_source_id",
                                    "web_page_id",
                                    "web_page_depth",
                                    "web_child_count",
                                    "chunk_count",
                                    "record_kind",
                                    "status",
                                    "error",
                                ],
                                "sort": [{"indexed_time": {"order": "desc"}}],
                            }
                        },
                        "chunk_count": {"value_count": {"field": "_id"}},
                        # Website sources are represented by one summary
                        # manifest, while ordinary documents have one record
                        # per chunk. Sum one per normal record but use the
                        # summary's aggregate count for website sources so the
                        # ordering matches the value displayed in the table.
                        "sort_chunk_count": {
                            "sum": {
                                "script": {
                                    "lang": "painless",
                                    "source": (
                                        "if (doc.containsKey('record_kind') "
                                        "&& doc['record_kind'].size() != 0 "
                                        "&& doc['record_kind'].value == 'web_source') { "
                                        "return doc.containsKey('chunk_count') "
                                        "&& doc['chunk_count'].size() != 0 "
                                        "? doc['chunk_count'].value : 0; } return 1;"
                                    ),
                                }
                            }
                        },
                    },
                }
            },
        }

    def _parse_terms_buckets(self, result: dict[str, Any], offset: int = 0) -> list[dict[str, Any]]:
        """Parse terms agg buckets, applying offset slice. Returns files list."""
        buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])
        buckets = buckets[offset:]
        return self._buckets_to_files(buckets)

    async def _get_file_count(
        self, opensearch_client: Any, query: dict[str, Any]
    ) -> tuple[int, bool]:
        """Approximate unique-file count (filename + document_id) via cardinality aggregation (O(1)).

        Returns (count, is_approximate).  is_approximate is always True
        (cardinality agg is inherently approximate).  A missing index (fresh
        install) yields (0, True); every other failure is re-raised so the API
        returns an error instead of a wrong total or an empty page.
        """
        body = {
            "size": 0,
            "query": query,
            "aggs": {"file_count": _distinct_files_agg()},
        }
        try:
            result = await opensearch_client.search(index=get_index_name(), body=body)
            return result.get("aggregations", {}).get("file_count", {}).get("value", 0), True
        except Exception as e:
            if not _is_index_not_found(e):
                logger.error("Failed to retrieve file count (v2)", error=str(e))
                raise
            logger.warning("File count skipped; index not found", error=str(e))
            return 0, True

    def _parse_composite_buckets(
        self, result: dict[str, Any]
    ) -> tuple[list[dict[str, Any]], dict | None]:
        """Parse composite agg buckets. Returns (files, next_after_key)."""
        agg = result.get("aggregations", {}).get("files", {})
        buckets = agg.get("buckets", [])
        next_after_key = agg.get("after_key")
        return self._buckets_to_files(buckets), next_after_key

    def _buckets_to_files(self, buckets: list[dict[str, Any]]) -> list[dict[str, Any]]:
        """Convert raw aggregation buckets (composite or terms) to file dicts."""
        files = []
        for bucket in buckets:
            hits = bucket.get("file_metadata", {}).get("hits", {}).get("hits", [])
            if not hits:
                continue
            source = hits[0].get("_source", {})
            files.append(
                {
                    "filename": source.get("filename")
                    or (
                        bucket["key"].get("filename", "")
                        if isinstance(bucket["key"], dict)
                        # Script-grouped terms keys are "filename\ndocument_id".
                        else str(bucket.get("key", "")).partition("\n")[0]
                    ),
                    "document_id": source.get("document_id", ""),
                    "mimetype": source.get("mimetype", ""),
                    "file_size": source.get("file_size", 0),
                    "source_url": source.get("source_url", ""),
                    "owner": source.get("owner", ""),
                    "owner_name": source.get("owner_name", ""),
                    "owner_email": source.get("owner_email", ""),
                    "connector_type": source.get("connector_type", ""),
                    "embedding_model": source.get("embedding_model", ""),
                    "embedding_provider": source.get("embedding_provider", ""),
                    "embedding_space_id": source.get("embedding_space_id", ""),
                    "embedding_dimensions": source.get("embedding_dimensions"),
                    "indexed_time": source.get("indexed_time", ""),
                    "chunk_count": (
                        source.get("chunk_count", 0)
                        if source.get("record_kind") == "web_source"
                        else bucket.get("chunk_count", {}).get("value", 0)
                    ),
                    "allowed_users": source.get("allowed_users", []),
                    "allowed_groups": source.get("allowed_groups", []),
                    "allowed_principal_labels": source.get("allowed_principal_labels", []),
                    "web_source_id": source.get("web_source_id"),
                    "web_page_id": source.get("web_page_id"),
                    "web_page_depth": source.get("web_page_depth"),
                    "web_child_count": source.get("web_child_count", 0),
                    "status": source.get("status", "active"),
                    "error": source.get("error", ""),
                }
            )
        return files