Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
2.5 KB
Raw Blame
"""Shared filename-existence checks against the documents index.

Duplicate-filename decisions happen at several altitudes — the UI pre-check
endpoints, the API sync pre-filters, and the per-file processor backstop.
They must share one query semantic or the layers can disagree about what
counts as a duplicate. Every caller goes through this module: one terms
aggregation over exact ``filename`` keywords, with alias expansion via
``get_filename_aliases`` where a single logical filename is being checked.

The aggregation matters: collecting filenames from search *hits* undercounts
when a many-chunk document crowds other matches out of the hits window.
"""

from collections.abc import Iterable

from utils.file_utils import get_filename_aliases
from utils.opensearch_queries import build_existing_filenames_agg_body

# OpenSearch rejects a terms query carrying more than `index.max_terms_count`
# values (default 65_536). Whole-container duplicate checks feed this function
# one candidate per remote blob — times the alias expansion — so a large bucket
# can exceed that on its own. Candidates go out in batches instead.
FILENAME_LOOKUP_BATCH_SIZE = 1024


async def find_existing_filenames(
    candidates: Iterable[str],
    opensearch_client,
    index: str,
) -> set[str]:
    """Return the subset of exact candidate filenames that have at least one
    indexed chunk visible to this client.

    Callers own alias expansion (pass every variant that should count) and
    error handling — OpenSearch exceptions propagate.
    """
    unique = sorted({c for c in candidates if c})
    if not unique:
        return set()
    found: set[str] = set()
    for start in range(0, len(unique), FILENAME_LOOKUP_BATCH_SIZE):
        batch = unique[start : start + FILENAME_LOOKUP_BATCH_SIZE]
        response = await opensearch_client.search(
            index=index,
            body=build_existing_filenames_agg_body(batch),
        )
        buckets = response.get("aggregations", {}).get("filenames", {}).get("buckets", [])
        found.update(bucket["key"] for bucket in buckets if bucket.get("key"))
    return found


async def filename_exists(
    filename: str,
    opensearch_client,
    index: str,
) -> bool:
    """True when this filename — or any of its ingestion aliases — has indexed
    chunks visible to this client."""
    aliases = get_filename_aliases(filename)
    if not aliases:
        return False
    return bool(await find_existing_filenames(aliases, opensearch_client, index))