Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
5.7 KB
Raw Blame
from collections.abc import Iterable

from opensearchpy.exceptions import NotFoundError

from services.search_cache import invalidate_corpus_caches
from utils.logging_config import get_logger

logger = get_logger(__name__)


async def collect_visible_document_ids(
    opensearch_client,
    *,
    index: str,
    query: dict,
    source: bool | list[str] = False,
    page_size: int = 1000,
    scroll_ttl: str = "2m",
) -> list[str]:
    """Collect visible document IDs for a query using the caller's OpenSearch client."""
    search_body = {"query": query, "size": page_size, "_source": source}
    response = await opensearch_client.search(
        index=index,
        body=search_body,
        scroll=scroll_ttl,
    )
    scroll_id = response.get("_scroll_id")
    document_ids: list[str] = []

    try:
        while True:
            hits = response.get("hits", {}).get("hits", [])
            document_ids.extend(hit["_id"] for hit in hits if hit.get("_id"))
            if not hits or len(hits) < page_size or not scroll_id:
                break
            response = await opensearch_client.scroll(scroll_id=scroll_id, scroll=scroll_ttl)
            scroll_id = response.get("_scroll_id") or scroll_id
    finally:
        if scroll_id and hasattr(opensearch_client, "clear_scroll"):
            try:
                await opensearch_client.clear_scroll(scroll_id=scroll_id)
            except Exception as e:
                logger.debug("Failed to clear OpenSearch scroll context", error=str(e))

    return document_ids


async def delete_document_ids(
    opensearch_client,
    *,
    index: str,
    document_ids: Iterable[str],
    refresh: bool = True,
) -> int:
    """Delete concrete OpenSearch document IDs through the caller's client.

    delete_by_query is silently no-opped under DLS / certain security plugins
    (returns deleted:N but leaves the docs in place). Single deletes keyed on
    the primary _id are reliable, so enumerate visible IDs first and then issue
    a delete per ID.
    """
    deleted_count = 0
    try:
        for document_id in document_ids:
            try:
                result = await opensearch_client.delete(
                    index=index,
                    id=document_id,
                    refresh=refresh,
                )
            except NotFoundError:
                continue

            if result.get("result") == "deleted":
                deleted_count += 1
    finally:
        if deleted_count:
            invalidate_corpus_caches()

    return deleted_count


class BulkDeleteError(RuntimeError):
    """A bulk delete failed part-way; ``deleted_count`` holds confirmed deletions.

    Only deletions acknowledged by a successful ``_bulk`` response are counted.
    If a request itself failed, any deletions it may have applied are unknown
    and are not included.
    """

    def __init__(self, message: str, deleted_count: int = 0):
        super().__init__(message)
        self.deleted_count = deleted_count


async def bulk_delete_document_ids(
    opensearch_client,
    *,
    index: str,
    document_ids: Iterable[str],
    refresh: bool = True,
    batch_size: int = 500,
) -> int:
    """Delete concrete OpenSearch document IDs using batched ``_bulk`` requests.

    Same ID-keyed delete semantics as :func:`delete_document_ids` (so it stays
    DLS-safe), but one request per ``batch_size`` IDs instead of one per ID, and
    a single index refresh at the end instead of one per delete. Intended for
    user-facing deletes of many chunks, where per-ID round trips and per-delete
    refreshes dominate latency.

    When ``refresh`` is true the index is refreshed once after any confirmed
    deletions, including before a failure is propagated. A per-request
    ``refresh=true`` would only cover the shards that request touched, so it is
    not used. A failed refresh is logged and never masks the delete result or
    the original error; corpus caches are invalidated regardless.

    Missing documents (404) are ignored. Any other per-item failure is raised as
    a :class:`BulkDeleteError` after all batches have been attempted. A failed
    request stops further batches and is also raised as a ``BulkDeleteError``
    (chained to the original exception). Either way, ``deleted_count`` on the
    error carries the deletions confirmed so far.

    Raises ``ValueError`` if ``batch_size`` is not positive.
    """
    if batch_size <= 0:
        raise ValueError("batch_size must be > 0")
    ids = list(document_ids)
    deleted_count = 0
    first_error: str | None = None
    try:
        for start in range(0, len(ids), batch_size):
            batch = ids[start : start + batch_size]
            try:
                response = await opensearch_client.bulk(
                    body=[{"delete": {"_index": index, "_id": doc_id}} for doc_id in batch],
                    refresh=False,
                )
            except Exception as e:
                raise BulkDeleteError(str(e), deleted_count) from e
            for item in response.get("items", []):
                action = item.get("delete", {})
                if action.get("result") == "deleted":
                    deleted_count += 1
                elif action.get("status") != 404 and first_error is None:
                    first_error = str(action.get("error") or action.get("status"))
    finally:
        if deleted_count:
            try:
                if refresh:
                    await opensearch_client.indices.refresh(index=index)
            except Exception as e:
                logger.warning("Failed to refresh index after bulk delete", error=str(e))
            finally:
                invalidate_corpus_caches()

    if first_error is not None:
        raise BulkDeleteError(f"Bulk delete failed for some chunks: {first_error}", deleted_count)
    return deleted_count