Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
18.7 KB
Raw Blame
"""Shared backend-owned OpenSearch document indexing.

Langflow can generate chunks and embeddings, but it must not hold credentials
that can write arbitrary documents. This writer is the single backend path for
indexing chunks into the documents index.
"""

from __future__ import annotations

import asyncio
import datetime
import hashlib
import json
from dataclasses import dataclass, field
from typing import Any

from services.search_cache import invalidate_corpus_caches
from utils.embedding_fields import ensure_embedding_field_exists, get_embedding_space_id
from utils.embeddings import create_index_body
from utils.group_acl import unique_acl_principal_labels, unique_acl_principals
from utils.logging_config import get_logger

logger = get_logger(__name__)

# A bulk() call executes every item independently. Thus, if one item hits a
# transient condition (cluster under load, shard unavailable) the whole response
# gets errors: true even though the rest of the document chunks wrote
# successfully. This makes the entire file look like it "failed" in task
# counters even when it's actually indexed and visible. But every op is an
# idempotent upsert so retries are safe — only the items still failing are
# resent, not the whole body.
_BULK_RETRYABLE_ERROR_TYPES = frozenset(
    {
        "es_rejected_execution_exception",
        "process_cluster_event_timeout_exception",
        "unavailable_shards_exception",
        "timeout_exception",
        # version_conflicts are intentionally excluded — they result from
        # concurrent writes to the same document ID, not transient failures.
        # Retrying would just hit the same conflict again and waste the budget.
    }
)
_MAX_BULK_RETRIES = 2
_BULK_RETRY_DELAY_SECONDS = 0.5


@dataclass
class DocumentIndexContext:
    document_id: str
    mimetype: str
    embedding_model: str
    embedding_provider: str | None = None
    filename: str | None = None
    owner: str | None = None
    owner_name: str | None = None
    owner_email: str | None = None
    file_size: int | None = None
    connector_type: str | None = None
    source_url: str | None = None
    connector_file_id: str | None = None
    allowed_users: list[str] = field(default_factory=list)
    allowed_groups: list[str] = field(default_factory=list)
    allowed_principals: list[str] = field(default_factory=list)
    allowed_principal_labels: list[dict[str, Any]] = field(default_factory=list)
    ingest_run_id: str | None = None
    is_sample_data: bool = False
    index_name: str | None = None
    parser: str | None = None
    chunk_size: int | None = None
    chunk_overlap: int | None = None
    indexed_time: str | None = None
    record_kind: str | None = None
    web_source_id: str | None = None
    web_page_id: str | None = None
    web_page_depth: int | None = None
    root_source_url: str | None = None
    canonical_url: str | None = None


@dataclass
class DocumentIndexChunk:
    chunk_id: str
    text: str
    vector: list[float]
    page: int | None = None
    metadata: dict[str, Any] = field(default_factory=dict)


class DocumentIndexWriter:
    """Write document chunks with a trusted backend OpenSearch client."""

    def __init__(self, opensearch_client: Any | None = None):
        self.opensearch_client = opensearch_client

    def _get_write_client(self) -> Any:
        from config.settings import clients

        client = self.opensearch_client or clients.opensearch
        if client is None:
            raise RuntimeError(
                "Backend OpenSearch write client is unavailable; cannot index document chunks"
            )
        return client

    async def index_chunks(
        self,
        context: DocumentIndexContext,
        chunks: list[DocumentIndexChunk],
        *,
        final: bool = False,
        refresh: bool | str = False,
    ) -> dict[str, Any]:
        """Index one batch of chunks.

        Repeated calls with the same chunk ids in the same ownership scope are
        idempotent because the write operation is an index/upsert.
        """
        from config.settings import get_index_name

        if not chunks:
            if final:
                await self._refresh(context.index_name or get_index_name())
            return {"indexed_chunks": 0, "ingest_run_id": context.ingest_run_id}

        first_vector = chunks[0].vector
        if not first_vector:
            raise ValueError("Cannot index chunks with empty embeddings")

        dimensions = len(first_vector)
        embedding_space_id = self._embedding_space_id(context)
        client = self._get_write_client()
        index_name = context.index_name or get_index_name()
        embedding_field = await self._ensure_index_and_embedding_field(
            client,
            index_name=index_name,
            embedding_model=context.embedding_model,
            embedding_provider=context.embedding_provider,
            embedding_space_id=embedding_space_id,
            dimensions=dimensions,
        )

        now = context.indexed_time or datetime.datetime.now(datetime.UTC).isoformat()
        bulk_body: list[dict[str, Any]] = []
        for chunk in chunks:
            if len(chunk.vector) != dimensions:
                raise ValueError(
                    "Embedding dimension mismatch in batch: "
                    f"expected {dimensions}, got {len(chunk.vector)} for {chunk.chunk_id}"
                )
            bulk_body.append(
                {
                    "index": {
                        "_index": index_name,
                        "_id": self._scoped_chunk_id(context, chunk.chunk_id),
                    }
                }
            )
            bulk_body.append(
                self._build_chunk_document(
                    context=context,
                    chunk=chunk,
                    embedding_field=embedding_field,
                    indexed_time=now,
                )
            )

        try:
            result = await self._bulk_with_retry(client, bulk_body, refresh=refresh)
            self._raise_for_bulk_errors(result)
            if final:
                await self._refresh(index_name)
        finally:
            # Even a failed batch may have written some chunks.
            invalidate_corpus_caches()

        logger.info(
            "Indexed document chunks",
            index_name=index_name,
            document_id=context.document_id,
            ingest_run_id=context.ingest_run_id,
            chunk_count=len(chunks),
            final=final,
        )
        return {
            "indexed_chunks": len(chunks),
            "ingest_run_id": context.ingest_run_id,
            "document_id": context.document_id,
        }

    @staticmethod
    def _scoped_chunk_id(context: DocumentIndexContext, chunk_id: str) -> str:
        """Keep idempotent chunk upserts isolated to one ownership scope."""
        scope = "shared" if context.owner is None else f"owner:{context.owner}"
        scope_digest = hashlib.sha256(scope.encode("utf-8")).hexdigest()[:24]
        return f"{scope_digest}_{chunk_id}"

    async def delete_ingest_run(self, ingest_run_id: str, *, index_name: str | None = None) -> int:
        """Delete partially indexed chunks for a failed callback run."""
        if not ingest_run_id:
            return 0
        from config.settings import get_index_name

        client = self._get_write_client()
        resolved_index = index_name or get_index_name()
        body = {"query": {"term": {"ingest_run_id": ingest_run_id}}}
        response = await client.delete_by_query(
            index=resolved_index,
            body=body,
            refresh=True,
            conflicts="proceed",
        )
        deleted = int(response.get("deleted", 0)) if isinstance(response, dict) else 0
        invalidate_corpus_caches()
        logger.info(
            "Deleted failed ingest run chunks",
            index_name=resolved_index,
            ingest_run_id=ingest_run_id,
            deleted=deleted,
        )
        return deleted

    async def _ensure_index_and_embedding_field(
        self,
        client: Any,
        *,
        index_name: str,
        embedding_model: str,
        embedding_provider: str | None,
        embedding_space_id: str,
        dimensions: int,
    ) -> str:
        if not await client.indices.exists(index=index_name):
            await client.indices.create(
                index=index_name,
                body=await create_index_body(
                    embedding_model,
                    dimensions,
                    embedding_provider=embedding_provider,
                    embedding_space_id=embedding_space_id,
                ),
            )
        return await ensure_embedding_field_exists(
            client,
            embedding_space_id,
            index_name,
            dimensions,
        )

    @staticmethod
    def _embedding_space_id(context: DocumentIndexContext) -> str:
        """Use exact provider provenance when present; preserve legacy ids otherwise."""
        if context.embedding_provider:
            return get_embedding_space_id(context.embedding_provider, context.embedding_model)
        return context.embedding_model

    def _build_chunk_document(
        self,
        *,
        context: DocumentIndexContext,
        chunk: DocumentIndexChunk,
        embedding_field: str,
        indexed_time: str,
    ) -> dict[str, Any]:
        metadata = self._normalized_metadata(chunk.metadata)
        document_id = context.document_id or str(metadata.get("document_id") or chunk.chunk_id)
        filename = context.filename or str(metadata.get("filename") or "")
        mimetype = context.mimetype or str(metadata.get("mimetype") or "")

        doc: dict[str, Any] = {
            "document_id": document_id,
            "filename": filename,
            "mimetype": mimetype,
            "page": chunk.page if chunk.page is not None else metadata.get("page", 0),
            "text": chunk.text,
            embedding_field: chunk.vector,
            "embedding_model": context.embedding_model,
            "embedding_dimensions": len(chunk.vector),
            "file_size": context.file_size
            if context.file_size is not None
            else metadata.get("file_size"),
            "connector_type": context.connector_type or metadata.get("connector_type") or "local",
            "source_url": context.source_url or metadata.get("source_url") or "",
            "allowed_users": list(context.allowed_users),
            "allowed_groups": list(context.allowed_groups),
            "allowed_principals": unique_acl_principals(context.allowed_principals),
            "allowed_principal_labels": unique_acl_principal_labels(
                context.allowed_principal_labels
            ),
            "indexed_time": indexed_time,
            "metadata": metadata.get("metadata", {}),
        }

        if context.embedding_provider:
            doc["embedding_provider"] = context.embedding_provider.strip().lower()
            doc["embedding_space_id"] = self._embedding_space_id(context)

        parser = context.parser or metadata.get("parser")
        if parser:
            doc["parser"] = parser

        for field_name in ("chunk_size", "chunk_overlap"):
            context_value = getattr(context, field_name)
            value = context_value if context_value is not None else metadata.get(field_name)
            if value is None:
                continue
            try:
                doc[field_name] = int(value)
            except (TypeError, ValueError):
                # Skip assignment if coercion fails to avoid type conflicts
                pass

        if context.owner is not None:
            doc["owner"] = context.owner
        if context.owner_name is not None:
            doc["owner_name"] = context.owner_name
        if context.owner_email is not None:
            doc["owner_email"] = context.owner_email
        if context.ingest_run_id:
            doc["ingest_run_id"] = context.ingest_run_id
        if context.connector_file_id:
            doc["connector_file_id"] = context.connector_file_id
        elif metadata.get("connector_file_id"):
            doc["connector_file_id"] = metadata["connector_file_id"]
        if context.is_sample_data:
            doc["is_sample_data"] = "true"
        for field_name in (
            "record_kind",
            "web_source_id",
            "web_page_id",
            "web_page_depth",
            "root_source_url",
            "canonical_url",
        ):
            value = getattr(context, field_name)
            if value is not None:
                doc[field_name] = value
        for time_field in ("created_time", "modified_time"):
            if metadata.get(time_field):
                doc[time_field] = metadata[time_field]

        return doc

    @staticmethod
    def _normalized_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
        normalized = dict(metadata or {})
        for key in (
            "allowed_users",
            "allowed_groups",
            "allowed_principals",
            "allowed_principal_labels",
        ):
            value = normalized.get(key)
            if isinstance(value, str):
                try:
                    parsed = json.loads(value)
                except (TypeError, json.JSONDecodeError):
                    continue
                if isinstance(parsed, list):
                    normalized[key] = parsed
        if "filesize" in normalized and "file_size" not in normalized:
            normalized["file_size"] = normalized["filesize"]
        return normalized

    @staticmethod
    def _item_error(item: Any) -> dict[str, Any] | None:
        action = item.get("index") or item.get("create") or item.get("update") or item
        return action.get("error")

    @classmethod
    def _item_error_is_retryable(cls, item: Any) -> bool:
        error = cls._item_error(item)
        # When there is no error the item succeeded — we treat it as retryable
        # so a successful item never blocks a retry when this runs over the
        # still_pending list (which only ever holds items that actually failed).
        return not error or error.get("type") in _BULK_RETRYABLE_ERROR_TYPES

    async def _bulk_with_retry(
        self, client: Any, bulk_body: list[dict[str, Any]], *, refresh: bool | str
    ) -> Any:
        """Only retry the items that actually failed, not the whole bulk body.

        Each entry in the bulk response lines up by position with the
        action/document pairs we sent. If we resend everything on retry we
        end up re-submitting pairs that already succeeded — and under heavy
        load a different subset can fail each time, meaning the file still
        gets marked failed even though every pair succeeded at some point.
        Instead we track each pair by its original position and only rebuild
        the retry body from the ones still failing.
        """
        if len(bulk_body) % 2 != 0:
            raise ValueError(
                f"bulk_body must contain action/document pairs; got {len(bulk_body)} elements"
            )
        pairs = [tuple(bulk_body[i : i + 2]) for i in range(0, len(bulk_body), 2)]
        pending_indices = list(range(len(pairs)))
        final_items: list[Any] = [None] * len(pairs)

        for attempt in range(_MAX_BULK_RETRIES + 1):
            retry_body = [part for i in pending_indices for part in pairs[i]]
            result = await client.bulk(body=retry_body, refresh=refresh)
            if not isinstance(result, dict):
                return result

            items = result.get("items", [])

            # Guard against a bad or cut-short response — the server should
            # return exactly one item for each request we sent. If the list
            # is short or empty we can't safely match results back to the
            # right slots, so we treat it as an error rather than leaving
            # gaps filled with None.
            if len(items) != len(pending_indices):
                raise RuntimeError(
                    f"OpenSearch bulk response returned {len(items)} items for "
                    f"{len(pending_indices)} requests — malformed response"
                )

            still_pending = []
            for original_index, item in zip(pending_indices, items, strict=True):
                final_items[original_index] = item
                if self._item_error(item):
                    still_pending.append(original_index)

            if not still_pending:
                # Keep the top-level error flag as-is — if the server
                # said errors: true but none of the individual items show
                # an error, we still want to surface that rather than
                # quietly calling it a success.
                return {**result, "items": final_items, "errors": result.get("errors", False)}

            all_retryable = all(
                self._item_error_is_retryable(final_items[i]) for i in still_pending
            )
            if not all_retryable or attempt == _MAX_BULK_RETRIES:
                return {**result, "items": final_items, "errors": True}

            logger.warning(
                "Retrying OpenSearch bulk write after transient error",
                attempt=attempt + 1,
                remaining_items=len(still_pending),
            )
            await asyncio.sleep(_BULK_RETRY_DELAY_SECONDS * (attempt + 1))
            pending_indices = still_pending

        return {"items": final_items, "errors": True}

    @staticmethod
    def _raise_for_bulk_errors(result: Any) -> None:
        if not isinstance(result, dict) or not result.get("errors"):
            return
        # Keep only the items that actually failed (a bulk response interleaves
        # successes and failures) and carry their full OpenSearch error body so
        # the cause — e.g. a mapper_parsing_exception naming the offending field
        # — survives in the raised error. The caller logs this once with request
        # context; this helper only raises so the failure isn't logged twice.
        failures = []
        for item in result.get("items", []):
            action = item.get("index") or item.get("create") or item.get("update") or item
            if not action.get("error"):
                continue
            failures.append(
                {
                    "id": action.get("_id"),
                    "status": action.get("status"),
                    "error": action.get("error"),
                }
            )
            if len(failures) >= 5:
                break
        if not failures:
            # `errors` was set but no item carried an error body (rare/contradictory);
            # fall back to the first few raw items so the message keeps some detail.
            failures = result.get("items", [])[:3]
        raise RuntimeError(f"OpenSearch bulk indexing failed: {failures}")

    async def _refresh(self, index_name: str) -> None:
        client = self._get_write_client()
        await client.indices.refresh(index=index_name)