Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
3.5 KB
Raw Blame
from utils.embedding_fields import build_knn_vector_field, get_embedding_field_name
from utils.logging_config import get_logger

logger = get_logger(__name__)


async def create_index_body(
    embedding_model: str | None = None,
    embedding_dimensions: int | None = None,
    *,
    embedding_provider: str | None = None,
    embedding_space_id: str | None = None,
) -> dict:
    """Create a static index body configuration.

    Returns:
        OpenSearch index body configuration
    """
    from config.embedding_constants import get_declared_default_embedding_model
    from config.settings import (
        ACL_PRINCIPAL_LABELS_MAPPING,
        OPENSEARCH_NUMBER_OF_REPLICAS,
        OPENSEARCH_NUMBER_OF_SHARDS,
        VECTOR_DIM,
        get_openrag_config,
    )

    config = get_openrag_config()
    resolved_embedding_model = (
        embedding_model
        or config.knowledge.embedding_model
        or get_declared_default_embedding_model(config.knowledge.embedding_provider)
    )

    properties = {
        "document_id": {"type": "keyword"},
        "filename": {"type": "keyword"},
        "mimetype": {"type": "keyword"},
        "page": {"type": "integer"},
        "text": {"type": "text"},
        # Legacy field - kept for backward compatibility and for clusters where
        # Langflow cannot perform mapping updates with a DLS-filtered JWT.
        "chunk_embedding": build_knn_vector_field(VECTOR_DIM),
        # Track which embedding model was used for this chunk
        "embedding_model": {"type": "keyword"},
        "embedding_provider": {"type": "keyword"},
        "embedding_space_id": {"type": "keyword"},
        "embedding_dimensions": {"type": "integer"},
        "source_url": {"type": "keyword"},
        "connector_type": {"type": "keyword"},
        "record_kind": {"type": "keyword"},
        "web_source_id": {"type": "keyword"},
        "web_page_id": {"type": "keyword"},
        "web_page_depth": {"type": "integer"},
        "web_child_count": {"type": "integer"},
        "root_source_url": {"type": "keyword"},
        "canonical_url": {"type": "keyword"},
        "status": {"type": "keyword"},
        "error": {"type": "text"},
        "ingest_run_id": {"type": "keyword"},
        # Object-store entity tag of the source file, for sync change detection.
        "content_etag": {"type": "keyword"},
        "owner": {"type": "keyword"},
        "owner_email": {"type": "keyword"},
        "allowed_users": {"type": "keyword"},
        "allowed_groups": {"type": "keyword"},
        "allowed_principals": {"type": "keyword"},
        "allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING,
        "created_time": {"type": "date"},
        "modified_time": {"type": "date"},
        "indexed_time": {"type": "date"},
        "metadata": {"type": "object"},
    }

    if embedding_dimensions:
        field_identity = embedding_space_id
        if not field_identity and embedding_provider:
            from utils.embedding_fields import get_embedding_space_id

            field_identity = get_embedding_space_id(embedding_provider, resolved_embedding_model)
        properties[get_embedding_field_name(field_identity or resolved_embedding_model)] = (
            build_knn_vector_field(embedding_dimensions)
        )

    return {
        "settings": {
            "index": {"knn": True},
            "number_of_shards": OPENSEARCH_NUMBER_OF_SHARDS,
            "number_of_replicas": OPENSEARCH_NUMBER_OF_REPLICAS,
        },
        "mappings": {"properties": properties},
    }