Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
22.7 KB
Raw Blame
"""Default OpenRAG docs ingestion / refresh / upgrade-reingest.

This module owns the bundled-docs onboarding flow: deciding whether to
ingest from a URL or from local files, choosing the Langflow or
direct-OpenRAG pipeline, detecting upstream content changes via HTTP
ETag/Last-Modified, and deleting stale chunks before reingestion.

Public entry points used outside this module:
- ingest_default_documents_when_ready  (api/settings.py)
- refresh_default_openrag_docs         (api/settings.py)
- _get_remote_docs_signature           (test_main_docs_signature.py)
"""

import hashlib
import os

import httpx

from config.paths import get_documents_path
from config.settings import (
    DEFAULT_DOCS_CRAWL_DEPTH,
    DEFAULT_DOCS_INGEST_SOURCE,
    DEFAULT_DOCS_URL,
    LANGFLOW_URL_INGEST_FLOW_ID,
    config_manager,
    get_index_name,
    get_openrag_config,
)
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient
from utils.version_utils import OPENRAG_VERSION

logger = get_logger(__name__)

# Files to exclude from startup ingestion
EXCLUDED_INGESTION_FILES = {"warmup_ocr.pdf"}
URL_INGEST_EXCLUDED_INGESTION_FILES = {"openrag-documentation.pdf"}


def _get_documents_dir():
    """Get the documents directory path, handling both Docker and local environments."""
    path = get_documents_path()
    logger.debug(f"Using documents path: {path}")
    return path


def _should_use_url_default_docs_ingest() -> bool:
    """Return whether default docs ingestion should use URL crawling."""
    return DEFAULT_DOCS_INGEST_SOURCE == "url" and bool(DEFAULT_DOCS_URL)


async def ingest_openrag_docs_when_ready(
    document_service,
    models_service,
    task_service,
    langflow_file_service,
    session_manager,
    jwt_token=None,
):
    """Ingest OpenRAG docs during onboarding."""
    use_url_ingest = _should_use_url_default_docs_ingest()
    task_id = None
    if use_url_ingest:
        try:
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_START
            )
            if get_openrag_config().knowledge.disable_ingest_with_langflow:
                task_id = await _ingest_default_documents_url(
                    task_service=task_service,
                    docs_url=DEFAULT_DOCS_URL,
                    crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
                    jwt_token=jwt_token,
                )
            else:
                logger.info(
                    "Ingesting default documents using Langflow",
                    docs_url=DEFAULT_DOCS_URL,
                )
                task_id = await _ingest_default_documents_url_langflow(
                    langflow_file_service=langflow_file_service,
                    session_manager=session_manager,
                    task_service=task_service,
                    docs_url=DEFAULT_DOCS_URL,
                    crawl_depth=DEFAULT_DOCS_CRAWL_DEPTH,
                    jwt_token=jwt_token,
                )
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_COMPLETE
            )
        except Exception as e:
            logger.error("Default URL documents ingestion failed", error=str(e))
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_URL_FAILED
            )
    return task_id


async def ingest_default_documents_when_ready(
    document_service,
    models_service,
    task_service,
    langflow_file_service,
    session_manager,
    jwt_token=None,
):
    """Ingest default OpenRAG docs during onboarding."""
    try:
        logger.info(
            "Ingesting default documents when ready",
            disable_langflow_ingest=get_openrag_config().knowledge.disable_ingest_with_langflow,
            ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
        )
        await TelemetryClient.send_event(
            Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_START
        )
        task_id = await ingest_openrag_docs_when_ready(
            document_service,
            models_service,
            task_service,
            langflow_file_service,
            session_manager,
            jwt_token=jwt_token,
        )

        base_dir = _get_documents_dir()
        if not os.path.isdir(base_dir):
            raise FileNotFoundError(f"Default documents directory not found: {base_dir}")

        excluded_files = set(EXCLUDED_INGESTION_FILES)
        if _should_use_url_default_docs_ingest():
            excluded_files.update(URL_INGEST_EXCLUDED_INGESTION_FILES)

        file_paths = [
            os.path.join(root, fn)
            for root, _, files in os.walk(base_dir)
            for fn in files
            if fn not in excluded_files
        ]

        if not file_paths:
            raise FileNotFoundError(f"No default documents found in {base_dir}")

        if get_openrag_config().knowledge.disable_ingest_with_langflow:
            new_task_id = await _ingest_default_documents_openrag(
                document_service,
                models_service,
                task_service,
                file_paths,
                connector_type="local",
                jwt_token=jwt_token,
            )
            task_id = new_task_id or task_id
        else:
            new_task_id = await _ingest_default_documents_langflow(
                langflow_file_service,
                session_manager,
                task_service,
                file_paths,
                connector_type="local",
                jwt_token=jwt_token,
            )
            task_id = new_task_id or task_id

        await TelemetryClient.send_event(
            Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_COMPLETE
        )

        return task_id

    except Exception as e:
        logger.error("Default documents ingestion failed", error=str(e))
        await TelemetryClient.send_event(
            Category.DOCUMENT_INGESTION, MessageId.ORB_DOC_DEFAULT_FAILED
        )
        raise


async def _ingest_default_documents_langflow(
    langflow_file_service,
    session_manager,
    task_service,
    file_paths,
    connector_type: str = "openrag_docs",
    jwt_token=None,
):
    """Ingest default documents using Langflow upload-ingest-delete pipeline."""

    logger.info(
        "Using Langflow ingestion pipeline for default documents",
        file_count=len(file_paths),
    )

    from session_manager import AnonymousUser

    anonymous_user = AnonymousUser()
    effective_jwt = jwt_token

    if not effective_jwt and session_manager:
        effective_jwt = session_manager.get_effective_jwt_token(anonymous_user.user_id, None)

    default_tweaks = {
        "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-WaE28": {
            "docs_metadata": [
                {"key": "owner", "value": None},
                {"key": "owner_name", "value": anonymous_user.name},
                {"key": "owner_email", "value": anonymous_user.email},
                {"key": "connector_type", "value": "openrag_docs"},
                {"key": "is_sample_data", "value": "true"},
            ]
        }
    }

    task_id = await task_service.create_langflow_upload_task(
        user_id=None,
        file_paths=file_paths,
        langflow_file_service=langflow_file_service,
        session_manager=session_manager,
        jwt_token=effective_jwt,
        owner_name=anonymous_user.name,
        owner_email=anonymous_user.email,
        session_id=None,
        tweaks=default_tweaks,
        settings=None,
        replace_duplicates=True,
        connector_type=connector_type,
        temp_file_paths=[],
        # The system refreshing its own ownerless sample documents, not a user
        # replacing someone else's — no user permission applies.
        allow_anonymous_delete=True,
    )

    logger.info(
        "Started Langflow ingestion task for default documents",
        task_id=task_id,
        file_count=len(file_paths),
    )
    return task_id


async def _ingest_default_documents_url_langflow(
    langflow_file_service,
    session_manager,
    task_service,
    docs_url: str,
    crawl_depth: int,
    jwt_token=None,
):
    """Ingest default URL docs using the Langflow URL ingestion pipeline."""
    if not docs_url:
        raise ValueError("DEFAULT_DOCS_URL is not configured")

    logger.info(
        "Using Langflow URL ingestion pipeline for default documents",
        docs_url=docs_url,
        crawl_depth=crawl_depth,
    )

    from session_manager import AnonymousUser

    anonymous_user = AnonymousUser()
    effective_jwt = jwt_token

    if not effective_jwt and session_manager:
        effective_jwt = session_manager.get_effective_jwt_token(anonymous_user.user_id, None)

    default_tweaks = {
        "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-dihKf": {
            "docs_metadata": [
                {"key": "owner", "value": None},
                {"key": "owner_name", "value": anonymous_user.name},
                {"key": "owner_email", "value": anonymous_user.email},
                {"key": "connector_type", "value": "openrag_docs"},
                {"key": "is_sample_data", "value": "true"},
            ]
        }
    }

    task_id = await task_service.create_langflow_url_upload_task(
        owner_user_id=None,
        docs_url=docs_url,
        crawl_depth=crawl_depth,
        langflow_file_service=langflow_file_service,
        session_manager=session_manager,
        jwt_token=effective_jwt,
        owner_name=anonymous_user.name,
        owner_email=anonymous_user.email,
        connector_type="openrag_docs",
        tweaks=default_tweaks,
    )

    logger.info(
        "Started Langflow URL ingestion task for default documents",
        task_id=task_id,
        docs_url=docs_url,
    )
    return task_id


async def _ingest_default_documents_url(
    task_service,
    docs_url: str,
    crawl_depth: int,
    jwt_token=None,
):
    """Ingest default docs from URL using OpenRAG ingestion logic (no Langflow)."""
    if not docs_url:
        raise ValueError("DEFAULT_DOCS_URL is not configured")

    from session_manager import AnonymousUser

    anonymous_user = AnonymousUser()

    logger.info(
        "Running default URL docs ingestion with OpenRAG processor",
        docs_url=docs_url,
        crawl_depth=crawl_depth,
    )

    task_id = await task_service.create_url_upload_task(
        owner_user_id=None,
        docs_url=docs_url,
        crawl_depth=crawl_depth,
        jwt_token=jwt_token,
        owner_name=anonymous_user.name,
        owner_email=anonymous_user.email,
        connector_type="openrag_docs",
        is_sample_data=True,
    )

    logger.info(
        "Started URL ingestion task for default documents",
        task_id=task_id,
        docs_url=docs_url,
    )
    return task_id


async def _delete_existing_default_docs(session_manager, connector_type: str, jwt_token=None):
    """Delete previously ingested default OpenRAG docs before reingestion."""
    from config.settings import clients
    from session_manager import AnonymousUser
    from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids

    write_client = clients.opensearch
    if write_client is None:
        raise RuntimeError("Backend OpenSearch write client is unavailable")

    if session_manager is None:
        logger.warning(
            "Session manager unavailable; skipping default docs cleanup before reingestion"
        )
        return

    anonymous_user = AnonymousUser()
    effective_jwt = jwt_token
    if not effective_jwt and session_manager:
        effective_jwt = session_manager.get_effective_jwt_token(anonymous_user.user_id, None)

    opensearch_client = session_manager.get_user_opensearch_client(
        anonymous_user.user_id, effective_jwt
    )
    delete_query = {
        "query": {
            "bool": {
                "should": [
                    # URL-based default docs are ingested as system_default and
                    # owned by the anonymous onboarding user.
                    {
                        "bool": {
                            "must": [
                                {"term": {"connector_type": connector_type}},
                                {"term": {"owner_email": anonymous_user.email}},
                            ]
                        }
                    },
                ],
                "minimum_should_match": 1,
            }
        }
    }
    index_name = get_index_name()
    document_ids = await collect_visible_document_ids(
        opensearch_client,
        index=index_name,
        query=delete_query["query"],
    )
    deleted_chunks = await delete_document_ids(
        write_client,
        index=index_name,
        document_ids=document_ids,
    )
    logger.info(
        "Deleted existing default OpenRAG docs before reingestion",
        deleted_chunks=deleted_chunks,
    )


async def _reingest_default_docs_on_upgrade_if_needed(
    document_service,
    models_service,
    task_service,
    langflow_file_service,
    session_manager,
    jwt_token=None,
):
    """Reingest default OpenRAG docs once when app version changes."""
    config = get_openrag_config()

    previous_version = config.onboarding.openrag_docs_ingested_version
    current_version = OPENRAG_VERSION
    should_reingest = bool(previous_version) and previous_version != current_version

    # Legacy installs may not have a stored docs ingestion version.
    # Use the presence of the OpenRAG docs filter as the signal that docs were
    # already onboarded, independent of whether config.edited is set.
    if not previous_version and config.onboarding.openrag_docs_filter_id:
        should_reingest = True

    if not should_reingest:
        return False

    logger.info(
        "Detected OpenRAG upgrade; reingesting default docs",
        previous_version=previous_version,
        current_version=current_version,
    )
    await _delete_existing_default_docs(
        session_manager, connector_type="openrag_docs", jwt_token=jwt_token
    )
    await ingest_openrag_docs_when_ready(
        document_service,
        models_service,
        task_service,
        langflow_file_service,
        session_manager,
        jwt_token=jwt_token,
    )
    config.onboarding.openrag_docs_ingested_version = current_version
    if _should_use_url_default_docs_ingest():
        # Refresh signature metadata after upgrade reingestion so startup
        # signature checks don't trigger an immediate duplicate ingest.
        config.onboarding.openrag_docs_remote_signature = await _get_remote_docs_signature(
            DEFAULT_DOCS_URL
        )
    else:
        config.onboarding.openrag_docs_remote_signature = None
    if not config_manager.save_config_file(config):
        logger.warning(
            "Default docs were reingested but failed to persist metadata",
            current_version=current_version,
            signature=config.onboarding.openrag_docs_remote_signature,
        )
    return True


async def _get_remote_docs_signature(docs_url: str):
    """Get a signature for remote docs to detect content updates."""
    try:
        async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
            head_response = await client.head(docs_url)
            if head_response.status_code >= 400:
                get_response = await client.get(docs_url)
                if get_response.status_code >= 400:
                    logger.warning(
                        "Failed to fetch remote docs signature",
                        docs_url=docs_url,
                        status_code=get_response.status_code,
                    )
                    return None
                return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()

            etag = (head_response.headers.get("etag") or "").strip()
            last_modified = (head_response.headers.get("last-modified") or "").strip()
            if etag:
                # Prefer ETag when available: it is typically the strongest
                # cache validator and stays stable if extra cache headers
                # appear/disappear without content changes.
                return f"etag={etag}"
            if last_modified:
                return f"last_modified={last_modified}"

            # HEAD has no body. If cache headers are missing, fetch the page body.
            get_response = await client.get(docs_url)
            if get_response.status_code >= 400:
                logger.warning(
                    "Failed to fetch remote docs signature body fallback",
                    docs_url=docs_url,
                    status_code=get_response.status_code,
                )
                return None
            return hashlib.sha256(get_response.text.encode("utf-8")).hexdigest()
    except Exception as e:
        logger.error(
            "Unable to retrieve remote docs signature",
            docs_url=docs_url,
            error=str(e),
        )
        return None


async def refresh_default_openrag_docs(
    document_service,
    models_service,
    task_service,
    langflow_file_service,
    session_manager,
    force: bool = False,
    reason: str = "startup",
    jwt_token=None,
):
    """Refresh OpenRAG docs if remote content changed or when forced."""
    await TelemetryClient.send_event(
        Category.DOCUMENT_INGESTION,
        MessageId.ORB_DOC_REFRESH_START,
        metadata={"reason": reason, "force": force},
    )
    try:
        if not _should_use_url_default_docs_ingest():
            logger.info(
                "Skipping OpenRAG docs refresh: URL ingestion is not active",
                ingest_source=DEFAULT_DOCS_INGEST_SOURCE,
                disable_langflow_ingest=get_openrag_config().knowledge.disable_ingest_with_langflow,
                has_url_ingest_flow_id=bool(LANGFLOW_URL_INGEST_FLOW_ID),
                has_docs_url=bool(DEFAULT_DOCS_URL),
            )
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION,
                MessageId.ORB_DOC_REFRESH_SKIPPED,
                metadata={
                    "reason": reason,
                    "force": force,
                    "skip_reason": "url_ingestion_inactive",
                },
            )
            return False

        config = get_openrag_config()
        if not config.edited:
            logger.info("Skipping OpenRAG docs refresh: onboarding not completed")
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION,
                MessageId.ORB_DOC_REFRESH_SKIPPED,
                metadata={
                    "reason": reason,
                    "force": force,
                    "skip_reason": "onboarding_not_completed",
                },
            )
            return False

        signature = await _get_remote_docs_signature(DEFAULT_DOCS_URL)
        if not signature and not force:
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION,
                MessageId.ORB_DOC_REFRESH_SKIPPED,
                metadata={
                    "reason": reason,
                    "force": force,
                    "skip_reason": "signature_unavailable",
                },
            )
            return False

        previous_signature = config.onboarding.openrag_docs_remote_signature
        should_refresh = force or (signature is not None and signature != previous_signature)
        if not should_refresh:
            logger.info(
                "OpenRAG docs refresh skipped: remote signature unchanged",
                signature=signature,
            )
            await TelemetryClient.send_event(
                Category.DOCUMENT_INGESTION,
                MessageId.ORB_DOC_REFRESH_SKIPPED,
                metadata={
                    "reason": reason,
                    "force": force,
                    "skip_reason": "signature_unchanged",
                },
            )
            return False

        logger.info(
            "Refreshing default OpenRAG docs",
            reason=reason,
            force=force,
            previous_signature=previous_signature,
            new_signature=signature,
        )
        await _delete_existing_default_docs(
            session_manager, connector_type="openrag_docs", jwt_token=jwt_token
        )
        await ingest_openrag_docs_when_ready(
            document_service,
            models_service,
            task_service,
            langflow_file_service,
            session_manager,
            jwt_token=jwt_token,
        )
        config.onboarding.openrag_docs_ingested_version = OPENRAG_VERSION
        # Keep docs version/signature metadata consistent after a refresh.
        # If signature retrieval failed, persist None explicitly instead of
        # leaving a stale previous signature value.
        config.onboarding.openrag_docs_remote_signature = signature
        if not config_manager.save_config_file(config):
            logger.warning(
                "OpenRAG docs refreshed but failed to persist metadata",
                version=config.onboarding.openrag_docs_ingested_version,
                signature=config.onboarding.openrag_docs_remote_signature,
            )
        await TelemetryClient.send_event(
            Category.DOCUMENT_INGESTION,
            MessageId.ORB_DOC_REFRESH_COMPLETE,
            metadata={"reason": reason, "force": force},
        )
        return True
    except Exception as e:
        await TelemetryClient.send_event(
            Category.DOCUMENT_INGESTION,
            MessageId.ORB_DOC_REFRESH_FAILED,
            metadata={
                "reason": reason,
                "force": force,
                "error_type": type(e).__name__,
            },
        )
        raise


async def _ingest_default_documents_openrag(
    document_service,
    models_service,
    task_service,
    file_paths,
    connector_type: str = "openrag_docs",
    jwt_token=None,
):
    """Ingest default documents using traditional OpenRAG processor."""
    logger.info(
        "Using traditional OpenRAG ingestion for default documents",
        file_count=len(file_paths),
    )

    from models.processors import DocumentFileProcessor
    from session_manager import AnonymousUser

    anonymous_user = AnonymousUser()

    processor = DocumentFileProcessor(
        document_service,
        models_service=models_service,
        owner_user_id=None,
        jwt_token=jwt_token,
        owner_name=anonymous_user.name,
        owner_email=anonymous_user.email,
        is_sample_data=True,
        connector_type=connector_type,
        # The system ingesting its own ownerless sample documents, not a user
        # replacing someone else's — no user permission applies.
        allow_anonymous_delete=True,
    )

    task_id = await task_service.create_custom_task("anonymous", file_paths, processor)
    logger.info(
        "Started traditional OpenRAG ingestion task",
        task_id=task_id,
        file_count=len(file_paths),
    )
    return task_id