Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
24.2 KB
Raw Blame
from typing import Any

from opensearchpy.exceptions import NotFoundError

from config.settings import get_index_name
from services.search_cache import invalidate_corpus_caches
from utils.file_utils import clean_connector_filename
from utils.logging_config import get_logger

from .base import (
    CONTENT_ETAG_METADATA_KEY,
    BaseConnector,
    ConnectorDocument,
    normalize_etag,
)
from .connection_manager import ConnectionManager

logger = get_logger(__name__)


def _next_page_token(file_list: dict[str, Any]) -> str | None:
    """Read the continuation token out of a ``list_files`` result.

    Every connector in this repo returns ``next_page_token``; SharePoint is the
    one that populates it with a real value (a Graph ``$skiptoken``). This used
    to read ``nextPageToken`` only — a key no connector emits — so the token was
    always None and paging stopped after the first call. Both spellings are
    accepted now so a connector written against either convention still pages.
    """
    return file_list.get("next_page_token") or file_list.get("nextPageToken")


class ConnectorService:
    """Service to manage document connectors and process files"""

    def __init__(
        self,
        patched_async_client=None,
        embed_model: str = "",
        index_name: str = "",
        task_service=None,
        session_manager=None,
        models_service=None,
        document_service=None,
        docling_service=None,
        flows_service=None,
        langflow_service=None,
    ):
        self.clients = patched_async_client
        self.embed_model = embed_model
        self.index_name = index_name
        self.task_service = task_service
        self.session_manager = session_manager
        self.connection_manager = ConnectionManager()
        self.models_service = models_service
        self.document_service = document_service
        self.docling_service = docling_service
        self.flows_service = flows_service
        self.langflow_service = langflow_service

    async def initialize(self):
        """Initialize the service by loading existing connections"""
        await self.connection_manager.load_connections()

    async def get_connector(self, connection_id: str) -> BaseConnector | None:
        """Get a connector by connection ID"""
        return await self.connection_manager.get_connector(connection_id)

    async def _get_effective_sync_jwt(
        self,
        user_id: str,
        jwt_token: str | None = None,
    ) -> str | None:
        """Return a current OpenSearch JWT for connector sync work."""
        if not self.session_manager:
            return jwt_token

        user = self.session_manager.get_user(user_id)
        if user is None and user_id:
            from session_manager import User

            user = User(
                user_id=user_id,
                email=user_id,
                name=user_id,
                provider="connector",
            )
            self.session_manager.users[user_id] = user
        if user is None:
            return self.session_manager.get_effective_jwt_token(user_id, jwt_token)

        effective_token = jwt_token or user.jwt_token
        if (
            effective_token is None
            and getattr(self.session_manager, "private_key", None) is not None
        ):
            return self.session_manager.create_jwt_token(user)
        return self.session_manager.get_effective_jwt_token(user.user_id, effective_token)

    async def _update_connector_metadata(
        self,
        document: ConnectorDocument,
        owner_user_id: str,
        connector_type: str,
        jwt_token: str = None,
        indexed_filename: str | None = None,
    ):
        """Update indexed chunks with connector-specific metadata"""
        from utils.acl_utils import update_document_acl

        logger.debug("Looking for chunks", document_id=document.id)

        # Get user's OpenSearch client
        opensearch_client = self.session_manager.get_user_opensearch_client(
            owner_user_id, jwt_token
        )
        write_client = self.clients.opensearch
        if write_client is None:
            raise RuntimeError("Backend OpenSearch write client is unavailable")

        # Update ACL if changed (hash-based skip optimization).
        # Match both document_id and connector_file_id: both the Langflow and
        # non-Langflow ingestion paths store the raw connector id in
        # connector_file_id (document_id holds a content/id hash), except for
        # pre-migration chunks indexed before that split existed, which only
        # have document_id set to the raw connector id.
        acl_result = await update_document_acl(
            document_id=document.id,
            acl=document.acl,
            opensearch_client=opensearch_client,
            write_opensearch_client=write_client,
            id_fields=("document_id", "connector_file_id"),
        )

        # Log ACL update result
        if acl_result["status"] == "unchanged":
            logger.debug(f"ACL unchanged for {document.id}, skipped update")
        elif acl_result["status"] == "updated":
            logger.info(
                f"Updated ACL for {document.id}, {acl_result['chunks_updated']} chunks updated"
            )
        elif acl_result["status"] == "error":
            logger.error(f"ACL update error for {document.id}: {acl_result.get('error')}")

        # Update other metadata fields (source_url, timestamps, etc.)
        # Use the backend client for writes; the scoped client above is only
        # used for DLS visibility/ACL-change checks.
        try:
            await write_client.update_by_query(
                index=get_index_name(),
                body={
                    # Match both fields: both ingestion paths carry the raw
                    # connector id in connector_file_id (document_id is a
                    # content/id hash); pre-migration chunks only have it in
                    # document_id.
                    "query": {
                        "bool": {
                            "should": [
                                {"term": {"document_id": document.id}},
                                {"term": {"connector_file_id": document.id}},
                                # See check_document_exists (models/processors.py):
                                # some indices predate the explicit keyword
                                # mapping for this field.
                                {"term": {"connector_file_id.keyword": document.id}},
                            ],
                            "minimum_should_match": 1,
                        }
                    },
                    "script": {
                        "source": """
                            ctx._source.source_url = params.source_url;
                            ctx._source.connector_type = params.connector_type;
                            if (params.filename != null) {
                                ctx._source.filename = params.filename;
                            }
                            if (params.created_time != null) {
                                ctx._source.created_time = params.created_time;
                            }
                            if (params.modified_time != null) {
                                ctx._source.modified_time = params.modified_time;
                            }
                            if (params.content_etag != null) {
                                ctx._source.content_etag = params.content_etag;
                            }
                            if (params.metadata != null) {
                                ctx._source.metadata = params.metadata;
                            }
                        """,
                        "params": {
                            "source_url": document.source_url,
                            "connector_type": connector_type,
                            "filename": indexed_filename or document.filename,
                            "created_time": document.created_time.isoformat()
                            if document.created_time
                            else None,
                            "modified_time": document.modified_time.isoformat()
                            if document.modified_time
                            else None,
                            # Promoted out of metadata to a top-level keyword so
                            # sync can read it back with a terms aggregation (the
                            # same shape connector_file_id uses) instead of
                            # fetching every chunk's _source.
                            "content_etag": normalize_etag(
                                (document.metadata or {}).get(CONTENT_ETAG_METADATA_KEY)
                            ),
                            "metadata": document.metadata,
                        },
                    },
                },
                refresh=True,
            )
            invalidate_corpus_caches()
            logger.debug(f"Updated metadata for document {document.id}")
        except Exception as e:
            # A missing index means the chunks aren't where this write expects
            # them (e.g. a residual index-name mismatch, issue 81583). The
            # document is already indexed; metadata enrichment is best-effort
            # and re-runs on the next sync, so don't fail the file over it —
            # matching get_synced_file_ids_for_connector / should_update_acl.
            if (
                isinstance(e, NotFoundError)
                and e.status_code == 404
                and e.error == "index_not_found_exception"
            ):
                logger.warning(
                    "Skipping connector metadata enrichment — index not found",
                    document_id=document.id,
                    index=get_index_name(),
                    error=str(e),
                )
                return
            logger.error(
                "OpenSearch metadata update failed",
                document_id=document.id,
                error=str(e),
            )
            raise

    async def sync_connector_files(
        self,
        connection_id: str,
        user_id: str,
        max_files: int = None,
        jwt_token: str = None,
        filename_filter: set = None,
        ingest_settings: dict[str, Any] | None = None,
        replace_duplicates: bool = False,
        shared: bool | None = False,
        *,
        allow_anonymous_delete: bool,
    ) -> str:
        """
        Sync files from a connector connection using existing task tracking system.

        Args:
            connection_id: The connection ID
            user_id: The user ID
            max_files: Maximum number of files to sync
            jwt_token: Optional JWT token
            filename_filter: Optional set of filenames to filter - only files with names
                           in this set will be synced. Used to prevent deleted files
                           from being re-synced.
            ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …)
                forwarded to ``ConnectorFileProcessor``.
            allow_anonymous_delete: The syncing user's resolved
                ``knowledge:delete:anonymous``. No default; see
                ``TaskProcessor.delete_document_by_filename``.
        """
        jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)

        if not self.task_service:
            raise ValueError(
                "TaskService not available - connector sync requires task service dependency"
            )

        logger.debug(
            "Starting sync for connection",
            connection_id=connection_id,
            max_files=max_files,
        )

        connector = await self.get_connector(connection_id)
        if not connector:
            raise ValueError(f"Connection '{connection_id}' not found or not authenticated")

        logger.debug("Got connector", authenticated=connector.is_authenticated)

        if not connector.is_authenticated:
            raise ValueError(f"Connection '{connection_id}' not authenticated")

        if shared and connector.CONNECTOR_TYPE != "ibm_cos":
            raise ValueError("shared flag is only supported for the ibm_cos connector")

        # Collect files to process (limited by max_files)
        files_to_process: list[dict[str, Any]] = []
        page_token = None

        # A zero cap means "sync nothing" and has to short-circuit before the
        # first list_files call: every cap below is spelled `if max_files and …`,
        # so 0 would fall through as "no cap" and enumerate the whole source.
        # None (no cap) and positive caps take the loop as before.
        while max_files != 0:
            # Pass max_files straight through — None means "no cap". Asking for a
            # synthetic page size instead silently truncated every sync: the
            # connectors that paginate internally (all three bucket ones, and
            # Google Drive) honour the cap and then report next_page_token=None,
            # so there was no token to continue with and everything past the
            # first page was simply dropped.
            logger.debug("Calling list_files", max_files=max_files, page_token=page_token)
            file_list = await connector.list_files(page_token, max_files=max_files)
            logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))
            files = file_list["files"]
            page_token = _next_page_token(file_list)

            if not files and not page_token:
                break

            for file_info in files:
                if max_files and len(files_to_process) >= max_files:
                    break
                # Filter by filename if filter is provided
                if filename_filter is not None:
                    file_name = file_info.get("name", "")
                    if file_name not in filename_filter:
                        logger.debug(
                            "Skipping file not in filter",
                            filename=file_name,
                        )
                        continue
                files_to_process.append(file_info)

            # Stop if we have enough files or no more pages
            if (max_files and len(files_to_process) >= max_files) or not page_token:
                break

        # Get user information
        user = self.session_manager.get_user(user_id) if self.session_manager else None
        owner_name = user.name if user else None
        owner_email = user.email if user else None

        # Create custom processor for connector files
        from models.processors import ConnectorFileProcessor
        from services.document_service import DocumentService

        processor = ConnectorFileProcessor(
            self,
            connection_id,
            files_to_process,
            user_id,
            jwt_token=jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
            document_service=(
                self.task_service.document_service
                if self.task_service and self.task_service.document_service
                else DocumentService(session_manager=self.session_manager)
            ),
            models_service=self.models_service,
            ingest_settings=ingest_settings,
            replace_duplicates=replace_duplicates,
            connector_type=connector.CONNECTOR_TYPE,
            shared=shared,
            allow_anonymous_delete=allow_anonymous_delete,
        )

        # Use file IDs as items (no more fake file paths!)
        file_ids = [file_info["id"] for file_info in files_to_process]
        original_filenames = {
            file_info["id"]: clean_connector_filename(
                file_info["name"], file_info.get("mimeType") or file_info.get("mimetype")
            )
            for file_info in files_to_process
            if "name" in file_info
        }

        # Create custom task using TaskService
        task_id = await self.task_service.create_custom_task(
            user_id,
            file_ids,
            processor,
            original_filenames=original_filenames,
        )

        return task_id

    async def sync_specific_files(
        self,
        connection_id: str,
        user_id: str,
        file_ids: list[str],
        jwt_token: str = None,
        file_infos: list[dict[str, Any]] = None,
        ingest_settings: dict[str, Any] | None = None,
        replace_duplicates: bool = False,
        preview_mode: bool = False,
        shared: bool | None = False,
        *,
        allow_anonymous_delete: bool,
    ) -> str:
        """
        Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).
        Automatically expands folders to their contents.

        Args:
            connection_id: The connection ID
            user_id: The user ID
            file_ids: List of file IDs to sync
            jwt_token: Optional JWT token for authentication
            file_infos: Optional list of file info dicts with {id, name, mimeType, downloadUrl, size}
                       When provided, download URLs can be used directly without Graph API calls.
            ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …) passed to
                ``ConnectorFileProcessor`` when Langflow ingest is disabled.
            allow_anonymous_delete: The syncing user's resolved
                ``knowledge:delete:anonymous``. No default; see
                ``TaskProcessor.delete_document_by_filename``.
        """
        jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)

        if not self.task_service:
            raise ValueError(
                "TaskService not available - connector sync requires task service dependency"
            )

        connector = await self.get_connector(connection_id)
        if not connector:
            raise ValueError(f"Connection '{connection_id}' not found or not authenticated")

        if not connector.is_authenticated:
            raise ValueError(f"Connection '{connection_id}' not authenticated")

        if shared and connector.CONNECTOR_TYPE != "ibm_cos":
            raise ValueError("shared flag is only supported for the ibm_cos connector")

        if not file_ids:
            raise ValueError("No file IDs provided")

        # Get user information
        user = self.session_manager.get_user(user_id) if self.session_manager else None
        owner_name = user.name if user else None
        owner_email = user.email if user else None

        # If file_infos provided, cache them in the connector for later use
        # This allows get_file_content to use download URLs directly
        if file_infos and hasattr(connector, "set_file_infos"):
            connector.set_file_infos(file_infos)
            logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")

        expanded_file_ids = file_ids  # Default to original IDs
        expanded_files_info = []

        try:
            # cfg is None on bucket connectors (azure_blob/aws_s3/ibm_cos): they
            # have no per-call file/folder selection to expand, and file_ids are
            # already the exact ids to sync. Only cfg-backed connectors
            # (Google Drive/OneDrive/SharePoint) expand folders here. Guarding on
            # cfg-is-not-None rather than hasattr is deliberate: BaseConnector
            # declares cfg=None as a class default, so hasattr is True for every
            # connector and would route bucket syncs through list_selected_files
            # -> list_files() (the whole account), discarding the selected ids.
            if getattr(connector, "cfg", None) is not None:
                result = await connector.list_selected_files(file_ids)
                expanded_files = result.get("files", [])
                expanded_file_ids = [f["id"] for f in expanded_files]

                for f in expanded_files:
                    expanded_files_info.append(f)

                # Requested IDs that vanished during expansion are either
                # folders (replaced by their children) or gone at the source
                # (deleted/trashed). Re-add the non-folder ones so the
                # processor can run its deleted-at-source cleanup
                # (get_file_content -> 404 -> delete indexed chunks).
                expanded_set = set(expanded_file_ids)
                known_folder_ids = {
                    f["id"] for f in (file_infos or []) if f.get("isFolder") and f.get("id")
                }
                missing_ids = [
                    fid
                    for fid in file_ids
                    if fid not in expanded_set and fid not in known_folder_ids
                ]
                if missing_ids:
                    logger.info(
                        f"Re-adding {len(missing_ids)} file id(s) missing after expansion "
                        f"(possibly deleted at source)"
                    )
                    expanded_file_ids = expanded_file_ids + missing_ids

            if not expanded_file_ids:
                logger.warning(
                    f"No files found after expanding file_ids. "
                    f"Original IDs: {file_ids}. This may indicate all IDs were folders "
                    f"with no contents, or files that were filtered out."
                )
                if file_infos:
                    non_folder_infos = [f for f in file_infos if not f.get("isFolder")]
                    non_folder_ids = [f["id"] for f in non_folder_infos if f.get("id")]
                    if non_folder_ids:
                        logger.info(
                            "Using original file IDs with cached download URLs (folders excluded)"
                        )
                        expanded_file_ids = non_folder_ids
                    else:
                        raise ValueError("No files to sync after expanding folders")
                else:
                    raise ValueError("No files to sync after expanding folders")

        except Exception as e:
            logger.error(f"Failed to expand file_ids via list_files(): {e}")
            if isinstance(e, ValueError):
                raise
            if file_infos:
                non_folder_ids = [
                    f["id"] for f in file_infos if f.get("id") and not f.get("isFolder")
                ]
                expanded_file_ids = non_folder_ids or file_ids
            else:
                expanded_file_ids = file_ids

        # Create custom processor for specific connector files
        from models.processors import ConnectorFileProcessor
        from services.document_service import DocumentService

        # Use expanded_file_ids which has folders already expanded
        processor = ConnectorFileProcessor(
            self,
            connection_id,
            expanded_file_ids,
            user_id,
            jwt_token=jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
            document_service=(
                self.task_service.document_service
                if self.task_service and self.task_service.document_service
                else DocumentService(session_manager=self.session_manager)
            ),
            models_service=self.models_service,
            ingest_settings=ingest_settings,
            replace_duplicates=replace_duplicates,
            connector_type=connector.CONNECTOR_TYPE,
            shared=shared,
            allow_anonymous_delete=allow_anonymous_delete,
        )

        # Create custom task using TaskService
        original_filenames = {}

        # Combine file_infos and expanded_files_info
        all_infos = (file_infos or []) + expanded_files_info
        if all_infos:
            original_filenames = {
                f["id"]: clean_connector_filename(
                    f["name"], f.get("mimeType") or f.get("mime_type") or f.get("mimetype", "")
                )
                for f in all_infos
                if "id" in f and "name" in f
            }

        task_id = await self.task_service.create_custom_task(
            user_id,
            expanded_file_ids,
            processor,
            original_filenames=original_filenames,
            preview_mode=preview_mode,
        )

        return task_id

    async def _get_connector(self, connection_id: str) -> BaseConnector | None:
        """Get a connector by connection ID (alias for get_connector)"""
        return await self.get_connector(connection_id)