Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
89.6 KB
Raw Blame
import asyncio
import mimetypes
import os
import time
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any, Literal

from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
from observability.spans import trace_root, trace_span
from services.search_cache import invalidate_corpus_caches
from session_manager import AnonymousUser
from utils.document_processing import (
    extract_relevant,
    process_text_file,
    resplit_chunks_character_windows,
    split_chunks_by_max_tokens,
)
from utils.file_utils import (
    auto_cleanup_tempfile,
    clean_connector_filename,
    get_file_extension,
    get_filename_aliases,
    langflow_safe_filename_and_mimetype,
)
from utils.filename_claims import claim_holder, filename_claims
from utils.hash_utils import hash_id
from utils.logging_config import get_logger
from utils.opensearch_queries import build_owned_filename_query, build_replace_filename_query

from .tasks import FileTask, TaskStatus, UploadTask

logger = get_logger(__name__)

DOCLING_PARSER_LABEL = "Docling Serve 1.20.0"
TEXT_PARSER_LABEL = "Text Parser"

DUPLICATE_FILENAME_WARNING = "A file with this name already exists."


def _docling_counts(doc: Any) -> dict[str, int]:
    """What Docling produced, for the parse span: element counts, never text."""
    if not isinstance(doc, dict):
        return {}
    return {key: len(doc.get(key) or ()) for key in ("pages", "texts", "tables", "pictures")}


def _chunk_type_counts(chunks: list[dict]) -> dict[str, int]:
    """Chunk count, and per type (text/table/picture) where non-zero."""
    counts: dict[str, int] = {"chunks": len(chunks)}
    for chunk in chunks:
        key = f"chunks_{chunk.get('type') or 'unknown'}"
        counts[key] = counts.get(key, 0) + 1
    return counts


def _prompt_tokens(response: Any) -> int:
    """Input tokens an embedding response reports, or 0."""
    usage = response.get("usage") if isinstance(response, dict) else None
    if not isinstance(usage, dict):
        return 0
    tokens = usage.get("prompt_tokens") or usage.get("total_tokens") or 0
    return tokens if isinstance(tokens, int) else 0


# Both outcomes finish the file the same way — SKIPPED, counted successful,
# duplicate warning — but only "skip" means a document with this name is
# actually indexed. "skip_in_flight" means another file in this batch is still
# writing it, which is a different fact about the index.
DUPLICATE_SKIP_ACTIONS = ("skip", "skip_in_flight")
DUPLICATE_CONTENT_WARNING = (
    "Identical content already exists in the knowledge base under a different filename."
)

if TYPE_CHECKING:
    from connectors.base import DocumentACL


def _verification_client(fallback_client):
    """Client for post-ingestion verification ("did the chunks land in the
    index?"). That is a system integrity check, not a user-visibility check,
    so prefer the platform writer client: it does not depend on the JWT/JWKS
    trust chain that user-scoped clients need (OpenSearch loads the backend's
    JWKS lazily, so the first user-JWT queries after a cold start can 401).
    Falls back to the caller's client when the writer is unavailable."""
    return clients.opensearch if clients.opensearch is not None else fallback_client


def resolve_shared_owner_fields(
    user_id: str | None,
    owner_name: str | None,
    owner_email: str | None,
    shared: bool,
) -> tuple[str | None, str | None, str | None]:
    """Return (owner, owner_name, owner_email) for indexing.

    When shared=True, owner is None so the indexed chunk omits the owner field
    entirely, triggering the OpenSearch DLS must_not-exists-owner clause that
    makes the document visible to all users in the instance. owner_name and
    owner_email are set to AnonymousUser values, matching how default/sample
    documents are loaded.
    """
    if shared:
        _anon = AnonymousUser()
        return None, _anon.name, _anon.email
    return user_id, owner_name, owner_email


class TaskProcessor:
    """Base class for task processors with shared processing logic"""

    def __init__(self, document_service=None, models_service=None, docling_service=None):
        self.document_service = document_service
        self.models_service = models_service
        self.docling_service = docling_service

    async def check_document_exists(
        self,
        file_hash: str,
        opensearch_client,
        on_error: Literal["assume_missing", "assume_exists"] = "assume_missing",
        *,
        wait_for_visibility: bool = False,
        field: str = "document_id",
    ) -> bool:
        """
        Check if a document with the given hash already exists in OpenSearch.
        Consolidated hash checking for all processors.

        ``on_error`` picks the answer when OpenSearch stays unreachable after
        retries — the check is ambiguous then, and the safe default differs by
        caller:
          * ``"assume_missing"`` (dedupe callers): safer to reprocess than skip.
          * ``"assume_exists"`` (post-ingestion verification callers): an
            infrastructure error must not fail a file that Langflow already
            reported as ingested.

        When ``wait_for_visibility`` is True, an empty result is retried a few
        times with backoff before concluding the document is absent. This is for
        post-ingest verification: chunks that were just written may not be
        searchable yet within OpenSearch's near-real-time refresh window
        (default ~1s), and the user-scoped client cannot force an
        ``indices:admin/refresh`` (it lacks the privilege).
        """
        max_retries = 3
        retry_delay = 1.0

        # Some deployments' indices predate connector_file_id's addition to the
        # explicit mapping (config/settings.py), so OpenSearch dynamically
        # mapped it as analyzed `text` (with a `.keyword` multi-field) instead
        # of the intended `keyword` type. A plain term query against such a
        # field tokenizes the query value and rarely matches the raw id, so
        # also match its `.keyword` multi-field. document_id has always been
        # explicitly `keyword` since index creation and never has this issue.
        query: dict[str, Any]
        if field == "connector_file_id":
            query = {
                "bool": {
                    "should": [
                        {"term": {field: file_hash}},
                        {"term": {f"{field}.keyword": file_hash}},
                    ]
                }
            }
        else:
            query = {"term": {field: file_hash}}

        for attempt in range(max_retries):
            try:
                response = await opensearch_client.search(
                    index=get_index_name(),
                    body={
                        "size": 1,
                        "_source": False,
                        "query": query,
                    },
                )
                hits = response.get("hits", {}).get("hits", [])
                if hits:
                    return True
                # No hits. For post-ingest verification, the document may not be
                # visible yet within the near-real-time refresh window — retry.
                if wait_for_visibility and attempt < max_retries - 1:
                    await asyncio.sleep(retry_delay)
                    retry_delay *= 2
                    continue
                return False
            except (TimeoutError, Exception) as e:
                if attempt == max_retries - 1:
                    logger.error(
                        "OpenSearch exists check failed after retries",
                        file_hash=file_hash,
                        error=str(e),
                        attempt=attempt + 1,
                    )
                    if on_error == "assume_exists":
                        logger.warning(
                            "Exists check inconclusive due to connection issues; "
                            "assuming document exists",
                            file_hash=file_hash,
                        )
                        return True
                    # Safer to reprocess than skip for dedupe callers.
                    logger.warning(
                        "Assuming document doesn't exist due to connection issues",
                        file_hash=file_hash,
                    )
                    return False
                else:
                    logger.warning(
                        "OpenSearch exists check failed, retrying",
                        file_hash=file_hash,
                        error=str(e),
                        attempt=attempt + 1,
                        retry_in=retry_delay,
                    )
                    await asyncio.sleep(retry_delay)
                    retry_delay *= 2  # Exponential backoff
        return on_error == "assume_exists"

    async def check_filename_exists(
        self,
        filename: str,
        opensearch_client,
        *,
        wait_for_visibility: bool = False,
    ) -> bool:
        """
        Check if a document with the given filename already exists in OpenSearch.
        Returns True if any chunks with this filename exist.

        When ``wait_for_visibility`` is True, an empty result is retried a few
        times with backoff before concluding the document is absent. This is for
        post-ingest verification: chunks that were just written may not be
        searchable yet within OpenSearch's near-real-time refresh window
        (default ~1s), and the user-scoped client cannot force an
        ``indices:admin/refresh`` (it lacks the privilege).
        """
        from utils.opensearch_filenames import find_existing_filenames

        max_retries = 3
        retry_delay = 1.0

        candidate_filenames = get_filename_aliases(filename)
        if not candidate_filenames:
            return False

        for attempt in range(max_retries):
            try:
                # One bulk existence check covering every alias — the shared
                # query semantic used by all duplicate-detection altitudes.
                if await find_existing_filenames(
                    candidate_filenames, opensearch_client, get_index_name()
                ):
                    return True
                # No alias exists. For post-ingest verification, the document
                # may not be visible yet within the near-real-time refresh
                # window — re-check after a short delay.
                if wait_for_visibility and attempt < max_retries - 1:
                    await asyncio.sleep(retry_delay)
                    retry_delay *= 2
                    continue
                return False

            except (TimeoutError, Exception) as e:
                if attempt == max_retries - 1:
                    logger.error(
                        "OpenSearch filename check failed after retries",
                        filename=filename,
                        error=str(e),
                        attempt=attempt + 1,
                    )
                    # On final failure, assume document doesn't exist (safer to reprocess than skip)
                    logger.warning(
                        "Assuming filename doesn't exist due to connection issues",
                        filename=filename,
                    )
                    return False
                else:
                    logger.warning(
                        "OpenSearch filename check failed, retrying",
                        filename=filename,
                        error=str(e),
                        attempt=attempt + 1,
                        retry_in=retry_delay,
                    )
                    await asyncio.sleep(retry_delay)
                    retry_delay *= 2  # Exponential backoff
        return False

    async def resolve_duplicate_filename(
        self,
        filename: str,
        opensearch_client,
        *,
        replace: bool,
        owner_user_id: str | None,
        allow_anonymous_delete: bool,
        shared: bool = False,
        claim_holder: str | None = None,
    ) -> Literal["proceed", "skip", "skip_in_flight", "replaced"]:
        """Single duplicate-filename policy shared by every processor.

        Checks whether a document with this filename (or one of its aliases)
        is already indexed and applies the caller's replace decision:

          * ``"proceed"``  — no duplicate; continue ingestion.
          * ``"skip"``     — a document with this name is indexed and
                             ``replace`` is False; the caller should finish via
                             ``mark_duplicate_skipped``.
          * ``"skip_in_flight"`` — another file in this batch holds the name and
                             has not finished writing it. Finishes the same way,
                             but nothing is indexed under the name *yet*, so a
                             caller must not treat it as a statement about what
                             is in the index (see ``_reconcile_shared_owner``).
          * ``"replaced"`` — duplicate and ``replace`` is True; the existing
                             chunks were deleted and the index refreshed, so
                             ingestion can continue.

        ``claim_holder`` identifies this file's run (see ``utils.filename_claims``)
        and holds the name for its duration, so a second file heading for the
        same name inside the same batch resolves as a duplicate instead of
        racing the index check. TaskService releases the claim when the file
        reaches a terminal state. Callers that pass None get the OpenSearch
        policy alone — every processor passes one.

        ``allow_anonymous_delete`` is handed to ``delete_document_by_filename``
        and, like there, has no default: see that docstring.
        """
        if claim_holder is not None and not filename_claims.claim(
            claim_holder, filename, owner_user_id=owner_user_id, shared=shared
        ):
            # Another file in flight is already heading for this name; whichever
            # of them lands first is the one this name belongs to.
            return "skip_in_flight"

        if not await self.check_filename_exists(filename, opensearch_client):
            return "proceed"
        if not replace:
            return "skip"

        logger.info(f"Replacing existing document: {filename}")
        deleted = await self.delete_document_by_filename(
            filename,
            opensearch_client,
            owner_user_id=owner_user_id,
            shared=shared,
            allow_anonymous_delete=allow_anonymous_delete,
        )
        if deleted == 0:
            logger.warning(
                "Replacement requested but deletion removed no chunks",
                filename=filename,
            )
            return "skip"
        # Refresh so the delete is visible before re-ingest. refresh is
        # index-wide (indices:admin/refresh) and cannot be DLS-scoped, so it
        # must run under the admin/service client, not the user client.
        try:
            await clients.opensearch.indices.refresh(index=get_index_name())
        except Exception as refresh_error:
            logger.warning(
                "Failed to refresh index after delete",
                error=str(refresh_error),
            )
        return "replaced"

    @staticmethod
    def _claim_holder(upload_task: UploadTask, file_task: FileTask) -> str:
        """Identity of this file's run, for the in-flight filename claim."""
        return claim_holder(upload_task.task_id, file_task.file_path)

    def mark_duplicate_skipped(self, upload_task: UploadTask, file_task: FileTask) -> None:
        """Uniform terminal state for a duplicate that was not replaced:
        SKIPPED, counted toward successful files, with a warning the task view
        surfaces. A declined replacement is a chosen outcome, not an error."""
        file_task.status = TaskStatus.SKIPPED
        file_task.error = None
        file_task.result = {
            "status": "skipped",
            "reason": "duplicate_filename",
            "warning": DUPLICATE_FILENAME_WARNING,
        }
        file_task.updated_at = time.time()
        upload_task.successful_files += 1

    async def delete_document_by_filename(
        self,
        filename: str,
        opensearch_client,
        owner_user_id: str | None = None,
        shared: bool = False,
        *,
        allow_anonymous_delete: bool,
    ) -> int:
        """Delete all chunks of a document with the given filename from
        OpenSearch.  Returns the number of chunks deleted.

        ``shared`` describes how the replacement is about to be *written*, not
        what is already indexed, so it must not narrow what we delete: with an
        owner in hand, and the permission below, the scope is "owned by this
        user OR ownerless".
        Choosing an owner-only scope for a shared document matched none of its
        chunks, and the caller read that zero as "nothing to replace" and
        skipped the file — leaving the stale copy in the index even though the
        duplicate check (which is owner-agnostic) had just found it and the user
        had confirmed the overwrite.

        ``allow_anonymous_delete`` is the caller's resolved
        ``knowledge:delete:anonymous``. Ownerless chunks are visible to everyone
        in the instance, so replacing a document that turns out to be one is a
        deletion of shared content: without that permission the scope stays
        owner-only and someone else's shared document is left alone (the file
        then resolves as a duplicate the user may not replace). It has no
        default on purpose. It used to default to True, and the entry points
        that never resolved the permission — local upload, S3, the Langflow
        path — inherited that silently, so any user could replace a shared
        document through them. Every caller now has to say which it is, down
        to the processor constructors and the task-service helpers that feed
        them; one acting for the system rather than for a user passes True and
        says why.
        ``shared`` does not stand in for it. An explicit shared sync already
        holds the permission (connector_sync refuses one without it), but on
        the re-sync paths ``shared`` is inferred from what is indexed, and an
        inferred answer authorizes nothing: honouring it let a caller without
        the permission replace a shared document simply by re-syncing it."""
        from config.settings import clients, get_index_name
        from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
        from utils.opensearch_queries import (
            build_anonymous_filename_query,
            build_owned_filename_query,
            build_replace_filename_query,
        )

        try:
            write_client = clients.opensearch
            if write_client is None:
                raise RuntimeError("Backend OpenSearch write client is unavailable")

            if not owner_user_id:
                if shared:

                    def build_query(fname, _owner):
                        return build_anonymous_filename_query(fname)
                else:
                    logger.warning(
                        "Skipped delete_by_filename because owner_user_id is missing",
                        filename=filename,
                    )
                    return 0

            elif allow_anonymous_delete:
                build_query = build_replace_filename_query
            else:
                build_query = build_owned_filename_query

            candidate_filenames = get_filename_aliases(filename)
            if not candidate_filenames:
                logger.info(
                    "Skipped delete_by_filename because filename input is empty",
                    filename=filename,
                )
                return 0

            deleted_count = 0
            for candidate in candidate_filenames:
                document_ids = await collect_visible_document_ids(
                    opensearch_client,
                    index=get_index_name(),
                    query=build_query(candidate, owner_user_id),
                )
                deleted_count += await delete_document_ids(
                    write_client,
                    index=get_index_name(),
                    document_ids=document_ids,
                )
            logger.info(
                "Deleted existing document chunks", filename=filename, deleted_count=deleted_count
            )
            return deleted_count

        except Exception as e:
            logger.error("Failed to delete existing document", filename=filename, error=str(e))
            raise

    async def _delete_connector_chunks(
        self,
        file_id: str,
        opensearch_client,
        owner_user_id: str,
        keep_filenames: list[str] | None = None,
        shared: bool = False,
        connector_type: str | None = None,
        *,
        ignore_errors: bool = True,
    ) -> int:
        """Delete indexed chunks for a connector file by its STABLE id.

        Deletion semantics (dual-field id match, connector/owner/shared scoping,
        rename ``keep_filenames``) live in ``connectors.chunk_cleanup``.

        Rename cleanup is best-effort (``ignore_errors=True``): logs and returns
        0 so a miss never fails an ingest that can still re-index the new name.
        Source-deleted cleanup passes ``ignore_errors=False`` so a real index
        failure fails the file task instead of reporting success.

        ``connector_type`` scopes the match to one connector type — the same
        value the chunks were indexed under — so an id that collides with a
        different connector's id can't take its chunks down with it.

        Callers pass the sharing mode resolved from what is actually indexed
        (``_resolve_shared``), so this deletes under exactly that layout —
        ownerless chunks for a share-all file, owner-scoped ones otherwise.
        It must NOT widen to "owned OR ownerless" on top of that: the resolved
        mode already matches the file, and widening would let a sync of an owned
        file reach ownerless chunks that another user's connection ingested,
        with none of the ``knowledge:delete:anonymous`` gating that deliberate
        shared deletion goes through.
        """
        from connectors.chunk_cleanup import delete_connector_file_chunks

        if not file_id:
            return 0
        try:
            return await delete_connector_file_chunks(
                [file_id],
                opensearch_client,
                connector_type=connector_type,
                owner_user_id=owner_user_id,
                shared=shared,
                keep_filenames=keep_filenames,
            )
        except Exception as e:
            logger.error(
                "Failed to delete connector chunks",
                file_id=file_id,
                error=str(e),
            )
            if ignore_errors:
                return 0
            raise

    async def process_document_standard(
        self,
        file_path: str,
        file_hash: str,
        owner_user_id: str = None,
        original_filename: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        file_size: int = None,
        connector_type: str = "local",
        embedding_model: str = None,
        chunk_size: int = None,
        chunk_overlap: int = None,
        is_sample_data: bool = False,
        acl: "DocumentACL | None" = None,
        connector_file_id: str | None = None,
        ocr: bool | None = None,
        picture_descriptions: bool | None = None,
        shared: bool = False,
        document_id: str | None = None,
        replace_existing: bool = False,
        source_url: str | None = None,
        web_source_id: str | None = None,
        web_page_id: str | None = None,
        web_page_depth: int | None = None,
        root_source_url: str | None = None,
        canonical_url: str | None = None,
        before_index_write: Callable[[], Awaitable[object]] | None = None,
    ):
        """
        Standard processing pipeline for non-Langflow processors:
        docling conversion + embeddings + OpenSearch indexing.

        Args:
            embedding_model: Embedding model to use (defaults to the current
                embedding model from settings)
            chunk_size: Optional character window size for re-splitting extracted
                chunks (non-Langflow path, e.g. connector UI ``chunkSize``).
            chunk_overlap: Overlap between windows; must be less than ``chunk_size``.
            acl: DocumentACL instance with access control information
            ocr: Per-request OCR override (None = use global config).
            picture_descriptions: Per-request picture descriptions override.
        """
        from services.document_service import chunk_texts_for_embeddings

        # Use provided embedding model or configured model.
        # get_embedding_model() returns empty string when Langflow ingest is enabled,
        # but OpenRAG processors still need a concrete embedding model.
        config = get_openrag_config()
        configured_embedding_model = config.knowledge.embedding_model
        embedding_provider = getattr(config.knowledge, "embedding_provider", None) or "openai"
        embedding_model = embedding_model or configured_embedding_model or get_embedding_model()

        if chunk_size is None:
            chunk_size = config.knowledge.chunk_size
        if chunk_overlap is None:
            chunk_overlap = config.knowledge.chunk_overlap

        stable_document_id = document_id or file_hash
        file_ext = os.path.splitext(file_path)[1].lower()

        # One trace per file, with a span per stage; no-ops while tracing is off.
        with trace_root(
            "ingest.file",
            user_id=owner_user_id,
            tags=["ingest_path:openrag", f"connector:{connector_type}"],
            attributes={
                "document_id": stable_document_id,
                "ingest_path": "openrag",
                "connector_type": connector_type,
                "embedding_model": embedding_model,
                "embedding_provider": embedding_provider,
                "file.extension": file_ext or None,
                "file.size": file_size,
                "sample_data": is_sample_data or None,
            },
        ) as ingest_trace:
            ingest_trace.set_content(filename=original_filename or os.path.basename(file_path))

            # Get user's OpenSearch client with JWT for OIDC auth
            opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
                owner_user_id, jwt_token
            )

            # URL sources own normalized-content deduplication. A page re-sync
            # deliberately replaces chunks under its stable document ID.
            if not replace_existing and await self.check_document_exists(
                stable_document_id, opensearch_client
            ):
                ingest_trace.set(status="unchanged")
                return {"status": "unchanged", "id": stable_document_id}

            logger.info(
                "Processing document with embedding model",
                embedding_model=embedding_model,
                file_hash=stable_document_id,
            )

            # Check if this is a .txt or .md file - use simple processing instead of docling
            if file_ext in (".txt", ".md"):
                # Simple text file processing without docling
                logger.info(
                    "Processing as plain text file (bypassing docling)",
                    file_path=file_path,
                    file_hash=file_hash,
                )
                with trace_span("ingest.parse", parser=TEXT_PARSER_LABEL) as parse:
                    slim_doc = process_text_file(file_path)
                    parse.set(chunks=len(slim_doc["chunks"]))
                slim_doc["parser"] = TEXT_PARSER_LABEL
            else:
                with trace_span(
                    "ingest.parse",
                    parser=DOCLING_PARSER_LABEL,
                    # Unset per-request options fall back to config, as in Docling.
                    ocr=ocr if ocr is not None else getattr(config.knowledge, "ocr", None),
                    picture_descriptions=picture_descriptions
                    if picture_descriptions is not None
                    else getattr(config.knowledge, "picture_descriptions", None),
                    table_structure=getattr(config.knowledge, "table_structure", None),
                ) as parse:
                    full_doc = await self.docling_service.convert_file(
                        file_path,
                        user_id=owner_user_id,
                        auth_header=jwt_token,
                        ocr=ocr,
                        picture_descriptions=picture_descriptions,
                    )
                    parse.set(**_docling_counts(full_doc))
                with trace_span("ingest.extract") as extract:
                    slim_doc = extract_relevant(full_doc)
                    extract.set(**_chunk_type_counts(slim_doc["chunks"]))
                slim_doc["parser"] = DOCLING_PARSER_LABEL
            ingest_trace.set(mimetype=slim_doc.get("mimetype"))

            # Override filename with original_filename if provided
            if original_filename:
                slim_doc["filename"] = original_filename

            if chunk_size is not None:
                try:
                    cs = int(chunk_size)
                except (TypeError, ValueError):
                    cs = 0
                if cs > 0:
                    try:
                        co = int(chunk_overlap) if chunk_overlap is not None else 0
                    except (TypeError, ValueError):
                        co = 0
                    if co < cs:
                        with trace_span(
                            "ingest.resplit",
                            chunk_size=cs,
                            chunk_overlap=max(0, co),
                            chunks_in=len(slim_doc["chunks"]),
                        ) as resplit:
                            slim_doc["chunks"] = resplit_chunks_character_windows(
                                slim_doc["chunks"], cs, max(0, co)
                            )
                            resplit.set(chunks_out=len(slim_doc["chunks"]))

            litellm_embedding_model = (
                await self.models_service.get_litellm_model_name(
                    embedding_model,
                    provider=embedding_provider,
                )
                if self.models_service is not None
                else embedding_model
            )

            litellm_model_lower = litellm_embedding_model.lower() if litellm_embedding_model else ""
            if "watsonx" in litellm_model_lower:
                max_tokens = 500
            elif "ollama" in litellm_model_lower:
                max_tokens = 2000
            else:
                max_tokens = 8000

            with trace_span(
                "ingest.token_cap", max_tokens=max_tokens, chunks_in=len(slim_doc["chunks"])
            ) as token_cap:
                # Filter out chunks with empty or whitespace-only text before generating embeddings.
                # This ensures the length of chunks matches the length of the embeddings array,
                # since chunk_texts_for_embeddings also drops empty texts.
                slim_doc["chunks"] = [
                    c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()
                ]
                non_empty = len(slim_doc["chunks"])

                # Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.
                slim_doc["chunks"] = split_chunks_by_max_tokens(
                    slim_doc["chunks"], max_tokens, litellm_embedding_model
                )
                # Re-filter out chunks with empty or whitespace-only text that may have resulted from splitting
                slim_doc["chunks"] = [
                    c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()
                ]
                token_cap.set(non_empty=non_empty, chunks_out=len(slim_doc["chunks"]))
            texts = [c["text"] for c in slim_doc["chunks"]]

            text_batches = chunk_texts_for_embeddings(
                texts, max_tokens=max_tokens, model=litellm_embedding_model
            )
            embeddings = []
            # Every provider goes through the gateway, not just the aliased ones.
            # The gateway resolves credentials per call from config; the direct
            # client resolved them from process-global environment written once per
            # process, which is never unset when a provider is deconfigured and is
            # shared by every provider in the process. Gating on "is this provider
            # aliased?" also meant the correctness of ingestion depended on an
            # unrelated routing detail.
            from services.llm_gateway import embeddings as gateway_embeddings
            from services.llm_gateway import qualified_model_id

            gateway_model = qualified_model_id(embedding_provider, embedding_model)

            with trace_span("ingest.embed", batches=len(text_batches), inputs=len(texts)) as embed:
                prompt_tokens = 0
                for batch in text_batches:
                    response = await gateway_embeddings({"model": gateway_model, "input": batch})
                    data = response.get("data", [])
                    embeddings.extend(
                        [
                            item["embedding"] if isinstance(item, dict) else item.embedding
                            for item in data
                        ]
                    )
                    prompt_tokens += _prompt_tokens(response)
                embed.set(
                    vectors=len(embeddings),
                    dimensions=len(embeddings[0]) if embeddings else None,
                    prompt_tokens=prompt_tokens or None,
                )

            if not embeddings or len(embeddings) == 0:
                logger.error(
                    "No embeddings generated — document may be empty or unreadable",
                    file_hash=file_hash,
                    embedding_model=embedding_model,
                )
                ingest_trace.fail("NoEmbeddings")
                return {
                    "status": "error",
                    "error": "No text content could be extracted from document",
                }

            from services.document_index_writer import (
                DocumentIndexChunk,
                DocumentIndexContext,
                DocumentIndexWriter,
            )

            document_index_writer = getattr(self.document_service, "document_index_writer", None)
            if document_index_writer is None:
                document_index_writer = DocumentIndexWriter()

            # Owner is always the authenticated uploading/syncing user unless shared=True,
            # in which case owner fields are omitted so DLS makes the doc visible to all users.
            owner, owner_name, owner_email = resolve_shared_owner_fields(
                owner_user_id, owner_name, owner_email, shared
            )
            if acl:
                allowed_users = acl.allowed_users or []
                allowed_groups = acl.allowed_groups or []
                allowed_principals = acl.allowed_principals or []
                allowed_principal_labels = acl.allowed_principal_labels or []
            else:
                allowed_users = []
                allowed_groups = []
                allowed_principals = []
                allowed_principal_labels = []

            filename = original_filename if original_filename else slim_doc["filename"]
            index_context = DocumentIndexContext(
                document_id=stable_document_id,
                filename=filename,
                mimetype=slim_doc["mimetype"],
                embedding_model=embedding_model,
                embedding_provider=embedding_provider,
                owner=owner,
                owner_name=owner_name,
                owner_email=owner_email,
                file_size=file_size,
                connector_type=connector_type,
                source_url=source_url,
                record_kind="web_page" if web_source_id else None,
                web_source_id=web_source_id,
                web_page_id=web_page_id,
                web_page_depth=web_page_depth,
                root_source_url=root_source_url,
                canonical_url=canonical_url,
                allowed_users=allowed_users,
                allowed_groups=allowed_groups,
                allowed_principals=allowed_principals,
                allowed_principal_labels=allowed_principal_labels,
                is_sample_data=is_sample_data,
            )
            parser_name = slim_doc.get("parser")
            if not parser_name:
                if file_ext in (".txt", ".md"):
                    parser_name = TEXT_PARSER_LABEL
                else:
                    parser_name = DOCLING_PARSER_LABEL

            chunk_metadata = {"parser": parser_name}
            if chunk_size is not None:
                chunk_metadata["chunk_size"] = chunk_size
            if chunk_overlap is not None:
                chunk_metadata["chunk_overlap"] = chunk_overlap
            if connector_file_id:
                chunk_metadata["connector_file_id"] = connector_file_id

            index_chunks = [
                DocumentIndexChunk(
                    chunk_id=f"{stable_document_id}_{i}",
                    text=chunk["text"],
                    vector=vect,
                    page=chunk["page"],
                    metadata=chunk_metadata,
                )
                for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True))
            ]

            with trace_span("ingest.index") as index:
                # This guard is intentionally outside the stale-cleanup handler.
                # A URL source can be deleted while its crawl is still processing;
                # in that case indexing must stop rather than recreating chunks for
                # the deleted source.
                if before_index_write is not None:
                    await before_index_write()

                # Clear stale chunks from a prior indexing of this document. Chunks are
                # stored under ids {file_hash}_{i}; if the new chunk count is lower
                # than the prior one, trailing chunks would otherwise survive the
                # writer's idempotent upsert.
                # DLS-safe: enumerate visible chunk ids with the scoped user client,
                # then delete concrete ids with the trusted backend client.
                try:
                    from utils.opensearch_delete import (
                        collect_visible_document_ids,
                        delete_document_ids,
                    )

                    write_client = clients.opensearch
                    if write_client is None:
                        raise RuntimeError("Backend OpenSearch write client is unavailable")

                    stale_chunk_ids = await collect_visible_document_ids(
                        opensearch_client,
                        index=get_index_name(),
                        query={"term": {"document_id": stable_document_id}},
                    )
                    await delete_document_ids(
                        write_client,
                        index=get_index_name(),
                        document_ids=stale_chunk_ids,
                        refresh=True,
                    )
                    index.set(stale_chunks_deleted=len(stale_chunk_ids))
                except Exception as e:
                    logger.warning(
                        "Failed to clear stale chunks before re-index; proceeding",
                        file_hash=stable_document_id,
                        error=str(e),
                    )
                    index.set(stale_cleanup_failed=True)

                await document_index_writer.index_chunks(index_context, index_chunks, final=True)
                index.set(chunks_indexed=len(index_chunks))

            ingest_trace.set(status="indexed", chunks=len(index_chunks))
            return {
                "status": "indexed",
                "id": stable_document_id,
                "chunk_count": len(index_chunks),
            }

    async def process_item(self, upload_task: UploadTask, item: Any, file_task: FileTask) -> None:
        """
        Process a single item in the task.

        This is a base implementation that should be overridden by subclasses.
        When TaskProcessor is used directly (not via subclass), this method
        is not called - only the utility methods like process_document_standard
        are used.

        Args:
            upload_task: The overall upload task
            item: The item to process (could be file path, file info, etc.)
            file_task: The specific file task to update
        """
        raise NotImplementedError(
            "process_item should be overridden by subclasses when used in task processing"
        )


class DocumentFileProcessor(TaskProcessor):
    """Default processor for regular file uploads"""

    def __init__(
        self,
        document_service,
        models_service,
        owner_user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        is_sample_data: bool = False,
        connector_type: str = "local",
        docling_service=None,
        replace_duplicates: bool = False,
        session_manager=None,
        settings: dict | None = None,
        *,
        allow_anonymous_delete: bool,
    ):
        super().__init__(
            document_service,
            models_service,
            docling_service=docling_service
            or (document_service.docling_service if document_service else None),
        )
        self.owner_user_id = owner_user_id
        self.jwt_token = jwt_token
        self.owner_name = owner_name
        self.owner_email = owner_email
        self.is_sample_data = is_sample_data
        self.connector_type = connector_type
        self.replace_duplicates = replace_duplicates
        self.session_manager = session_manager or (
            document_service.session_manager if document_service else None
        )
        self.settings = settings
        # The uploading user's resolved knowledge:delete:anonymous; see
        # delete_document_by_filename.
        self.allow_anonymous_delete = allow_anonymous_delete
        if self.session_manager is None:
            raise ValueError("session_manager is required for DocumentFileProcessor")

    async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
        """Process a regular file path using consolidated methods"""
        file_task.status = TaskStatus.RUNNING
        file_task.updated_at = time.time()

        try:
            # Use the ORIGINAL filename stored in file_task (not the transformed temp path)
            # This ensures we check/store the original filename with spaces, etc.
            original_filename = file_task.filename or os.path.basename(item)

            # Check if document with same filename already exists
            if self.session_manager is None:
                raise ValueError("session_manager is required to get OpenSearch client")
            opensearch_client = self.session_manager.get_user_opensearch_client(
                self.owner_user_id, self.jwt_token
            )

            duplicate_action = await self.resolve_duplicate_filename(
                original_filename,
                opensearch_client,
                replace=self.replace_duplicates,
                owner_user_id=self.owner_user_id,
                claim_holder=self._claim_holder(upload_task, file_task),
                allow_anonymous_delete=self.allow_anonymous_delete,
            )
            if duplicate_action in DUPLICATE_SKIP_ACTIONS:
                self.mark_duplicate_skipped(upload_task, file_task)
                return

            # Compute hash
            file_hash = hash_id(item)
            # Chunks are indexed with document_id=file_hash (see
            # process_document_standard -> DocumentIndexContext), so record it on
            # the file_task for preview-mode index proof lookups.
            file_task.document_id = file_hash

            # Get file size
            try:
                file_size = os.path.getsize(item)
            except Exception:
                file_size = 0

            # Parse ACL from settings if present
            from connectors.base import DocumentACL

            acl = None
            if self.settings and (
                self.settings.get("allowed_users") is not None
                or self.settings.get("allowed_groups") is not None
            ):
                acl = DocumentACL(
                    owner=self.owner_user_id,
                    allowed_users=self.settings.get("allowed_users", []),
                    allowed_groups=self.settings.get("allowed_groups", []),
                )

            standard_kwargs: dict[str, Any] = {}
            if self.settings:
                s = self.settings
                em = s.get("embeddingModel")
                if isinstance(em, str) and em.strip():
                    standard_kwargs["embedding_model"] = em.strip()
                for ui_key, param in (
                    ("chunkSize", "chunk_size"),
                    ("chunkOverlap", "chunk_overlap"),
                ):
                    raw = s.get(ui_key)
                    if raw is not None:
                        try:
                            standard_kwargs[param] = int(raw)
                        except (TypeError, ValueError):
                            pass

            config = get_openrag_config()
            standard_kwargs["ocr"] = config.knowledge.ocr
            standard_kwargs["picture_descriptions"] = config.knowledge.picture_descriptions

            # Use consolidated standard processing
            result = await self.process_document_standard(
                file_path=item,
                file_hash=file_hash,
                owner_user_id=self.owner_user_id,
                original_filename=original_filename,
                jwt_token=self.jwt_token,
                owner_name=self.owner_name,
                owner_email=self.owner_email,
                file_size=file_size,
                connector_type=self.connector_type,
                is_sample_data=self.is_sample_data,
                acl=acl,
                **standard_kwargs,
            )

            if result.get("status") == "error":
                file_task.status = TaskStatus.FAILED
                file_task.error = result.get("error") or "Failed to process document"
                file_task.updated_at = time.time()
                upload_task.failed_files += 1
            else:
                file_task.status = TaskStatus.COMPLETED
                file_task.result = result
                file_task.updated_at = time.time()
                upload_task.successful_files += 1

        except Exception as e:
            file_task.status = TaskStatus.FAILED
            file_task.error = str(e) or repr(e)
            file_task.updated_at = time.time()
            upload_task.failed_files += 1
            raise


class ConnectorFileProcessor(TaskProcessor):
    """Processor for connector file uploads"""

    def __init__(
        self,
        connector_service,
        connection_id: str,
        files_to_process: list,
        user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        document_service=None,
        models_service=None,
        ingest_settings: dict[str, Any] | None = None,
        replace_duplicates: bool = False,
        connector_type: str | None = None,
        preview_mode: bool = False,
        shared: bool | None = False,
        *,
        allow_anonymous_delete: bool,
    ):
        super().__init__(
            document_service=document_service,
            models_service=models_service,
            docling_service=document_service.docling_service if document_service else None,
        )
        self.connector_service = connector_service
        self.connection_id = connection_id
        self.files_to_process = files_to_process
        self.user_id = user_id
        self.jwt_token = jwt_token
        self.owner_name = owner_name
        self.owner_email = owner_email
        self.ingest_settings = ingest_settings
        self.replace_duplicates = replace_duplicates
        self.connector_type = connector_type
        self.preview_mode = preview_mode
        self.shared = shared
        # The syncing user's resolved knowledge:delete:anonymous, threaded down
        # to the replace path so an overwrite cannot delete a shared document
        # the user is not allowed to delete.
        self.allow_anonymous_delete = allow_anonymous_delete

    async def _indexed_shared_state(
        self,
        file_id: str,
        opensearch_client,
        connector_type: str | None,
    ) -> bool | None:
        """Whether this connector file is ALREADY indexed as shared (ownerless).

        Returns True when the indexed chunks carry no ``owner`` (the share-all
        layout), False when they carry one, and None when the file isn't indexed
        yet or the lookup failed — callers fall back to their own default.

        Read with the caller's (user-scoped) client, so DLS keeps documents we
        cannot see out of the answer.
        """
        if not file_id:
            return None
        from connectors.chunk_cleanup import build_connector_file_chunks_query

        try:
            response = await opensearch_client.search(
                index=get_index_name(),
                body={
                    "size": 1,
                    "_source": ["owner"],
                    "query": build_connector_file_chunks_query(
                        [file_id], connector_type=connector_type
                    ),
                },
            )
        except Exception as e:
            logger.warning(
                "Could not read indexed sharing state for connector file",
                file_id=file_id,
                error=str(e),
            )
            return None
        hits = response.get("hits", {}).get("hits", [])
        if not hits:
            return None
        return not hits[0].get("_source", {}).get("owner")

    async def _resolve_shared(
        self,
        file_id: str,
        opensearch_client,
        connector_type: str | None,
    ) -> bool:
        """Resolve the sharing mode to index this file under.

        ``self.shared`` is the caller's explicit intent — the connector upload
        UI's "Make documents available to all users" toggle. It is None on the
        re-sync paths, which run with no user in the loop and no record of what
        each file was ingested as; those inherit whatever is already in the
        index. Without that inheritance a sync would treat a shared file as
        private, and every owner-scoped step downstream (the replace-delete, the
        chunk cleanup, the owner fields on re-index) would either match nothing
        or silently un-share the document.

        Files not yet indexed fall back to private, matching the previous
        default for a caller that expressed no intent.
        """
        if self.shared is not None:
            return self.shared
        return bool(await self._indexed_shared_state(file_id, opensearch_client, connector_type))

    async def _reconcile_shared_owner(self, filename: str, shared: bool) -> None:
        """Update owner fields on already-indexed chunks for `filename` to match
        the `shared` setting resolved for this file.

        Called where the document indexed under this name is THIS file's own —
        the unchanged-content path, and a skip against a real duplicate. Never
        for a "skip_in_flight": there the name belongs to another file that is
        still writing it, and this query matches by filename, so reconciling
        would stamp this file's sharing answer onto that one's chunks.

        On those paths the file's content and name haven't changed since a prior
        sync, but the connector's "Make documents available to all users"
        setting may have been toggled since. Without this, those chunks would
        keep whatever owner they
        got on their original ingest forever, since a byte-identical re-sync
        never reaches resolve_shared_owner_fields(). Scoped to chunks owned by
        this user — and to ownerless ones only under the same
        ``knowledge:delete:anonymous`` boundary delete_document_by_filename
        uses — so it can't touch another user's document that happens to share
        this filename.

        That boundary matters here even more than it does for a delete. An
        ownerless document is visible to the whole instance, and this script
        writes an owner onto what it matches: without the permission check, a
        private sync of a file whose name collides with a shared document would
        quietly claim that document for the syncing user, taking it out of
        everyone else's view. ``_indexed_shared_state`` cannot prevent it — it
        keys on the connector file id, and a collision by definition comes from
        a document this connector never ingested, so the lookup returns None and
        ``shared`` falls back to the sync's own (private) intent.

        `shared` comes from _resolve_shared, never straight from self.shared: on
        a re-sync it reflects the file's current indexed state, so this is a
        no-op rather than an unrequested ownership change.
        """
        write_client = clients.opensearch
        if write_client is None:
            return
        owner, owner_name, owner_email = resolve_shared_owner_fields(
            self.user_id, self.owner_name, self.owner_email, shared
        )
        # A shared write already cleared the permission upstream (connector_sync
        # rejects shared syncs without it), so it keeps the wider scope.
        build_query = (
            build_replace_filename_query
            if shared or self.allow_anonymous_delete
            else build_owned_filename_query
        )
        for candidate in get_filename_aliases(filename):
            try:
                await write_client.update_by_query(
                    index=get_index_name(),
                    body={
                        "query": build_query(candidate, self.user_id),
                        "script": {
                            "source": """
                                if (params.shared) {
                                    ctx._source.remove('owner');
                                } else {
                                    ctx._source.owner = params.owner;
                                }
                                ctx._source.owner_name = params.owner_name;
                                ctx._source.owner_email = params.owner_email;
                            """,
                            "params": {
                                "shared": shared,
                                "owner": owner,
                                "owner_name": owner_name,
                                "owner_email": owner_email,
                            },
                        },
                    },
                    refresh=True,
                )
                invalidate_corpus_caches()
            except Exception as e:
                logger.warning(
                    "Failed to reconcile owner fields for skipped duplicate",
                    filename=candidate,
                    error=str(e),
                )

    async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
        """Process a connector file using unified methods"""
        file_task.status = TaskStatus.RUNNING
        file_task.updated_at = time.time()

        try:
            file_id = item  # item is the connector file ID

            # Get the connector and connection info
            connector = await self.connector_service.get_connector(self.connection_id)
            connection = await self.connector_service.connection_manager.get_connection(
                self.connection_id
            )
            if not connector or not connection:
                raise ValueError(f"Connection '{self.connection_id}' not found")

            connector_type = self.connector_type or connection.connector_type

            # Validate file extension early if filename is available
            VALID_EXTENSIONS = {
                "adoc",
                "asciidoc",
                "asc",
                "bmp",
                "csv",
                "dotx",
                "dotm",
                "docm",
                "docx",
                "htm",
                "html",
                "jpeg",
                "jpg",
                "md",
                "pdf",
                "png",
                "potx",
                "ppsx",
                "pptm",
                "potm",
                "ppsm",
                "pptx",
                "tiff",
                "txt",
                "xls",
                "xlsx",
                "xhtml",
                "webp",
            }
            # Only pre-validate when we have a real filename. When the filename
            # falls back to the connector file_id (e.g. a deletion event re-added
            # by sync_specific_files, where no name is known), skip this check so
            # the deletion reaches the 404 -> chunk-cleanup path below. Files that
            # still exist are re-validated after download (see below).
            if file_task.filename and file_task.filename != file_id:
                ext = file_task.filename.split(".")[-1].lower() if "." in file_task.filename else ""
                if ext not in VALID_EXTENSIONS:
                    file_task.status = TaskStatus.FAILED
                    file_task.error = f"The file '{file_task.filename}' has an incompatible type."
                    file_task.updated_at = time.time()
                    upload_task.failed_files += 1
                    return

            # Get file content from connector
            try:
                document = await connector.get_file_content(file_id)
            except (FileNotFoundError, ValueError) as e:
                msg = str(e).lower()
                if "not found" in msg or "404" in msg:
                    # File gone at source — remove its indexed chunks by the
                    # stable connector id (matches both connector_file_id and
                    # document_id) so it stops appearing in search/chat.
                    opensearch_client = (
                        self.document_service.session_manager.get_user_opensearch_client(
                            self.user_id, self.jwt_token
                        )
                    )
                    try:
                        deleted_chunks = await self._delete_connector_chunks(
                            file_id,
                            opensearch_client,
                            self.user_id,
                            shared=await self._resolve_shared(
                                file_id, opensearch_client, connector_type
                            ),
                            connector_type=connector_type,
                            ignore_errors=False,
                        )
                    except Exception:
                        file_task.status = TaskStatus.FAILED
                        file_task.error = (
                            "File no longer exists at source, but removing it "
                            "from the index failed."
                        )
                        file_task.updated_at = time.time()
                        upload_task.failed_files += 1
                        return

                    logger.info(
                        "File no longer exists at source — removed from index",
                        file_id=file_id,
                        connection_id=self.connection_id,
                        deleted_chunks=deleted_chunks,
                        source_error=str(e),
                    )
                    # Successful cleanup: the file is gone at the source and its
                    # chunks were removed. Mark completed (not skipped) so the
                    # tasks view does not treat this as a warning. Enhanced
                    # listing still surfaces this reason so the dialog can show
                    # it as Removed instead of omitting it like a normal ingest.
                    file_task.status = TaskStatus.COMPLETED
                    file_task.error = None
                    file_task.result = {
                        "status": "completed",
                        "reason": "deleted_at_source",
                        "deleted_chunks": deleted_chunks,
                        "message": (
                            f"File no longer exists at source; removed from index "
                            f"({deleted_chunks} chunk(s) deleted)."
                        ),
                    }
                    file_task.updated_at = time.time()
                    upload_task.successful_files += 1
                    return
                raise

            # Update filename in task once we have it from the connector
            file_task.filename = clean_connector_filename(document.filename, document.mimetype)

            # Re-check filename validation
            name = file_task.filename or document.filename or ""
            ext = name.split(".")[-1].lower() if "." in name else ""
            if ext not in VALID_EXTENSIONS:
                file_task.status = TaskStatus.FAILED
                file_task.error = f"The file '{name}' has an incompatible type."
                file_task.updated_at = time.time()
                upload_task.failed_files += 1
                return

            if not self.user_id:
                raise ValueError("user_id not provided to ConnectorFileProcessor")

            opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
                self.user_id, self.jwt_token
            )

            # Resolve once, up front: every owner-scoped step below (the
            # replace-delete, the rename cleanup, the owner fields written at
            # index time) has to agree on how this file is shared, or the sync
            # skips it and leaves the stale copy in place.
            shared = await self._resolve_shared(document.id, opensearch_client, connector_type)

            duplicate_action = await self.resolve_duplicate_filename(
                file_task.filename,
                opensearch_client,
                replace=self.replace_duplicates,
                owner_user_id=self.user_id,
                shared=shared,
                claim_holder=self._claim_holder(upload_task, file_task),
                allow_anonymous_delete=self.allow_anonymous_delete,
            )
            if duplicate_action in DUPLICATE_SKIP_ACTIONS:
                # Only for a real duplicate. The reconcile rewrites owner fields
                # on every chunk with this filename, and "skip_in_flight" means
                # the chunks under that name are being written right now by a
                # DIFFERENT document, whose sharing state was resolved from its
                # own file id and can differ from this one's. Reconciling then
                # would stamp this file's answer onto another file's chunks.
                if duplicate_action == "skip":
                    await self._reconcile_shared_owner(file_task.filename, shared)
                self.mark_duplicate_skipped(upload_task, file_task)
                return

            # Rename cleanup: a connector file keeps a stable id across renames,
            # but chunks are keyed by filename/content-hash, so a renamed file
            # leaves its OLD-name chunks orphaned. Drop chunks for this id whose
            # filename differs from the current one. If any were removed (a real
            # rename), force a re-ingest below so the file is re-indexed under
            # the new name instead of short-circuiting as "unchanged".
            # Match against file_task.filename — the cleaned name the file is
            # actually indexed under — so duplicate/rename detection lines up
            # with how chunks are keyed.
            renamed = (
                await self._delete_connector_chunks(
                    document.id,
                    opensearch_client,
                    self.user_id,
                    keep_filenames=get_filename_aliases(file_task.filename),
                    shared=shared,
                    connector_type=connector_type,
                )
                > 0
            )

            # Create temporary file from document content
            suffix = os.path.splitext(file_task.filename)[1]
            if not suffix:
                suffix = get_file_extension(document.mimetype)
            with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
                # Write content to temp file
                with open(tmp_path, "wb") as f:
                    f.write(document.content)

                # Compute hash
                file_hash = hash_id(tmp_path)

                if not renamed and await self.check_document_exists(file_hash, opensearch_client):
                    await self._reconcile_shared_owner(file_task.filename, shared)
                    file_task.status = TaskStatus.COMPLETED
                    file_task.result = {"status": "unchanged", "id": file_hash}
                    file_task.updated_at = time.time()
                    upload_task.successful_files += 1
                    return

                from config.settings import DISABLE_INGEST_WITH_LANGFLOW

                if (
                    not DISABLE_INGEST_WITH_LANGFLOW
                    and self.connector_service.langflow_service is not None
                ):
                    # Delete existing chunks for this document before Langflow re-ingestion
                    try:
                        from utils.opensearch_delete import (
                            collect_visible_document_ids,
                            delete_document_ids,
                        )

                        # Match both fields: bucket-connector chunks carry the
                        # raw connector id in connector_file_id (document_id is
                        # a hash), while pre-migration chunks only have it in
                        # document_id.
                        chunk_ids = await collect_visible_document_ids(
                            opensearch_client,
                            index=get_index_name(),
                            query={
                                "bool": {
                                    "should": [
                                        {"term": {"document_id": document.id}},
                                        {"term": {"connector_file_id": document.id}},
                                        # See check_document_exists: some indices
                                        # predate the explicit keyword mapping for
                                        # this field.
                                        {"term": {"connector_file_id.keyword": document.id}},
                                    ]
                                }
                            },
                        )
                        deleted_count = await delete_document_ids(
                            opensearch_client,
                            index=get_index_name(),
                            document_ids=chunk_ids,
                            refresh=True,
                        )
                        logger.info(
                            "Deleted existing chunks before Langflow re-ingestion",
                            document_id=document.id,
                            deleted_count=deleted_count,
                        )
                    except Exception as delete_err:
                        logger.warning(
                            "Failed to delete existing chunks before Langflow re-ingestion",
                            document_id=document.id,
                            error=str(delete_err),
                        )

                    # Ingest via unified Langflow pipeline (two-phase Docling + Langflow run)
                    langflow_filename, processed_mimetype = langflow_safe_filename_and_mimetype(
                        file_task.filename, document.mimetype
                    )
                    file_tuple = (langflow_filename, document.content, processed_mimetype)

                    # Extract ACL information
                    allowed_users: list[str] = []
                    allowed_groups: list[str] = []
                    allowed_principals: list[str] = []
                    allowed_principal_labels: list[dict[str, Any]] = []
                    if document.acl:
                        try:
                            allowed_users = document.acl.allowed_users or []
                            allowed_groups = document.acl.allowed_groups or []
                            allowed_principals = document.acl.allowed_principals or []
                            allowed_principal_labels = document.acl.allowed_principal_labels or []
                        except AttributeError:
                            pass

                    # Prepare tweaks
                    connector_tweak_settings = None
                    if isinstance(self.ingest_settings, dict):
                        connector_tweak_settings = dict(self.ingest_settings)
                        connector_tweak_settings.pop("embeddingModel", None)

                    tweaks = self.connector_service.langflow_service.merge_ui_ingest_settings_into_tweaks(
                        {}, connector_tweak_settings
                    )

                    config = get_openrag_config()
                    effective_ingest_settings = (
                        dict(self.ingest_settings) if self.ingest_settings else {}
                    )
                    effective_ingest_settings["ocr"] = config.knowledge.ocr
                    effective_ingest_settings["pictureDescriptions"] = (
                        config.knowledge.picture_descriptions
                    )

                    effective_owner, effective_owner_name, effective_owner_email = (
                        resolve_shared_owner_fields(
                            self.user_id, self.owner_name, self.owner_email, shared
                        )
                    )
                    file_task.document_id = document.id
                    result = await self.connector_service.langflow_service.upload_and_ingest_file(
                        file_tuple=file_tuple,
                        session_id=None,
                        tweaks=tweaks,
                        settings=effective_ingest_settings,
                        jwt_token=self.jwt_token,
                        owner=effective_owner,
                        owner_name=effective_owner_name,
                        owner_email=effective_owner_email,
                        connector_type=connector_type,
                        docling_polling_service=self.connector_service.task_service.docling_polling_service
                        if self.connector_service.task_service
                        else None,
                        file_task=file_task,
                        document_id=document.id,
                        connector_file_id=document.id,
                        source_url=document.source_url,
                        allowed_users=allowed_users,
                        allowed_groups=allowed_groups,
                        allowed_principals=allowed_principals,
                        allowed_principal_labels=allowed_principal_labels,
                        original_filename=file_task.filename,
                        original_mimetype=document.mimetype,
                        preview_mode=self.preview_mode,
                        upload_task_id=upload_task.task_id,
                        preview_user_id=self.user_id,
                    )
                    # Langflow returns "success" even when no text was extracted
                    # (e.g. image files without OCR). Verify the document actually
                    # landed in OpenSearch before declaring success.
                    # wait_for_visibility polls on an empty result to ride out
                    # OpenSearch's ~1s near-real-time window (the user-scoped
                    # client cannot force an indices:admin/refresh — it 403s).
                    if not await self.check_document_exists(
                        document.id,
                        _verification_client(opensearch_client),
                        on_error="assume_exists",
                        wait_for_visibility=True,
                        field="connector_file_id",
                    ):
                        result = {
                            "status": "error",
                            "error": "No text content could be extracted from document",
                        }

                    # Persist connector metadata (incl. modified_time) onto the
                    # Langflow-indexed chunks (keyed by document_id) so bucket-connector
                    # change detection has a stored timestamp to compare against on the
                    # next sync. Mirrors the standard path's enrichment below.
                    if result.get("status") != "error":
                        await self.connector_service._update_connector_metadata(
                            document,
                            self.user_id,
                            connector_type,
                            self.jwt_token,
                            indexed_filename=file_task.filename,
                        )
                else:
                    # Standard OpenRAG processing pipeline (process_document_standard)
                    standard_kwargs: dict[str, Any] = {}
                    if isinstance(self.ingest_settings, dict):
                        s = self.ingest_settings
                        em = s.get("embeddingModel")
                        if isinstance(em, str) and em.strip():
                            standard_kwargs["embedding_model"] = em.strip()
                        for ui_key, param in (
                            ("chunkSize", "chunk_size"),
                            ("chunkOverlap", "chunk_overlap"),
                        ):
                            raw = s.get(ui_key)
                            if raw is not None:
                                try:
                                    standard_kwargs[param] = int(raw)
                                except (TypeError, ValueError):
                                    pass
                    config = get_openrag_config()
                    standard_kwargs["ocr"] = config.knowledge.ocr
                    standard_kwargs["picture_descriptions"] = config.knowledge.picture_descriptions

                    result = await self.process_document_standard(
                        file_path=tmp_path,
                        file_hash=file_hash,
                        owner_user_id=self.user_id,
                        original_filename=file_task.filename,
                        jwt_token=self.jwt_token,
                        owner_name=self.owner_name,
                        owner_email=self.owner_email,
                        file_size=len(document.content),
                        connector_type=connector_type,
                        acl=document.acl,
                        connector_file_id=document.id,
                        shared=shared,
                        **standard_kwargs,
                    )

                    # Update indexed chunks with connector-specific metadata
                    if result["status"] in ["indexed", "unchanged"]:
                        await self.connector_service._update_connector_metadata(
                            document,
                            self.user_id,
                            connector_type,
                            self.jwt_token,
                            indexed_filename=file_task.filename,
                        )

                    # Add connector-specific metadata
                    result.update(
                        {
                            "source_url": document.source_url,
                            "document_id": document.id,
                        }
                    )

            if result.get("status") == "error":
                file_task.status = TaskStatus.FAILED
                file_task.error = result.get("error") or "Failed to process document"
                file_task.updated_at = time.time()
                upload_task.failed_files += 1
            else:
                file_task.status = TaskStatus.COMPLETED
                file_task.result = result
                file_task.updated_at = time.time()
                upload_task.successful_files += 1

        except Exception as e:
            file_task.status = TaskStatus.FAILED
            file_task.error = str(e) or repr(e)
            file_task.updated_at = time.time()
            upload_task.failed_files += 1
            raise


class S3FileProcessor(TaskProcessor):
    """Processor for files stored in S3 buckets"""

    def __init__(
        self,
        document_service,
        bucket: str,
        s3_client=None,
        owner_user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        models_service=None,
        docling_service=None,
        replace_duplicates: bool = False,
        *,
        allow_anonymous_delete: bool,
    ):
        import boto3

        super().__init__(
            document_service,
            models_service,
            docling_service,
        )
        self.bucket = bucket
        self.s3_client = s3_client or boto3.client("s3")
        self.owner_user_id = owner_user_id
        self.jwt_token = jwt_token
        self.owner_name = owner_name
        self.owner_email = owner_email
        self.replace_duplicates = replace_duplicates
        # The uploading user's resolved knowledge:delete:anonymous; see
        # delete_document_by_filename.
        self.allow_anonymous_delete = allow_anonymous_delete

    async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
        """Download an S3 object and process it using DocumentService"""
        import time

        from models.tasks import TaskStatus

        file_task.status = TaskStatus.RUNNING
        file_task.updated_at = time.time()

        try:
            # The S3 key doubles as the indexed filename, so the duplicate
            # gate can run before downloading the object.
            opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
                self.owner_user_id, self.jwt_token
            )
            duplicate_action = await self.resolve_duplicate_filename(
                item,
                opensearch_client,
                replace=self.replace_duplicates,
                owner_user_id=self.owner_user_id,
                claim_holder=self._claim_holder(upload_task, file_task),
                allow_anonymous_delete=self.allow_anonymous_delete,
            )
            if duplicate_action in DUPLICATE_SKIP_ACTIONS:
                self.mark_duplicate_skipped(upload_task, file_task)
                return

            suffix = os.path.splitext(item)[1]
            with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
                # Download object to temporary file
                with open(tmp_path, "wb") as tmp_file:
                    self.s3_client.download_fileobj(self.bucket, item, tmp_file)

                # Compute hash
                file_hash = hash_id(tmp_path)

                # Get object size
                try:
                    obj_info = self.s3_client.head_object(Bucket=self.bucket, Key=item)
                    file_size = obj_info.get("ContentLength", 0)
                except Exception:
                    file_size = 0

                # Use consolidated standard processing
                result = await self.process_document_standard(
                    file_path=tmp_path,
                    file_hash=file_hash,
                    owner_user_id=self.owner_user_id,
                    original_filename=item,  # Use S3 key as filename
                    jwt_token=self.jwt_token,
                    owner_name=self.owner_name,
                    owner_email=self.owner_email,
                    file_size=file_size,
                    connector_type="s3",
                )

                result["path"] = f"s3://{self.bucket}/{item}"
                if result.get("status") == "error":
                    file_task.status = TaskStatus.FAILED
                    file_task.error = result.get("error") or "Failed to process document"
                    upload_task.failed_files += 1
                else:
                    file_task.status = TaskStatus.COMPLETED
                    file_task.result = result
                    upload_task.successful_files += 1

        except Exception as e:
            file_task.status = TaskStatus.FAILED
            file_task.error = str(e) or repr(e)
            upload_task.failed_files += 1
        finally:
            file_task.updated_at = time.time()


class LangflowFileProcessor(TaskProcessor):
    """Processor for Langflow file uploads with two-phase Docling + Langflow ingestion."""

    def __init__(
        self,
        langflow_file_service,
        session_manager,
        owner_user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        session_id: str = None,
        tweaks: dict = None,
        settings: dict = None,
        replace_duplicates: bool = False,
        connector_type: str = "local",
        docling_polling_service=None,
        preview_mode: bool = False,
        *,
        allow_anonymous_delete: bool,
    ):
        super().__init__()
        self.langflow_file_service = langflow_file_service
        self.session_manager = session_manager
        self.owner_user_id = owner_user_id
        self.jwt_token = jwt_token
        self.owner_name = owner_name
        self.owner_email = owner_email
        self.session_id = session_id
        self.tweaks = tweaks or {}
        self.settings = settings
        self.replace_duplicates = replace_duplicates
        self.connector_type = connector_type
        self.docling_polling_service = docling_polling_service
        self.preview_mode = preview_mode
        # The uploading user's resolved knowledge:delete:anonymous; see
        # delete_document_by_filename.
        self.allow_anonymous_delete = allow_anonymous_delete

    async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
        """Process a file path using LangflowFileService upload_and_ingest_file"""
        # Update task status
        file_task.status = TaskStatus.RUNNING
        file_task.updated_at = time.time()

        try:
            # Use the ORIGINAL filename stored in file_task (not the transformed temp path)
            # This ensures we check/store the original filename with spaces, etc.
            original_filename = file_task.filename or os.path.basename(item)

            # Check if document with same filename already exists
            opensearch_client = self.session_manager.get_user_opensearch_client(
                self.owner_user_id, self.jwt_token
            )

            duplicate_action = await self.resolve_duplicate_filename(
                original_filename,
                opensearch_client,
                replace=self.replace_duplicates,
                owner_user_id=self.owner_user_id,
                claim_holder=self._claim_holder(upload_task, file_task),
                allow_anonymous_delete=self.allow_anonymous_delete,
            )
            if duplicate_action in DUPLICATE_SKIP_ACTIONS:
                self.mark_duplicate_skipped(upload_task, file_task)
                return

            # Compute the content hash early — before reading file bytes into
            # memory and before submitting any work to Docling so the duplicate
            # guard fires as cheaply as possible.
            #
            # The guard only runs on the "proceed" path (no filename match).
            # When duplicate_action == "replaced" the caller asked to replace an
            # existing same-name document: the old chunks were already deleted
            # above, so we must not short-circuit here even if the hash matches
            # (that would leave the index empty after the delete).
            file_hash = hash_id(item)
            file_task.document_id = file_hash

            if duplicate_action == "proceed" and await self.check_document_exists(
                file_hash, opensearch_client
            ):
                # Identical content is already indexed under a different filename.
                # Report it as a warning rather than silently overwriting.
                # Mirrors the connector path (processors.py:1212) but uses a
                # distinct reason so the UI can distinguish it from a filename
                # collision.
                file_task.status = TaskStatus.SKIPPED
                file_task.error = None
                file_task.result = {
                    "status": "skipped",
                    "reason": "duplicate_content",
                    "warning": DUPLICATE_CONTENT_WARNING,
                    "document_id": file_hash,
                }
                file_task.updated_at = time.time()
                upload_task.successful_files += 1
                return

            # Read file content for processing
            with open(item, "rb") as f:
                content = f.read()

            # Create file tuple for upload using ORIGINAL filename
            # This ensures the document is indexed with the original name
            original_mimetype, _ = mimetypes.guess_type(original_filename)
            if not original_mimetype:
                original_mimetype = "application/octet-stream"

            # Langflow's docling chokes on text/plain — rename .txt -> .md.
            langflow_filename, content_type = langflow_safe_filename_and_mimetype(
                original_filename, original_mimetype
            )
            file_tuple = (langflow_filename, content, content_type)

            effective_jwt = self.jwt_token
            if self.session_manager and not effective_jwt:
                effective_jwt = self.session_manager.get_effective_jwt_token(
                    self.owner_user_id,
                    None,
                )

            # Prepare metadata tweaks similar to API endpoint
            final_tweaks = self.tweaks.copy() if self.tweaks else {}

            # Build settings with fresh OCR/pictureDescriptions from live
            # config so retries pick up configuration changes.
            config = get_openrag_config()
            effective_settings = dict(self.settings) if self.settings else {}
            effective_settings["ocr"] = config.knowledge.ocr
            effective_settings["pictureDescriptions"] = config.knowledge.picture_descriptions

            # Process file using langflow service. Passing the polling
            # service triggers the two-phase model: backend polls Docling,
            # then invokes Langflow only after SUCCESS. file_task is passed
            # so phase / docling_status are tracked on the task record.
            result = await self.langflow_file_service.upload_and_ingest_file(
                file_tuple=file_tuple,
                session_id=self.session_id,
                tweaks=final_tweaks,
                settings=effective_settings,
                jwt_token=effective_jwt,
                owner=self.owner_user_id,
                owner_name=self.owner_name,
                owner_email=self.owner_email,
                connector_type=self.connector_type,
                docling_polling_service=self.docling_polling_service,
                file_task=file_task,
                document_id=file_hash,
                original_filename=original_filename,
                original_mimetype=original_mimetype,
                preview_mode=self.preview_mode,
                upload_task_id=upload_task.task_id,
                preview_user_id=self.owner_user_id,
            )

            # Langflow returns "success" even when no text was extracted
            # (e.g. image files without OCR). Verify the document actually
            # landed in OpenSearch before declaring success. We key off the
            # filename — the identifier this path already uses for dedup and
            # delete (see check_filename_exists / delete_document_by_filename
            # above). The document_id (hash_id(item) == content hash) is now
            # threaded through to Langflow so preview-mode index proof can look
            # chunks up by document_id, but verification stays filename-based to
            # match this path's existing dedup/delete semantics.
            #
            # wait_for_visibility polls on an empty result so the just-written
            # chunks become visible within OpenSearch's near-real-time refresh
            # window. We cannot force a refresh here: the user-scoped client
            # lacks the indices:admin/refresh privilege (it 403s).
            if not await self.check_filename_exists(
                original_filename,
                _verification_client(opensearch_client),
                wait_for_visibility=True,
            ):
                file_task.status = TaskStatus.FAILED
                file_task.error = "No text content could be extracted from document"
                file_task.updated_at = time.time()
                upload_task.failed_files += 1
            else:
                # Update task with success
                file_task.status = TaskStatus.COMPLETED
                file_task.result = result
                file_task.updated_at = time.time()
                upload_task.successful_files += 1

        except Exception as e:
            # Update task with failure
            file_task.status = TaskStatus.FAILED
            file_task.error = str(e) or repr(e)
            file_task.updated_at = time.time()
            upload_task.failed_files += 1
            raise