Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
91.9 KB
Raw Blame
import asyncio
import os
import random
import re
import time
import traceback
import uuid
from collections.abc import Coroutine
from typing import Any, TypeVar

from models.tasks import (
    DoclingPhaseStatus,
    FileTask,
    IngestionPhase,
    TaskDeleteResult,
    TaskStatus,
    UploadTask,
)
from observability.tracing import trace_context
from session_manager import AnonymousUser
from utils.filename_claims import (
    INFLIGHT_CLAIM_WINNER_FAILED_ERROR,
    claim_holder,
    filename_claims,
)
from utils.gpu_detection import get_worker_count
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient

T = TypeVar("T")

logger = get_logger(__name__)

# Substrings that indicate a permanent file/content problem (not worth retrying).
_NON_RETRYABLE_FILE_ERROR_MARKERS = (
    "corrupt",
    "corrupted",
    "corruption",
    "invalid file",
    "invalid document",
    "unsupported format",
    "unsupported file",
    "malformed",
    "damaged",
    "not a zip file",
    "bad zipfile",
    "failed to parse",
    "could not be parsed",
    "cannot parse",
    "no text content could be extracted",
    "empty or unreadable",
    "unreadable",
    "validationerror",
)

# Docling polling errors that are transient (service/timeout), not bad file content.
_DOCLING_TRANSIENT_ERROR_MARKERS = (
    "(timeout)",
    "timed out",
    "polling exceeded",
    "polling timed out",
)

# Connector/download/network failures seen while file is still in Docling phase.
# These are infrastructure/transient and should be retryable.
_TRANSIENT_CONNECTIVITY_ERROR_MARKERS = (
    "all connection attempts failed",
    "connection refused",
    "connection reset",
    "temporary failure in name resolution",
    "name or service not known",
    "service unavailable",
    "network is unreachable",
    "server disconnected",
    "disconnected without sending a response",
    "remote protocol error",
    "broken pipe",
)


# Raster image formats that require OCR to produce any text content.
_OCR_REQUIRED_EXTENSIONS = frozenset({"bmp", "jpeg", "jpg", "png", "tiff", "webp"})


def _is_non_retryable_file_error(error: str) -> bool:
    lowered = error.lower()
    return any(marker in lowered for marker in _NON_RETRYABLE_FILE_ERROR_MARKERS)


def _is_ocr_required_file(filename: str) -> bool:
    """Return True if the file is a raster image that requires OCR to produce text."""
    if not filename or "." not in filename:
        return False
    return filename.rsplit(".", 1)[-1].lower() in _OCR_REQUIRED_EXTENSIONS


_TASK_CANCELLATION_ERROR_MARKERS = (
    "task cancelled by user",
    "file cancelled by user",
    "file processing task cancelled",
)


def _is_task_cancellation_error(error: str) -> bool:
    lowered = error.lower()
    return any(marker in lowered for marker in _TASK_CANCELLATION_ERROR_MARKERS)


# opensearch-py renders transport failures as "ClassName(<status>, ...)". For the
# by-query APIs (_update_by_query / _delete_by_query) the response carries no
# top-level "error" key, so Connection._raise_error falls back to using the whole
# JSON response body as the exception message — several hundred characters of
# payload where a reason string would normally be.
#
# Classifying these early matters twice over: an infrastructure fault never
# reaches the UI as a raw exception repr, and the JSON body cannot trip the
# substring heuristics further down (a mapper_parsing_exception body contains
# "failed to parse", which would otherwise read as a corrupted file).
#
# ConnectionError / ConnectionTimeout are deliberately excluded: they render with
# a different __str__ and are already covered by _is_transient_connectivity_error.
_OPENSEARCH_TRANSPORT_ERROR_RE = re.compile(
    r"\b(?:ConflictError|NotFoundError|RequestError|TransportError"
    r"|AuthenticationException|AuthorizationException)\((\d{3})[,)]"
)


def _opensearch_failure_metadata(error: str) -> dict | None:
    """Classify an opensearch-py transport failure surfaced as a task error.

    Returns None when the error is not an OpenSearch transport failure, so the
    caller falls through to the existing classification branches.
    """
    if not error:
        return None
    match = _OPENSEARCH_TRANSPORT_ERROR_RE.search(error)
    if not match:
        return None

    status = match.group(1)
    if status == "409":
        message = (
            "The search index was busy and this change conflicted with another "
            "update. Nothing is wrong with the file — retry ingestion."
        )
    else:
        message = (
            f"The search index rejected this document (OpenSearch error {status}). "
            "This is an infrastructure problem, not a problem with the file. "
            "Retry ingestion, and check the backend logs if it persists."
        )
    # RETRYABLE across the board: of the two available values, USER_ACTIONABLE
    # would be actively wrong — there is nothing the uploader can change about
    # their file to fix a search-index fault.
    return {
        "component": "opensearch",
        "failure_phase": "indexing",
        "user_facing_message": message,
        "actionable_by": "RETRYABLE",
    }


def _provider_credential_failure_metadata(error: str) -> dict | None:
    """Classify invalid/revoked provider API key failures for ingestion toasts/UI."""
    if not error:
        return None

    from api.provider_validation import (
        is_provider_credential_error,
        sanitize_provider_error_content,
    )

    cleaned = sanitize_provider_error_content(error)
    if not (is_provider_credential_error(error) or is_provider_credential_error(cleaned)):
        return None

    return {
        "component": "openrag",
        "failure_phase": "embedding",
        "user_facing_message": cleaned,
        "actionable_by": "USER_ACTIONABLE",
    }


def _context_window_failure_metadata(error: str) -> dict | None:
    """Classify a chunk the embedding model rejected as longer than its input limit."""
    from api.provider_validation import context_window_token_counts, is_context_window_error

    if not is_context_window_error(error):
        return None

    limit, requested = context_window_token_counts(error)
    if limit and requested:
        size = f" ({requested} tokens; the limit is {limit})"
    elif limit:
        size = f" (the limit is {limit} tokens)"
    else:
        size = ""
    return {
        "component": "openrag",
        "failure_phase": "embedding",
        "user_facing_message": (
            f"A section of this document is longer than the embedding model accepts{size}. "
            "Try a smaller chunk size or an embedding model with a larger input limit, "
            "then retry ingestion."
        ),
        "actionable_by": "USER_ACTIONABLE",
    }


def _is_transient_connectivity_error(error: str) -> bool:
    lowered = error.lower()
    return any(marker in lowered for marker in _TRANSIENT_CONNECTIVITY_ERROR_MARKERS)


def _is_docling_transient_error(error: str) -> bool:
    lowered = error.lower()
    # Legacy timeout messages emitted by Docling polling path.
    if "docling conversion did not complete" in lowered:
        return any(marker in lowered for marker in _DOCLING_TRANSIENT_ERROR_MARKERS)

    # Connector-originated transient network failures can still surface while
    # file phase is DOCLING (before conversion can complete).
    return _is_transient_connectivity_error(error)


def _transient_connectivity_user_message(error: str) -> str:
    lowered = error.lower()
    if any(
        marker in lowered
        for marker in (
            "disconnect",
            "without sending a response",
            "connection refused",
            "connection reset",
            "broken pipe",
            "remote protocol",
        )
    ):
        return "Ingestion service connection was lost. Please retry ingestion."
    if any(marker in lowered for marker in ("timeout", "timed out")):
        return "Document processing timed out. Please retry ingestion."
    return "Ingestion service was temporarily unavailable. Please retry ingestion."


def _is_langflow_transport_failure(error: str) -> bool:
    """HTTP client / Langflow transport failures on the connector ingest path."""
    lowered = error.lower()
    return any(
        marker in lowered
        for marker in (
            "server disconnected",
            "disconnected without sending a response",
            "remote protocol error",
            "read error",
            "write error",
        )
    )


class IngestionTimeoutError(Exception):
    """Raised when file processing exceeds the configured timeout"""

    pass


class TaskService:
    # Cleanup interval in seconds (2 hours)
    CLEANUP_INTERVAL_SECONDS = 2 * 60 * 60

    def __init__(
        self,
        document_service=None,
        models_service=None,
        ingestion_timeout=3600,
        docling_service=None,
        docling_polling_service=None,
        session_manager=None,
    ):
        self.document_service = document_service
        self.models_service = models_service
        self.docling_service = docling_service
        self.session_manager = session_manager
        # Backend-side Docling polling coordinator. Injected by the container
        # so LangflowFileProcessor receives it from the established DI chain
        # rather than constructing it inline. None disables the two-phase
        # flow and falls back to the legacy single-call ingestion path.
        self.docling_polling_service = docling_polling_service
        self.task_store: dict[str, dict[str, UploadTask]] = {}  # user_id -> {task_id -> UploadTask}
        self.background_tasks = set()
        self.ingestion_timeout = ingestion_timeout
        self._cleanup_task: asyncio.Task | None = None
        # Locks for task counter updates, keyed by task_id
        # Kept separate from UploadTask to maintain serialization compatibility
        self._task_locks: dict[str, asyncio.Lock] = {}
        # Track individual file processing tasks for cancellation: (task_id, file_path) -> asyncio.Task
        self._file_tasks: dict[tuple[str, str], asyncio.Task] = {}
        # Global semaphore to limit concurrent file processing across all tasks.
        # TaskService is a singleton, so this limits concurrency system-wide.
        self._worker_count = get_worker_count()
        self._processing_semaphore = asyncio.Semaphore(self._worker_count)

    def _get_task_lock(self, task_id: str) -> asyncio.Lock:
        """Get or create a lock for a specific task's counter updates"""
        if task_id not in self._task_locks:
            self._task_locks[task_id] = asyncio.Lock()
        return self._task_locks[task_id]

    def start_cleanup_scheduler(self) -> None:
        """Start the periodic cleanup background task.

        Should be called once after the event loop is running (e.g., during app startup).
        """
        if self._cleanup_task is None or self._cleanup_task.done():
            self._cleanup_task = asyncio.create_task(self._periodic_cleanup())
            logger.info(
                "Started periodic task cleanup scheduler",
                interval_seconds=self.CLEANUP_INTERVAL_SECONDS,
            )

    async def _periodic_cleanup(self) -> None:
        """Periodically clean up old completed/failed tasks."""
        while True:
            try:
                await asyncio.sleep(self.CLEANUP_INTERVAL_SECONDS)
                cleaned = await self.cleanup_old_tasks()
                if cleaned > 0:
                    logger.debug("Periodic cleanup completed", tasks_cleaned=cleaned)
            except asyncio.CancelledError:
                logger.debug("Periodic cleanup task cancelled")
                raise
            except Exception as e:
                logger.warning("Error during periodic cleanup", error=str(e))

    async def exponential_backoff_delay(
        self, retry_count: int, base_delay: float = 1.0, max_delay: float = 60.0
    ) -> None:
        """Apply exponential backoff with jitter"""
        delay = min(base_delay * (2**retry_count) + random.uniform(0, 1), max_delay)
        await asyncio.sleep(delay)

    async def _process_with_timeout(
        self, coro: Coroutine[Any, Any, T], timeout_seconds: int | None = None
    ) -> T:
        """Wrapper to add timeout protection to file processing

        Args:
            coro: Coroutine to execute with timeout
            timeout_seconds: Timeout in seconds (uses self.ingestion_timeout if None)

        Returns:
            The result of the coroutine

        Raises:
            IngestionTimeoutError: If processing exceeds timeout
        """
        timeout: int = timeout_seconds or self.ingestion_timeout
        try:
            return await asyncio.wait_for(coro, timeout=timeout)
        except TimeoutError:
            raise IngestionTimeoutError(
                f"File processing timed out after {timeout} seconds."
            ) from None

    async def create_upload_task(
        self,
        user_id: str,
        file_paths: list,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        original_filenames: dict | None = None,
        replace_duplicates: bool = False,
        settings: dict | None = None,
        preview_mode: bool = False,
        *,
        allow_anonymous_delete: bool,
        existing_task_id: str | None = None,
    ) -> str:
        """Create a new upload task for bulk file processing.

        ``allow_anonymous_delete`` is the uploading user's resolved
        ``knowledge:delete:anonymous`` and has no default; see
        ``TaskProcessor.delete_document_by_filename``.
        """
        # Use default DocumentFileProcessor with user context
        from models.processors import DocumentFileProcessor

        processor = DocumentFileProcessor(
            self.document_service,
            models_service=self.models_service,
            owner_user_id=user_id,
            jwt_token=jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
            docling_service=self.docling_service,
            replace_duplicates=replace_duplicates,
            session_manager=self.session_manager,
            settings=settings,
            allow_anonymous_delete=allow_anonymous_delete,
        )
        return await self.create_custom_task(
            user_id,
            file_paths,
            processor,
            original_filenames=original_filenames,
            temp_file_paths=file_paths,
            preview_mode=preview_mode,
            existing_task_id=existing_task_id,
        )

    async def create_langflow_upload_task(
        self,
        user_id: str,
        file_paths: list,
        langflow_file_service,
        session_manager,
        original_filenames: dict | None = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        session_id: str = None,
        tweaks: dict = None,
        settings: dict = None,
        replace_duplicates: bool = False,
        connector_type: str = "local",
        existing_task_id: str = None,
        temp_file_paths: list | None = None,
        preview_mode: bool = False,
        *,
        allow_anonymous_delete: bool,
    ) -> str:
        """Create a new upload task for Langflow file processing with upload and ingest.

        ``allow_anonymous_delete`` is the uploading user's resolved
        ``knowledge:delete:anonymous`` and has no default; see
        ``TaskProcessor.delete_document_by_filename``.
        """
        # Use LangflowFileProcessor with user context
        from models.processors import LangflowFileProcessor

        processor = LangflowFileProcessor(
            langflow_file_service=langflow_file_service,
            session_manager=session_manager,
            owner_user_id=user_id,
            jwt_token=jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
            session_id=session_id,
            tweaks=tweaks,
            settings=settings,
            replace_duplicates=replace_duplicates,
            connector_type=connector_type,
            docling_polling_service=self.docling_polling_service,
            preview_mode=preview_mode,
            allow_anonymous_delete=allow_anonymous_delete,
        )
        return await self.create_custom_task(
            user_id,
            file_paths,
            processor,
            original_filenames,
            existing_task_id=existing_task_id,
            temp_file_paths=temp_file_paths if temp_file_paths is not None else file_paths,
            preview_mode=preview_mode,
        )

    def record_rejected_uploads(
        self,
        user_id: str,
        rejections: list[tuple[str, int]],
        *,
        task_id: str | None = None,
    ) -> str:
        """Keep files the client did not send because they exceed the upload limit.

        Attached to ``task_id`` when that task exists, already counted as
        failed, so the panel shows them after the toasts are gone. Otherwise a
        completed task that contains only these failures.
        """
        from utils.upload_limits import max_upload_mb, upload_too_large_message

        if not rejections:
            raise ValueError("rejections must not be empty")

        limit_mb = max_upload_mb()
        store_user_id = user_id or AnonymousUser().user_id
        upload_task = self._find_upload_task(task_id) if task_id else None
        created = upload_task is None
        if upload_task is None:
            upload_task = UploadTask(
                task_id=task_id or str(uuid.uuid4()),
                total_files=0,
                status=TaskStatus.COMPLETED,
            )
            if store_user_id not in self.task_store:
                self.task_store[store_user_id] = {}
            self.task_store[store_user_id][upload_task.task_id] = upload_task

        for index, (filename, size_bytes) in enumerate(rejections):
            key = f"over-limit:{upload_task.task_id}:{index}:{filename}"
            upload_task.file_tasks[key] = FileTask(
                file_path=key,
                filename=filename,
                status=TaskStatus.FAILED,
                error=upload_too_large_message(filename, size_bytes, limit_mb),
            )
            upload_task.total_files += 1
            upload_task.failed_files += 1
            upload_task.processed_files += 1
        upload_task.updated_at = time.time()
        if created:
            upload_task.status = TaskStatus.COMPLETED
        return upload_task.task_id

    async def create_url_upload_task(
        self,
        owner_user_id: str,
        docs_url: str,
        crawl_depth: int,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        connector_type: str = "openrag_docs",
        is_sample_data: bool = False,
        existing_task_id: str = None,
    ) -> str:
        """Create a new upload task for traditional (non-Langflow) URL ingestion."""
        from models.url import UrlProcessor

        processor = UrlProcessor(
            document_service=self.document_service,
            models_service=self.models_service,
            docs_url=docs_url,
            crawl_depth=crawl_depth,
            owner_user_id=owner_user_id,
            jwt_token=jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
            connector_type=connector_type,
            is_sample_data=is_sample_data,
        )
        return await self.create_custom_task(
            owner_user_id, [docs_url], processor, existing_task_id=existing_task_id
        )

    async def create_langflow_url_upload_task(
        self,
        owner_user_id: str,
        docs_url: str,
        crawl_depth: int,
        langflow_file_service,
        session_manager,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        connector_type: str = "openrag_docs",
        prevent_outside: bool = True,
        tweaks: dict = None,
        existing_task_id: str = None,
    ) -> str:
        """Create a new upload task for Langflow URL ingestion."""
        from models.url import LangflowUrlProcessor

        processor = LangflowUrlProcessor(
            langflow_file_service=langflow_file_service,
            session_manager=session_manager,
            docs_url=docs_url,
            crawl_depth=crawl_depth,
            owner_user_id=owner_user_id,
            jwt_token=jwt_token,
            owner_name=owner_name,
            owner_email=owner_email,
            connector_type=connector_type,
            prevent_outside=prevent_outside,
            tweaks=tweaks,
        )
        return await self.create_custom_task(
            owner_user_id, [docs_url], processor, existing_task_id=existing_task_id
        )

    async def create_custom_task(
        self,
        user_id: str,
        items: list,
        processor,
        original_filenames: dict | None = None,
        existing_task_id: str = None,
        temp_file_paths: list | None = None,
        preview_mode: bool = False,
    ) -> str:
        """Create a new task with custom processor for any type of items"""
        import os

        # Store anonymous tasks under a stable key so they can be retrieved later
        store_user_id = user_id or AnonymousUser().user_id
        task_id = existing_task_id or str(uuid.uuid4())

        # Create file tasks with original filenames if provided
        normalized_originals = (
            {str(k): v for k, v in original_filenames.items()} if original_filenames else {}
        )
        file_tasks = {
            str(item): FileTask(
                file_path=str(item),
                filename=normalized_originals.get(str(item), os.path.basename(str(item))),
            )
            for item in items
        }

        task_already_exists = (
            existing_task_id
            and store_user_id in self.task_store
            and existing_task_id in self.task_store[store_user_id]
        )
        if task_already_exists:
            upload_task = self.task_store[store_user_id][existing_task_id]
            upload_task.file_tasks.update(file_tasks)
            upload_task.total_files += len(items)
            upload_task.status = TaskStatus.RUNNING
        else:
            upload_task = UploadTask(
                task_id=task_id,
                total_files=len(items),
                file_tasks=file_tasks,
                preview_mode=preview_mode,
            )
            upload_task.processor = processor
            if store_user_id not in self.task_store:
                self.task_store[store_user_id] = {}
            self.task_store[store_user_id][task_id] = upload_task

        # Store temp file paths for cleanup after processing
        if temp_file_paths:
            if upload_task.temp_file_paths is None:
                upload_task.temp_file_paths = []
            normalized_temp_paths = [str(path) for path in temp_file_paths]
            unknown_temp_paths = [
                path for path in normalized_temp_paths if path not in upload_task.file_tasks
            ]
            if unknown_temp_paths:
                logger.warning(
                    "temp_file_paths do not match file_tasks keys; retry retention may fail",
                    task_id=task_id,
                    unknown_temp_paths=unknown_temp_paths,
                )
            upload_task.temp_file_paths.extend(normalized_temp_paths)

        # Start background processing
        background_task = asyncio.create_task(
            self.background_custom_processor(store_user_id, task_id, items, processor)
        )
        self.background_tasks.add(background_task)
        background_task.add_done_callback(self.background_tasks.discard)

        # Store the task whenever this call creates one. URL sources reserve an
        # ID before enqueueing, so `existing_task_id` alone does not mean the
        # task existed already.
        if not task_already_exists:
            upload_task.background_task = background_task

        # Send telemetry event for task creation with metadata
        asyncio.create_task(
            TelemetryClient.send_event(
                Category.TASK_OPERATIONS,
                MessageId.ORB_TASK_CREATED,
                metadata={
                    "total_files": len(items),
                    "processor_type": processor.__class__.__name__,
                },
            )
        )

        return task_id

    def _get_display_filenames(self, upload_task: UploadTask) -> list[str]:
        filenames: list[str] = [
            task.filename or os.path.basename(task.file_path)
            for task in upload_task.file_tasks.values()
        ]

        if len(filenames) <= 3:
            # e.g. ['book-1.xlsx']
            # e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx']
            return filenames
        # e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx', '...']
        return filenames[:3] + ["..."]

    def _format_duration(self, duration: float | int) -> str:
        """
        Convert specified duration (seconds) into a human-readable string:
        - < 60 s     → "45s"
        - < 60 min   → "3m 42s"
        - ≥ 60 min   → "2h 14m 35s"
        """
        total_seconds: int = max(0, int(duration))

        if total_seconds < 60:
            return f"{total_seconds}s"

        mins, secs = divmod(total_seconds, 60)

        if mins < 60:
            return f"{mins}m {secs}s"

        hours, mins = divmod(mins, 60)

        return f"{hours}h {mins}m {secs}s"

    def _find_upload_task(self, task_id: str) -> UploadTask | None:
        """The task with this id, whichever user's store holds it.

        A claim conflict is not confined to one task: the registry keys names by
        ownership scope, not by task, so a file in one task can lose a name to a
        file in another — two uploads in flight at once, or an upload racing a
        connector sync.
        """
        for tasks in self.task_store.values():
            found = tasks.get(task_id)
            if found is not None:
                return found
        return None

    async def _fail_files_stranded_by(self, refused_holders: set[str]) -> None:
        """Fail the files that were skipped for a name this one never indexed.

        The duplicate gate decides a losing file's outcome the moment it loses
        the claim, before the winner has ingested anything. When the winner then
        fails, nothing landed under that name, so the loser's "skipped, a file
        with this name already exists" describes something that never happened —
        and it is counted as a successful file, which is a document the user
        never got. SKIPPED is also not a retry candidate (retry_failed_files
        takes FAILED only), so nothing would ever recover it.

        Failing it instead says what happened and puts it back within reach of a
        retry, which re-runs the gate and resolves correctly either way: it
        ingests if the name is genuinely free, and skips again as a duplicate if
        something else holds it by then.

        Each loser is corrected inside its own task, which may not be this one
        and may already have finished. That stays consistent: processed_files is
        untouched, one successful file becomes a failed one under that task's
        lock, and a COMPLETED task carrying failures is the ordinary shape retry
        operates on — it refuses only while a task is still RUNNING. The
        completion event for such a task was emitted with the earlier counts,
        which leaves a stale statistic behind rather than inconsistent state.
        """
        for holder in refused_holders:
            loser_task_id, _, file_key = holder.partition(":")
            if not file_key:
                continue
            owning_task = self._find_upload_task(loser_task_id)
            if owning_task is None:
                # Task already evicted from the store; nothing left to correct.
                continue
            stranded = owning_task.file_tasks.get(file_key)
            if stranded is None or stranded.status != TaskStatus.SKIPPED:
                continue
            # Only the in-flight skip is ours to reverse; a file skipped against
            # something already indexed is unaffected by this failure.
            if (stranded.result or {}).get("reason") != "duplicate_filename":
                continue

            stranded.status = TaskStatus.FAILED
            stranded.error = INFLIGHT_CLAIM_WINNER_FAILED_ERROR
            stranded.result = None
            stranded.updated_at = time.time()
            async with self._get_task_lock(loser_task_id):
                if owning_task.successful_files > 0:
                    owning_task.successful_files -= 1
                owning_task.failed_files += 1
                owning_task.updated_at = time.time()
            logger.info(
                "Reversed an in-flight duplicate skip after the ingesting file failed",
                task_id=loser_task_id,
                file_path=stranded.file_path,
                filename=stranded.filename,
            )

    async def background_custom_processor(
        self, user_id: str, task_id: str, items: list, processor=None
    ) -> None:
        """Background task to process items using custom processor"""
        try:
            upload_task: UploadTask = self.task_store[user_id][task_id]
            upload_task.status = TaskStatus.RUNNING
            upload_task.updated_at = time.time()

            processor = processor or upload_task.processor

            logger.info(
                "Upload / ingestion task started",
                task_number=upload_task.sequence_number,
                task_id=task_id,
                total_files=upload_task.total_files,
                filenames=self._get_display_filenames(upload_task),
                processor_type=processor.__class__.__name__,
                user_id=user_id,
                worker_count=self._worker_count,
            )

            # Process items with limited concurrency using the global semaphore
            # - Limits concurrency across all tasks, not just within this one
            # - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
            async def process_with_semaphore(item, item_key: str):
                async with self._processing_semaphore:
                    file_task = upload_task.file_tasks[item_key]

                    # Skip files that were already cancelled or failed before reaching the worker
                    if file_task.status in [
                        TaskStatus.FAILED,
                        TaskStatus.COMPLETED,
                        TaskStatus.SKIPPED,
                    ]:
                        logger.info(
                            "File processing task skipped (already terminal)",
                            task_number=upload_task.sequence_number,
                            task_id=task_id,
                            file_path=file_task.file_path,
                            status=file_task.status.value,
                        )
                        # Increment processed_files for cancelled files so task can complete
                        async with self._get_task_lock(task_id):
                            upload_task.processed_files += 1
                        return

                    # Check again after acquiring semaphore - file may have been cancelled while waiting
                    if file_task.status in [
                        TaskStatus.FAILED,
                        TaskStatus.COMPLETED,
                        TaskStatus.SKIPPED,
                    ]:
                        logger.info(
                            "File processing task skipped (cancelled while waiting for semaphore)",
                            task_number=upload_task.sequence_number,
                            task_id=task_id,
                            file_path=file_task.file_path,
                            status=file_task.status.value,
                        )
                        # Increment processed_files for cancelled files so task can complete
                        async with self._get_task_lock(task_id):
                            upload_task.processed_files += 1
                        return

                    file_task.status = TaskStatus.RUNNING
                    file_task.updated_at = time.time()

                    logger.info(
                        "File processing task running",
                        task_number=upload_task.sequence_number,
                        task_id=task_id,
                        file_path=file_task.file_path,
                    )

                    try:
                        # Add timeout protection to prevent indefinite hangs
                        # Traces opened for this file (and its model calls)
                        # carry the task; an upload's files share a session.
                        with trace_context(
                            session_id=task_id, attributes={"openrag.task_id": task_id}
                        ):
                            await self._process_with_timeout(
                                processor.process_item(upload_task, item, file_task),
                                timeout_seconds=self.ingestion_timeout,
                            )

                        logger.info(
                            "File processing task succeeded",
                            status="PASSED",
                            task_number=upload_task.sequence_number,
                            task_id=task_id,
                            file_path=file_task.file_path,
                        )

                    except asyncio.CancelledError:
                        # Handle cancellation explicitly

                        if file_task.status == TaskStatus.RUNNING:
                            file_task.status = TaskStatus.FAILED
                            file_task.error = "File processing task cancelled."
                            async with self._get_task_lock(task_id):
                                upload_task.failed_files += 1

                        logger.warning(
                            "File processing task cancelled",
                            status="FAILED",
                            task_number=upload_task.sequence_number,
                            task_id=task_id,
                            file_path=file_task.file_path,
                        )

                        raise  # Re-raise to propagate cancellation
                    except IngestionTimeoutError as e:
                        # Handle timeout explicitly
                        if file_task.status == TaskStatus.RUNNING:
                            file_task.status = TaskStatus.FAILED
                            file_task.error = str(e)
                            async with self._get_task_lock(task_id):
                                upload_task.failed_files += 1
                        # Don't re-raise - treat as normal failure, not cancellation

                        logger.error(
                            "File processing task timed out",
                            status="FAILED",
                            task_number=upload_task.sequence_number,
                            task_id=task_id,
                            file_path=file_task.file_path,
                            exception=str(e),
                        )

                    except Exception as e:
                        # Note: Processors already handle incrementing failed_files and
                        # setting file_task status/error, so we don't duplicate that here.
                        # Only update timestamp if processor didn't already set it
                        if file_task.status == TaskStatus.RUNNING:
                            file_task.status = TaskStatus.FAILED
                        # Don't overwrite cancellation errors - preserve user cancellation message
                        if file_task.error != "File cancelled by user":
                            if not file_task.error:
                                file_task.error = str(e) or repr(e)

                        logger.error(
                            "File processing task exception encountered",
                            status="FAILED",
                            traceback=traceback.format_exc(),
                            task_number=upload_task.sequence_number,
                            task_id=task_id,
                            file_path=file_task.file_path,
                            exception=str(e),
                        )

                    finally:
                        # Hand back the filename this file held while in flight
                        # (utils.filename_claims), so the next task — or a retry
                        # of this one — can take it. Every file passes through
                        # here, including the ones that never claimed anything.
                        refused = filename_claims.release(claim_holder(task_id, item_key))
                        if refused and file_task.status == TaskStatus.FAILED:
                            await self._fail_files_stranded_by(refused)
                        file_task.updated_at = time.time()
                        # Only increment processed_files if the file reached a terminal state
                        # This prevents counter inconsistency on cancellation.
                        # SKIPPED is terminal too — e.g. source-deleted files in
                        # connector sync — and must count, or the upload task
                        # never reaches processed_files >= total_files and stays
                        # open forever.
                        if file_task.status in [
                            TaskStatus.COMPLETED,
                            TaskStatus.FAILED,
                            TaskStatus.SKIPPED,
                        ]:
                            async with self._get_task_lock(task_id):
                                upload_task.processed_files += 1
                        upload_task.updated_at = time.time()

            # Create tasks and track them for file-level cancellation
            tasks = []
            for item in items:
                item_key = str(item)
                task = asyncio.create_task(process_with_semaphore(item, item_key))
                tasks.append(task)
                # Track file task for cancellation (will be cleaned up in finally block)
                file_task = upload_task.file_tasks.get(item_key)
                if file_task:
                    self._file_tasks[(task_id, file_task.file_path)] = task

            try:
                await asyncio.gather(*tasks, return_exceptions=True)
            finally:
                # Clean up file task references even if parent task is cancelled
                for item in items:
                    item_key = str(item)
                    file_task = upload_task.file_tasks.get(item_key)
                    if file_task:
                        self._file_tasks.pop((task_id, file_task.file_path), None)

            # Mark task as completed if all files (including appended ones) are done
            if upload_task.processed_files >= upload_task.total_files:
                # Force an index refresh BEFORE marking the task COMPLETED so that
                # callers polling for COMPLETED status can immediately query or delete
                # the newly indexed chunks without hitting the near-real-time refresh window.
                if upload_task.successful_files > 0:
                    try:
                        from config.settings import clients, get_index_name

                        await clients.opensearch.indices.refresh(index=get_index_name())
                    except Exception as e:
                        logger.debug("Index refresh after ingest failed (non-fatal)", error=str(e))

                upload_task.status = TaskStatus.COMPLETED
                upload_task.updated_at = time.time()

            # Clean up upload temps that are not retryable; keep local RETRYABLE
            # failures so retry can re-read the staged source file.
            self._cleanup_upload_temp_files(upload_task)

            status: str = "FAILED"

            if upload_task.failed_files == 0:
                status = "PASSED"
            elif upload_task.successful_files > 0:
                status = "FAILED (partial success)"

            logger.info(
                "Upload / ingestion task finished",
                status=status,
                task_number=upload_task.sequence_number,
                task_id=task_id,
                duration=self._format_duration(upload_task.duration_seconds),
                total_files=upload_task.total_files,
                processed_files=upload_task.processed_files,
                successful_files=upload_task.successful_files,
                failed_files=upload_task.failed_files,
                filenames=self._get_display_filenames(upload_task),
                processor_type=processor.__class__.__name__,
                user_id=user_id,
                worker_count=self._worker_count,
            )

            # Send telemetry for task completion
            asyncio.create_task(
                TelemetryClient.send_event(
                    Category.TASK_OPERATIONS,
                    MessageId.ORB_TASK_COMPLETE,
                    metadata={
                        "total_files": upload_task.total_files,
                        "successful_files": upload_task.successful_files,
                        "failed_files": upload_task.failed_files,
                    },
                )
            )

        except asyncio.CancelledError:
            if user_id in self.task_store and task_id in self.task_store[user_id]:
                # Task status and pending files already handled by cancel_task()
                upload_task = self.task_store[user_id][task_id]

                logger.warning(
                    "Upload / ingestion task cancelled",
                    status="FAILED",
                    task_number=upload_task.sequence_number,
                    task_id=task_id,
                    duration=self._format_duration(upload_task.duration_seconds),
                    total_files=upload_task.total_files,
                    processed_files=upload_task.processed_files,
                    successful_files=upload_task.successful_files,
                    failed_files=upload_task.failed_files,
                    filenames=self._get_display_filenames(upload_task),
                    processor_type=upload_task.processor.__class__.__name__,
                    user_id=user_id,
                    worker_count=self._worker_count,
                )
            else:
                logger.warning(
                    "Upload / ingestion task cancelled",
                    status="FAILED",
                    task_id=task_id,
                    user_id=user_id,
                    worker_count=self._worker_count,
                )

            raise  # Re-raise to properly handle cancellation
        except Exception as e:
            if user_id in self.task_store and task_id in self.task_store[user_id]:
                upload_task = self.task_store[user_id][task_id]
                upload_task.status = TaskStatus.FAILED
                upload_task.updated_at = time.time()

                logger.error(
                    "Upload / ingestion task exception encountered",
                    status="FAILED",
                    task_number=upload_task.sequence_number,
                    task_id=task_id,
                    duration=self._format_duration(upload_task.duration_seconds),
                    total_files=upload_task.total_files,
                    processed_files=upload_task.processed_files,
                    successful_files=upload_task.successful_files,
                    failed_files=upload_task.failed_files,
                    filenames=self._get_display_filenames(upload_task),
                    processor_type=upload_task.processor.__class__.__name__,
                    user_id=user_id,
                    worker_count=self._worker_count,
                    exception=str(e),
                )

                # Send telemetry for task failure
                asyncio.create_task(
                    TelemetryClient.send_event(
                        Category.TASK_OPERATIONS,
                        MessageId.ORB_TASK_FAILED,
                        metadata={
                            "total_files": upload_task.total_files,
                            "processed_files": upload_task.processed_files,
                            "successful_files": upload_task.successful_files,
                            "failed_files": upload_task.failed_files,
                        },
                    )
                )
            else:
                logger.error(
                    "Upload / ingestion exception encountered",
                    status="FAILED",
                    task_id=task_id,
                    user_id=user_id,
                    worker_count=self._worker_count,
                    exception=str(e),
                )

    def _resolve_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:
        """Look up a task by ID, falling back to anonymous/shared tasks."""
        if not task_id:
            return None
        for candidate_user_id in [user_id, AnonymousUser().user_id]:
            if (
                candidate_user_id in self.task_store
                and task_id in self.task_store[candidate_user_id]
            ):
                return self.task_store[candidate_user_id][task_id]
        return None

    def get_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:
        """Public accessor for upload task lookup."""
        return self._resolve_upload_task(user_id, task_id)

    def _resolve_upload_task_store(
        self, user_id: str, task_id: str
    ) -> tuple[str, UploadTask] | None:
        """Return (store_user_id, upload_task) for a task visible to this user."""
        if not task_id:
            return None
        for candidate_user_id in [user_id, AnonymousUser().user_id]:
            if (
                candidate_user_id in self.task_store
                and task_id in self.task_store[candidate_user_id]
            ):
                return candidate_user_id, self.task_store[candidate_user_id][task_id]
        return None

    async def retry_failed_files(
        self,
        user_id: str,
        task_id: str,
        *,
        file_paths: list[str] | None = None,
        retryable_only: bool = True,
    ) -> dict | None:
        """Re-queue failed files for ingestion when their source paths still exist.

        Only files classified as RETRYABLE are retried when *retryable_only* is
        True (the default). When *file_paths* is set, only those task paths are
        considered; paths missing from the task or not in a failed state are
        reported in *skipped*. This reuses the task's original processor — it
        does not accept new uploads from the client.

        Connector tasks often use provider IDs (not local absolute paths) as
        ``file_path`` keys. For those non-absolute identifiers, skip local
        filesystem existence checks and let the connector processor re-fetch.
        """
        resolved = self._resolve_upload_task_store(user_id, task_id)
        if resolved is None:
            return None

        store_user_id, upload_task = resolved

        paths_to_retry: list[str] = []
        skipped: list[dict] = []
        requested_paths = set(file_paths) if file_paths is not None else None

        # Keep status checks, candidate selection, and state transitions inside
        # one lock to avoid concurrent retry requests enqueueing duplicates.
        async with self._get_task_lock(task_id):
            processor = upload_task.processor
            if upload_task.status == TaskStatus.RUNNING:
                return {
                    "error": "task_in_progress",
                    "message": "Task is still running",
                    "task_id": task_id,
                }

            if processor is None:
                return {
                    "error": "no_processor",
                    "message": "Cannot retry: task processor is no longer available",
                    "task_id": task_id,
                }

            if requested_paths is not None:
                for path in requested_paths:
                    file_task = upload_task.file_tasks.get(path)
                    if file_task is None:
                        skipped.append({"file_path": path, "reason": "file_not_in_task"})
                    elif file_task.status != TaskStatus.FAILED:
                        skipped.append(
                            {
                                "file_path": path,
                                "filename": file_task.filename,
                                "reason": "not_failed",
                            }
                        )

            # Build retry candidates before mutating shared task/file state.
            retry_candidates: list[tuple[str, FileTask]] = []
            for file_path, file_task in list(upload_task.file_tasks.items()):
                if requested_paths is not None and file_path not in requested_paths:
                    continue
                if file_task.status != TaskStatus.FAILED:
                    continue

                if retryable_only:
                    metadata = self._infer_failure_metadata(file_task)
                    if not metadata or metadata.get("actionable_by") != "RETRYABLE":
                        skipped.append(
                            {
                                "file_path": file_path,
                                "filename": file_task.filename,
                                "reason": "not_retryable",
                            }
                        )
                        continue

                # Only enforce local source-file existence for absolute paths.
                # Connector-backed tasks typically store remote IDs as file_path.
                if os.path.isabs(file_path) and not os.path.isfile(file_path):
                    skipped.append(
                        {
                            "file_path": file_path,
                            "filename": file_task.filename,
                            "reason": "source_file_missing",
                        }
                    )
                    continue

                retry_candidates.append((file_path, file_task))

            now = time.time()
            for file_path, file_task in retry_candidates:
                if upload_task.failed_files > 0:
                    upload_task.failed_files -= 1
                if upload_task.processed_files > 0:
                    upload_task.processed_files -= 1

                file_task.status = TaskStatus.PENDING
                file_task.error = None
                file_task.result = None
                file_task.retry_count += 1
                file_task.docling_task_id = None
                # Connector tasks use remote IDs as file_path and ingest via Langflow.
                if os.path.isabs(file_path):
                    file_task.docling_status = DoclingPhaseStatus.PENDING
                    file_task.phase = IngestionPhase.DOCLING
                else:
                    file_task.docling_status = DoclingPhaseStatus.PENDING
                    file_task.phase = IngestionPhase.LANGFLOW
                file_task.updated_at = now
                paths_to_retry.append(file_path)

            if paths_to_retry:
                upload_task.status = TaskStatus.RUNNING
                upload_task.updated_at = now

        if not paths_to_retry:
            return {
                "task_id": task_id,
                "retried": 0,
                "skipped": skipped,
                "status": "no_op",
                "message": "No retryable files with available source data",
            }

        background_task = asyncio.create_task(
            self.background_custom_processor(store_user_id, task_id, paths_to_retry, processor)
        )
        upload_task.background_task = background_task
        self.background_tasks.add(background_task)

        def _clear_retry_background_task(done_task: asyncio.Task) -> None:
            self.background_tasks.discard(done_task)
            if upload_task.background_task is done_task:
                upload_task.background_task = None

        background_task.add_done_callback(_clear_retry_background_task)

        return {
            "task_id": task_id,
            "retried": len(paths_to_retry),
            "skipped": skipped,
            "status": "accepted",
        }

    @staticmethod
    def _keep_completed_file_in_list(file_task: FileTask) -> bool:
        """Source-deleted cleanups stay visible so the UI can show Removed.

        Normal completed ingests are omitted from the task file list to keep it
        tidy. A file that was removed from the index because it disappeared at
        the source is also completed, but dropping it would hide the cleanup.
        """
        result = file_task.result or {}
        return result.get("reason") == "deleted_at_source"

    def _serialize_file_task(self, file_task: FileTask) -> dict:
        """Serialize a FileTask to the standard dict shape."""
        return {
            "status": file_task.status.value,
            "result": file_task.result,
            "error": file_task.error,
            "retry_count": file_task.retry_count,
            "created_at": file_task.created_at,
            "updated_at": file_task.updated_at,
            "duration_seconds": file_task.duration_seconds,
            "filename": file_task.filename,
            "phase": file_task.phase.value,
            "docling_status": file_task.docling_status.value,
            "docling_task_id": file_task.docling_task_id,
        }

    def _infer_failure_metadata(self, file_task: FileTask) -> dict | None:
        """Infer structured failure metadata for a failed FileTask.

        Returns a dict with component, failure_phase, user_facing_message, and
        actionable_by when the failure can be classified, or None when the cause
        is unknown and no fields should be emitted.

        Priority order: transient / retryable docling outcomes (expired, polling
        timeout) before generic docling FAILED, which indicates conversion failure.
        """
        docling_status = file_task.docling_status
        phase = file_task.phase
        error = file_task.error or ""

        if _is_task_cancellation_error(error):
            return {
                "failure_phase": "cancelled",
                "user_facing_message": "Ingestion was cancelled.",
                "actionable_by": "USER_ACTIONABLE",
            }

        from utils.upload_limits import is_upload_size_error, upload_size_user_message

        # A size refusal is stored while docling_status is still PENDING, because
        # the proxy rejects the body before a Docling task exists. Later branches
        # would drop that text.
        if is_upload_size_error(error):
            return {
                "component": "docling",
                "failure_phase": "parsing",
                "user_facing_message": upload_size_user_message(error),
                "actionable_by": "USER_ACTIONABLE",
            }

        # Before the substring heuristics below, which would otherwise read this
        # as a plain duplicate: the file was never ingested, and retrying is
        # exactly the right move.
        if error == INFLIGHT_CLAIM_WINNER_FAILED_ERROR:
            return {
                "component": "openrag",
                "failure_phase": "unknown",
                "user_facing_message": (
                    f"{INFLIGHT_CLAIM_WINNER_FAILED_ERROR} Retry to ingest it."
                ),
                "actionable_by": "RETRYABLE",
            }

        # Before any substring heuristic: an OpenSearch transport failure carries a
        # JSON payload whose contents would otherwise match the file-corruption and
        # "already exists" markers below.
        opensearch_meta = _opensearch_failure_metadata(error)
        if opensearch_meta:
            return opensearch_meta

        # Password-protected PDFs. Newer docling no longer names the reason, so the
        # backend recovers it from the file bytes (docling_service.password_protected_pdf_error).
        if "incorrect password" in error.lower():
            return {
                "component": "docling",
                "failure_phase": "parsing",
                "user_facing_message": "This PDF is password-protected. Password-protected PDFs are not supported. Remove the password and upload the PDF again.",
                "actionable_by": "USER_ACTIONABLE",
            }

        if docling_status == DoclingPhaseStatus.EXPIRED:
            return {
                "component": "docling",
                "failure_phase": "parsing",
                "user_facing_message": (
                    "The document processing result could not be found. "
                    "The task may have expired. Please retry ingestion."
                ),
                "actionable_by": "RETRYABLE",
            }

        # First, check if the error is non-retryable based on common markers
        if _is_non_retryable_file_error(error):
            # Image files that produced no text almost certainly failed because OCR
            # is disabled — not because the file is corrupted. Give a targeted tip.
            filename = file_task.filename or file_task.file_path or ""
            if "no text content could be extracted" in error.lower() and _is_ocr_required_file(
                filename
            ):
                return {
                    "component": "docling",
                    "failure_phase": "parsing",
                    "user_facing_message": (
                        "No text content could be extracted from this image file. "
                        "Enable OCR in Settings > Knowledge Base and retry ingestion."
                    ),
                    "actionable_by": "RETRYABLE",
                }

            if phase == IngestionPhase.LANGFLOW:
                component = "langflow"
                failure_phase = "unknown"
            else:
                component = "docling"
                failure_phase = "parsing"

            # Extract detailed error message if possible, otherwise default to corrupted/invalid msg
            msg = (
                "The file appears corrupted or invalid and cannot be processed. "
                "Upload a valid file."
            )
            if "Docling result unavailable after SUCCESS status: " in error:
                msg = error.split("Docling result unavailable after SUCCESS status: ", 1)[1]
            elif "Docling conversion did not complete" in error:
                sub_msg = error.split("Docling conversion did not complete", 1)[1]
                if sub_msg.startswith(" (failed): "):
                    msg = sub_msg[len(" (failed): ") :]
                else:
                    msg = sub_msg.strip(" ():")

            return {
                "component": component,
                "failure_phase": failure_phase,
                "user_facing_message": msg,
                "actionable_by": "USER_ACTIONABLE",
            }

        # Handle docling conversion completion issues (failed / timeout)
        if phase == IngestionPhase.DOCLING and "Docling conversion did not complete" in error:
            user_facing_message = "Document processing timed out. Please retry ingestion."
            if "timeout" not in error.lower() and "expired" not in error.lower():
                msg = error.split("Docling conversion did not complete", 1)[1]
                if msg.startswith(" (failed): "):
                    user_facing_message = msg[len(" (failed): ") :]
                elif msg.startswith(" (timeout): "):
                    user_facing_message = "Document processing timed out. Please retry ingestion."
                else:
                    user_facing_message = msg.strip(" ():")
            return {
                "component": "docling",
                "failure_phase": "parsing",
                "user_facing_message": user_facing_message,
                "actionable_by": "RETRYABLE"
                if "timed out" in user_facing_message.lower()
                else "USER_ACTIONABLE",
            }

        if phase == IngestionPhase.DOCLING and _is_docling_transient_error(error):
            langflow_transport = _is_langflow_transport_failure(error)
            return {
                "component": "langflow" if langflow_transport else "docling",
                "failure_phase": "unknown" if langflow_transport else "parsing",
                "user_facing_message": _transient_connectivity_user_message(error),
                "actionable_by": "RETRYABLE",
            }

        if phase == IngestionPhase.DOCLING and docling_status == DoclingPhaseStatus.PROCESSING:
            # Check if this was actually a cancellation before assuming timeout
            if _is_task_cancellation_error(error):
                return {
                    "failure_phase": "cancelled",
                    "user_facing_message": "Ingestion was cancelled.",
                    "actionable_by": "USER_ACTIONABLE",
                }
            return {
                "component": "docling",
                "failure_phase": "parsing",
                "user_facing_message": "Document processing timed out. Please retry ingestion.",
                "actionable_by": "RETRYABLE",
            }

        if docling_status == DoclingPhaseStatus.FAILED:
            msg = "The file could not be processed into readable document content."
            if error:
                if "Docling result unavailable after SUCCESS status: " in error:
                    msg = error.split("Docling result unavailable after SUCCESS status: ", 1)[1]
                elif "Docling conversion did not complete" in error:
                    sub_msg = error.split("Docling conversion did not complete", 1)[1]
                    if sub_msg.startswith(" (failed): "):
                        msg = sub_msg[len(" (failed): ") :]
                    else:
                        msg = sub_msg.strip(" ():")
                else:
                    msg = error
            return {
                "component": "docling",
                "failure_phase": "parsing",
                "user_facing_message": msg,
                "actionable_by": "USER_ACTIONABLE",
            }

        if "already exists" in error:
            return {
                "component": "openrag",
                "failure_phase": "file_validation",
                "user_facing_message": "A file with this name already exists.",
                "actionable_by": "USER_ACTIONABLE",
            }

        # Prefer credential failures over transport messages. Langflow may report
        # "server disconnected" when the real cause is a revoked embedding API key;
        # resolve_ingest_error_message() rewrites file_task.error in that case, and
        # this check also covers errors that mention both disconnect + API key text.
        credential_meta = _provider_credential_failure_metadata(error)
        if credential_meta:
            return credential_meta

        # Before the Langflow fallbacks, which would report the provider's raw
        # sentence as a retryable failure: retrying the same chunk fails again.
        context_window_meta = _context_window_failure_metadata(error)
        if context_window_meta:
            return context_window_meta

        if _is_transient_connectivity_error(error) or _is_langflow_transport_failure(error):
            return {
                "component": "langflow",
                "failure_phase": "unknown",
                "user_facing_message": _transient_connectivity_user_message(error),
                "actionable_by": "RETRYABLE",
            }

        if phase == IngestionPhase.LANGFLOW:
            error_lower = error.lower()
            if any(
                marker in error_lower
                for marker in ("timeout", "timed out", "unavailable", "connection refused")
            ):
                return {
                    "component": "langflow",
                    "failure_phase": "unknown",
                    "user_facing_message": (
                        "Ingestion timed out or the service was unavailable. Please retry."
                    ),
                    "actionable_by": "RETRYABLE",
                }

            # Prefer a sanitized specific provider/Langflow message (model missing,
            # project misconfig, etc.) over a generic "unexpectedly" toast.
            from api.provider_validation import (
                is_generic_upstream_error,
                is_provider_credential_error,
                sanitize_provider_error_content,
            )

            cleaned = sanitize_provider_error_content(error)
            if (
                cleaned
                and not is_generic_upstream_error(cleaned)
                and "{" not in cleaned
                and "}" not in cleaned
            ):
                lowered = cleaned.lower()
                missing_deployment = "deployment" in lowered and any(
                    marker in lowered for marker in ("not found", "does not exist")
                )
                user_actionable = (
                    is_provider_credential_error(cleaned)
                    or missing_deployment
                    or any(
                        marker in lowered
                        for marker in (
                            "model",
                            "project",
                            "not found",
                            "not properly configured",
                            "no models",
                            "unauthorized",
                            "forbidden",
                            "permission",
                            "quota",
                            "rate limit",
                        )
                    )
                )
                return {
                    "component": "langflow",
                    "failure_phase": "unknown",
                    "user_facing_message": cleaned,
                    "actionable_by": ("USER_ACTIONABLE" if user_actionable else "RETRYABLE"),
                }

            return {
                "component": "langflow",
                "failure_phase": "unknown",
                "user_facing_message": (
                    "Ingestion failed unexpectedly. Please retry. "
                    "If it fails again, contact your administrator."
                ),
                "actionable_by": "RETRYABLE",
            }

        return None

    def get_task_status(self, user_id: str, task_id: str) -> dict | None:
        """Get the status of a specific upload task

        Includes fallback to shared tasks stored under the "anonymous" user key
        so default system tasks are visible to all users.
        """
        upload_task = self._resolve_upload_task(user_id, task_id)

        if upload_task is None:
            return None

        file_statuses = {}
        running_files_count = 0
        pending_files_count = 0

        for file_path, file_task in upload_task.file_tasks.items():
            file_statuses[file_path] = self._serialize_file_task(file_task)

            # Count running and pending files
            if file_task.status.value == "running":
                running_files_count += 1
            elif file_task.status.value == "pending":
                pending_files_count += 1

        return {
            "task_id": upload_task.task_id,
            "status": upload_task.status.value,
            "total_files": upload_task.total_files,
            "processed_files": upload_task.processed_files,
            "successful_files": upload_task.successful_files,
            "failed_files": upload_task.failed_files,
            "running_files": running_files_count,
            "pending_files": pending_files_count,
            "created_at": upload_task.created_at,
            "updated_at": upload_task.updated_at,
            "duration_seconds": upload_task.duration_seconds,
            "files": file_statuses,
        }

    def get_task_status2(self, user_id: str, task_id: str) -> dict | None:
        """Get the status of a specific upload task with structured failure metadata.

        Identical to get_task_status but enriches failed file entries with
        component, failure_phase, user_facing_message, and actionable_by fields
        when the failure cause can be classified.
        """
        upload_task = self._resolve_upload_task(user_id, task_id)

        if upload_task is None:
            return None

        file_statuses = {}
        running_files_count = 0
        pending_files_count = 0

        for file_path, file_task in upload_task.file_tasks.items():
            entry = self._serialize_file_task(file_task)
            if file_task.status == TaskStatus.FAILED:
                metadata = self._infer_failure_metadata(file_task)
                if metadata:
                    entry.update(metadata)
            file_statuses[file_path] = entry

            if file_task.status == TaskStatus.RUNNING:
                running_files_count += 1
            elif file_task.status == TaskStatus.PENDING:
                pending_files_count += 1

        return {
            "task_id": upload_task.task_id,
            "status": upload_task.status.value,
            "total_files": upload_task.total_files,
            "processed_files": upload_task.processed_files,
            "successful_files": upload_task.successful_files,
            "failed_files": upload_task.failed_files,
            "running_files": running_files_count,
            "pending_files": pending_files_count,
            "created_at": upload_task.created_at,
            "updated_at": upload_task.updated_at,
            "duration_seconds": upload_task.duration_seconds,
            "files": file_statuses,
        }

    def get_all_tasks2(self, user_id: str) -> list:
        """Get all tasks for a user with structured failure metadata on failed files.

        Identical to get_all_tasks but enriches failed file entries with
        component, failure_phase, user_facing_message, and actionable_by fields
        when the failure cause can be classified.
        """
        tasks_by_id = {}

        is_shared = False

        def add_tasks_from_store(store_user_id):
            if store_user_id not in self.task_store:
                return
            for task_id, upload_task in self.task_store[store_user_id].items():
                if task_id in tasks_by_id:
                    continue

                running_files_count = 0
                pending_files_count = 0
                file_statuses = {}

                for file_path, file_task in upload_task.file_tasks.items():
                    # The task panel only surfaces in-flight/failed files to keep
                    # the list tidy. Preview-mode tasks instead need every file
                    # (including completed ones) so the live preview carousel can
                    # still enumerate them and render their cached Docling layout.
                    # Source-deleted cleanups are completed but must stay visible
                    # as successful removals, not vanish like a normal ingest.
                    if (
                        file_task.status != TaskStatus.COMPLETED
                        or upload_task.preview_mode
                        or self._keep_completed_file_in_list(file_task)
                    ):
                        entry = self._serialize_file_task(file_task)
                        if file_task.status == TaskStatus.FAILED:
                            metadata = self._infer_failure_metadata(file_task)
                            if metadata:
                                entry.update(metadata)
                        file_statuses[file_path] = entry

                    if file_task.status == TaskStatus.RUNNING:
                        running_files_count += 1
                    elif file_task.status == TaskStatus.PENDING:
                        pending_files_count += 1

                tasks_by_id[task_id] = {
                    "task_id": upload_task.task_id,
                    "status": upload_task.status.value,
                    "total_files": upload_task.total_files,
                    "processed_files": upload_task.processed_files,
                    "successful_files": upload_task.successful_files,
                    "failed_files": upload_task.failed_files,
                    "running_files": running_files_count,
                    "pending_files": pending_files_count,
                    "created_at": upload_task.created_at,
                    "updated_at": upload_task.updated_at,
                    "duration_seconds": upload_task.duration_seconds,
                    "files": file_statuses,
                    "is_shared": is_shared,
                }

        add_tasks_from_store(user_id)
        is_shared = True
        add_tasks_from_store(AnonymousUser().user_id)

        tasks = list(tasks_by_id.values())
        tasks.sort(key=lambda x: x["created_at"], reverse=True)
        return tasks

    def get_all_tasks(self, user_id: str) -> list:
        """Get all tasks for a user

        Returns the union of the user's own tasks and shared default tasks stored
        under the "anonymous" user key. User-owned tasks take precedence
        if a task_id overlaps.
        """
        tasks_by_id = {}

        is_shared = False

        def add_tasks_from_store(store_user_id):
            if store_user_id not in self.task_store:
                return
            for task_id, upload_task in self.task_store[store_user_id].items():
                if task_id in tasks_by_id:
                    continue

                # Calculate running and pending counts and build file statuses
                running_files_count = 0
                pending_files_count = 0
                file_statuses = {}

                for file_path, file_task in upload_task.file_tasks.items():
                    if file_task.status.value != "completed" or self._keep_completed_file_in_list(
                        file_task
                    ):
                        file_statuses[file_path] = {
                            "status": file_task.status.value,
                            "result": file_task.result,
                            "error": file_task.error,
                            "retry_count": file_task.retry_count,
                            "created_at": file_task.created_at,
                            "updated_at": file_task.updated_at,
                            "duration_seconds": file_task.duration_seconds,
                            "filename": file_task.filename,
                            "phase": file_task.phase.value,
                            "docling_status": file_task.docling_status.value,
                            "docling_task_id": file_task.docling_task_id,
                        }

                    if file_task.status.value == "running":
                        running_files_count += 1
                    elif file_task.status.value == "pending":
                        pending_files_count += 1

                tasks_by_id[task_id] = {
                    "task_id": upload_task.task_id,
                    "status": upload_task.status.value,
                    "total_files": upload_task.total_files,
                    "processed_files": upload_task.processed_files,
                    "successful_files": upload_task.successful_files,
                    "failed_files": upload_task.failed_files,
                    "running_files": running_files_count,
                    "pending_files": pending_files_count,
                    "created_at": upload_task.created_at,
                    "updated_at": upload_task.updated_at,
                    "duration_seconds": upload_task.duration_seconds,
                    "files": file_statuses,
                    "is_shared": is_shared,
                }

        # First, add user-owned tasks; then shared anonymous;
        add_tasks_from_store(user_id)
        is_shared = True
        add_tasks_from_store(AnonymousUser().user_id)

        tasks = list(tasks_by_id.values())
        tasks.sort(key=lambda x: x["created_at"], reverse=True)
        return tasks

    async def cleanup_old_tasks(self, max_age_seconds: int = 3600) -> int:
        """Remove completed/failed tasks older than max_age_seconds

        Args:
            max_age_seconds: Maximum age in seconds for completed tasks (default: 1 hour)

        Returns:
            Number of tasks cleaned up
        """
        current_time = time.time()
        cleaned_count = 0

        # Complexity Analysis:
        # O(n) where n = total tasks across all users

        for user_id in list(self.task_store.keys()):
            for task_id in list(self.task_store[user_id].keys()):
                task = self.task_store[user_id][task_id]
                # Only cleanup completed, failed, or cancelled tasks that are old enough
                if (
                    task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED]
                    and current_time - task.updated_at > max_age_seconds
                ):
                    # Task is leaving memory; reclaim any retained upload temps
                    # (including RETRYABLE locals kept for in-flight retry).
                    self._cleanup_upload_temp_files(task, force=True)
                    del self.task_store[user_id][task_id]
                    # Clean up the associated lock
                    self._task_locks.pop(task_id, None)
                    cleaned_count += 1
                    logger.debug(
                        "Cleaned up old task",
                        task_id=task_id,
                        user_id=user_id,
                        age_seconds=current_time - task.updated_at,
                    )

            # Remove empty user entries
            if not self.task_store[user_id]:
                del self.task_store[user_id]

        if cleaned_count > 0:
            logger.info("Task cleanup completed", cleaned_count=cleaned_count)

        return cleaned_count

    async def cancel_task(self, user_id: str, task_id: str) -> bool:
        """Cancel a task if it exists and is not already completed.

        Supports cancellation of shared default tasks stored under the anonymous user.
        The outer task lock is held for the entire operation so that concurrent
        cancel requests (e.g. rapid UI clicks) are serialised: only the first
        caller performs the actual cancellation, subsequent ones see a terminal
        status and return False immediately.
        """
        # Resolve which user bucket owns the task before acquiring the lock.
        candidate_user_ids = [user_id, AnonymousUser().user_id]

        store_user_id = None
        for candidate_user_id in candidate_user_ids:
            if (
                candidate_user_id in self.task_store
                and task_id in self.task_store[candidate_user_id]
            ):
                store_user_id = candidate_user_id
                break

        if store_user_id is None:
            return False

        upload_task = self.task_store[store_user_id][task_id]

        # Phase 1: idempotency check — hold the lock only long enough to read
        # the current status and signal cancellation.  We must NOT hold the lock
        # while awaiting the background task: the background task's CancelledError
        # handler also acquires _get_task_lock (to increment failed_files), which
        # would deadlock with cancel_task holding that same lock.
        async with self._get_task_lock(task_id):
            # Can only cancel pending or running tasks.
            if upload_task.status in [
                TaskStatus.COMPLETED,
                TaskStatus.FAILED,
                TaskStatus.CANCELLED,
            ]:
                return False
            # Mark in-flight immediately so a concurrent cancel sees CANCELLED
            # and returns False without repeating the work below.
            upload_task.status = TaskStatus.CANCELLED

        # Phase 2: cancel and await the background task OUTSIDE the lock so the
        # background task's own lock acquisitions (failed_files increment, etc.)
        # can complete without deadlocking.
        if hasattr(upload_task, "background_task") and not upload_task.background_task.done():
            upload_task.background_task.cancel()
            try:
                await upload_task.background_task
            except asyncio.CancelledError:
                pass  # Expected when we cancel the task
            except Exception:
                pass  # Ignore other errors during cancellation

        # Phase 3: finalise state — background task is done, so no concurrent
        # writer can touch the file tasks any more.  Lock is still used for
        # safety in case a second cancel slipped through between phases.
        async with self._get_task_lock(task_id):
            upload_task.updated_at = time.time()
            now = time.time()
            for file_task in upload_task.file_tasks.values():
                if file_task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:
                    # PENDING files cancelled before entering process_with_semaphore
                    # need processed_files incremented here, since the finally block
                    # won't run. RUNNING files normally cancelled are already counted
                    # by the finally block, but incrementing here is safe (phase 3 runs
                    # after background task completes, so no double-counting).
                    upload_task.processed_files += 1
                    upload_task.failed_files += 1
                    file_task.status = TaskStatus.FAILED
                    file_task.error = "Task cancelled by user"
                    file_task.updated_at = now

        self._cleanup_upload_temp_files(upload_task, force=True)

        return True

    async def cancel_file(self, user_id: str, task_id: str, file_path: str) -> bool:
        """Cancel a single file within a task.

        Marks the file as FAILED with "cancelled by user" error. The file will be
        skipped if it's still PENDING, or marked as cancelled if already RUNNING.
        """
        # Check candidate user IDs first, then anonymous to find which user ID the task is mapped to
        candidate_user_ids = [user_id, AnonymousUser().user_id]

        store_user_id = None
        for candidate_user_id in candidate_user_ids:
            if (
                candidate_user_id in self.task_store
                and task_id in self.task_store[candidate_user_id]
            ):
                store_user_id = candidate_user_id
                break

        if store_user_id is None:
            return False

        upload_task = self.task_store[store_user_id][task_id]

        # Find the file task
        file_task = upload_task.file_tasks.get(file_path)
        if not file_task:
            return False

        # Lock to prevent race conditions
        async with self._get_task_lock(task_id):
            # Can only cancel pending or running files
            if file_task.status not in [TaskStatus.PENDING, TaskStatus.RUNNING]:
                return False

            # Mark file as failed (cancelled)
            was_running = file_task.status == TaskStatus.RUNNING
            file_task.status = TaskStatus.FAILED
            file_task.error = "File cancelled by user"
            file_task.updated_at = time.time()

            # Increment failed counter
            upload_task.failed_files += 1

            # If it was running, it will be counted in processed_files by the worker's finally block
            # If it was pending, we need to count it now since it will never enter the worker
            if not was_running:
                upload_task.processed_files += 1

            upload_task.updated_at = time.time()

        # Cancel the running asyncio task if it exists (outside the lock to avoid deadlock)
        file_task_key = (task_id, file_path)
        if file_task_key in self._file_tasks:
            asyncio_task = self._file_tasks[file_task_key]
            if not asyncio_task.done():
                asyncio_task.cancel()
                logger.info(
                    "Cancelled asyncio task for file",
                    task_id=task_id,
                    file_path=file_path,
                )

        return True

    def delete_task(self, user_id: str, task_id: str) -> TaskDeleteResult:
        """Remove a terminal (completed/failed/cancelled) task from memory.

        Deletes tasks owned by the calling user or shared anonymous tasks that
        are visible to the user (same set exposed by get_all_tasks).

        Returns:
            TaskDeleteResult.DELETED      – task found and removed.
            TaskDeleteResult.NOT_FOUND    – task ID does not exist for this user.
            TaskDeleteResult.IN_PROGRESS  – task exists but is not yet terminal.
        """
        resolved = self._resolve_upload_task_store(user_id, task_id)
        if resolved is None:
            return TaskDeleteResult.NOT_FOUND
        store_user_id, upload_task = resolved
        if upload_task.status not in [
            TaskStatus.COMPLETED,
            TaskStatus.FAILED,
            TaskStatus.CANCELLED,
        ]:
            return TaskDeleteResult.IN_PROGRESS
        self._cleanup_upload_temp_files(upload_task, force=True)
        del self.task_store[store_user_id][task_id]
        self._task_locks.pop(task_id, None)
        if not self.task_store[store_user_id]:
            del self.task_store[store_user_id]
        return TaskDeleteResult.DELETED

    def delete_all_terminal_tasks(self, user_id: str) -> list[str]:
        """Remove all completed/failed/cancelled tasks owned by a user.

        Only touches the calling user's own store. Shared tasks stored under
        the anonymous key are intentionally excluded: they are visible to all
        authenticated users, so a bulk clear by one user must not remove them
        for everyone else. Those tasks are aged out by cleanup_old_tasks.

        Returns the list of deleted task IDs.
        """
        if user_id not in self.task_store:
            return []

        to_delete = [
            tid
            for tid, t in self.task_store[user_id].items()
            if t.status in [TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED]
        ]
        for tid in to_delete:
            task = self.task_store[user_id][tid]
            self._cleanup_upload_temp_files(task, force=True)
            del self.task_store[user_id][tid]
            self._task_locks.pop(tid, None)
        if not self.task_store.get(user_id):
            self.task_store.pop(user_id, None)

        return to_delete

    def _file_task_for_temp_path(self, upload_task: UploadTask, temp_path: str) -> FileTask | None:
        """Resolve the FileTask for a staged upload temp path."""
        file_task = upload_task.file_tasks.get(temp_path)
        if file_task is not None:
            return file_task
        for candidate in upload_task.file_tasks.values():
            if candidate.file_path == temp_path:
                return candidate
        return None

    def _is_retryable_local_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
        """True when a staged temp belongs to a failed local RETRYABLE upload.

        Local uploads use the staged path as the file_tasks key and
        FileTask.file_path. When those diverge, this falls back to matching
        FileTask.file_path. Unmapped absolute temps are retained (see
        _should_retain_upload_temp) rather than deleted silently.
        """
        if not os.path.isabs(temp_path):
            return False
        file_task = self._file_task_for_temp_path(upload_task, temp_path)
        if file_task is None or file_task.status != TaskStatus.FAILED:
            return False
        metadata = self._infer_failure_metadata(file_task)
        return bool(metadata and metadata.get("actionable_by") == "RETRYABLE")

    def _is_unsettled_claim_loser_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
        """True when a staged temp belongs to a file skipped for a name another
        in-flight file still holds.

        Such a skip is provisional: if the file holding the name fails, nothing
        was indexed under it and _fail_files_stranded_by turns this file into a
        retryable failure. That correction can land after this task has already
        finished — the holder may belong to another task entirely — and a retry
        then needs the staged source that cleanup would otherwise have deleted.

        Retention is bounded the same way a retryable failure's is: whichever
        way the holder ends, the temp is reclaimed when the task ages out of the
        store (cleanup_old_tasks force-cleans what it evicts).
        """
        if not os.path.isabs(temp_path):
            return False
        file_task = self._file_task_for_temp_path(upload_task, temp_path)
        if file_task is None or file_task.status != TaskStatus.SKIPPED:
            return False
        if (file_task.result or {}).get("reason") != "duplicate_filename":
            return False
        # Local uploads key file_tasks by the staged path, which is what the
        # claim holder was built from.
        return filename_claims.is_awaiting_outcome(
            claim_holder(upload_task.task_id, file_task.file_path)
        )

    def _should_retain_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
        """Return True when an upload temp should be kept after processing."""
        if self._is_retryable_local_upload_temp(upload_task, temp_path):
            return True
        if self._is_unsettled_claim_loser_temp(upload_task, temp_path):
            return True
        if (
            os.path.isabs(temp_path)
            and self._file_task_for_temp_path(upload_task, temp_path) is None
        ):
            logger.warning(
                "Upload temp path has no matching file task; retaining staged file",
                temp_path=temp_path,
                task_id=upload_task.task_id,
            )
            return True
        return False

    def _cleanup_upload_temp_files(self, upload_task: UploadTask, *, force: bool = False) -> None:
        """Remove staged upload temp files that are not retryable.

        Keeps temps for failed local uploads classified as RETRYABLE so retry
        can reuse the original source path. Use *force* to delete all temps
        (e.g. user cancelled the task).
        """
        if not getattr(upload_task, "temp_file_paths", None):
            return

        from utils.file_utils import safe_unlink

        retained: list[str] = []
        for temp_path in upload_task.temp_file_paths:
            if not force and self._should_retain_upload_temp(upload_task, temp_path):
                retained.append(temp_path)
                continue
            safe_unlink(temp_path)
            if os.path.exists(temp_path):
                retained.append(temp_path)
                logger.warning(
                    "Failed to clean up temp file after processing",
                    temp_path=temp_path,
                )
            else:
                logger.debug("Cleaned up temp file", temp_path=temp_path)
        upload_task.temp_file_paths = retained

    async def shutdown(self):
        """Cleanup process pool and cancel all background tasks

        Ensures graceful shutdown by:
        1. Cancelling the periodic cleanup task
        2. Cancelling all running background tasks
        3. Waiting for cancellation to complete
        4. Force-cleaning staged upload temps for all tracked tasks
        5. Shutting down the process pool
        """
        logger.info("Shutting down TaskService", background_tasks_count=len(self.background_tasks))

        # Cancel the periodic cleanup task
        if self._cleanup_task is not None and not self._cleanup_task.done():
            self._cleanup_task.cancel()
            try:
                await self._cleanup_task
            except asyncio.CancelledError:
                pass

        # Cancel all background tasks
        for task in self.background_tasks:
            if not task.done():
                task.cancel()

        # Wait for all tasks to complete cancellation
        if self.background_tasks:
            results = await asyncio.gather(*self.background_tasks, return_exceptions=True)
            # Log any unexpected errors (not CancelledError)
            for result in results:
                if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
                    logger.warning(
                        "Background task raised exception during shutdown", error=str(result)
                    )

        for user_tasks in self.task_store.values():
            for upload_task in user_tasks.values():
                self._cleanup_upload_temp_files(upload_task, force=True)