OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import asyncio
import os
import random
import re
import time
import traceback
import uuid
from collections.abc import Coroutine
from typing import Any, TypeVar
from models.tasks import (
DoclingPhaseStatus,
FileTask,
IngestionPhase,
TaskDeleteResult,
TaskStatus,
UploadTask,
)
from observability.tracing import trace_context
from session_manager import AnonymousUser
from utils.filename_claims import (
INFLIGHT_CLAIM_WINNER_FAILED_ERROR,
claim_holder,
filename_claims,
)
from utils.gpu_detection import get_worker_count
from utils.logging_config import get_logger
from utils.telemetry import Category, MessageId, TelemetryClient
T = TypeVar("T")
logger = get_logger(__name__)
# Substrings that indicate a permanent file/content problem (not worth retrying).
_NON_RETRYABLE_FILE_ERROR_MARKERS = (
"corrupt",
"corrupted",
"corruption",
"invalid file",
"invalid document",
"unsupported format",
"unsupported file",
"malformed",
"damaged",
"not a zip file",
"bad zipfile",
"failed to parse",
"could not be parsed",
"cannot parse",
"no text content could be extracted",
"empty or unreadable",
"unreadable",
"validationerror",
)
# Docling polling errors that are transient (service/timeout), not bad file content.
_DOCLING_TRANSIENT_ERROR_MARKERS = (
"(timeout)",
"timed out",
"polling exceeded",
"polling timed out",
)
# Connector/download/network failures seen while file is still in Docling phase.
# These are infrastructure/transient and should be retryable.
_TRANSIENT_CONNECTIVITY_ERROR_MARKERS = (
"all connection attempts failed",
"connection refused",
"connection reset",
"temporary failure in name resolution",
"name or service not known",
"service unavailable",
"network is unreachable",
"server disconnected",
"disconnected without sending a response",
"remote protocol error",
"broken pipe",
)
# Raster image formats that require OCR to produce any text content.
_OCR_REQUIRED_EXTENSIONS = frozenset({"bmp", "jpeg", "jpg", "png", "tiff", "webp"})
def _is_non_retryable_file_error(error: str) -> bool:
lowered = error.lower()
return any(marker in lowered for marker in _NON_RETRYABLE_FILE_ERROR_MARKERS)
def _is_ocr_required_file(filename: str) -> bool:
"""Return True if the file is a raster image that requires OCR to produce text."""
if not filename or "." not in filename:
return False
return filename.rsplit(".", 1)[-1].lower() in _OCR_REQUIRED_EXTENSIONS
_TASK_CANCELLATION_ERROR_MARKERS = (
"task cancelled by user",
"file cancelled by user",
"file processing task cancelled",
)
def _is_task_cancellation_error(error: str) -> bool:
lowered = error.lower()
return any(marker in lowered for marker in _TASK_CANCELLATION_ERROR_MARKERS)
# opensearch-py renders transport failures as "ClassName(<status>, ...)". For the
# by-query APIs (_update_by_query / _delete_by_query) the response carries no
# top-level "error" key, so Connection._raise_error falls back to using the whole
# JSON response body as the exception message — several hundred characters of
# payload where a reason string would normally be.
#
# Classifying these early matters twice over: an infrastructure fault never
# reaches the UI as a raw exception repr, and the JSON body cannot trip the
# substring heuristics further down (a mapper_parsing_exception body contains
# "failed to parse", which would otherwise read as a corrupted file).
#
# ConnectionError / ConnectionTimeout are deliberately excluded: they render with
# a different __str__ and are already covered by _is_transient_connectivity_error.
_OPENSEARCH_TRANSPORT_ERROR_RE = re.compile(
r"\b(?:ConflictError|NotFoundError|RequestError|TransportError"
r"|AuthenticationException|AuthorizationException)\((\d{3})[,)]"
)
def _opensearch_failure_metadata(error: str) -> dict | None:
"""Classify an opensearch-py transport failure surfaced as a task error.
Returns None when the error is not an OpenSearch transport failure, so the
caller falls through to the existing classification branches.
"""
if not error:
return None
match = _OPENSEARCH_TRANSPORT_ERROR_RE.search(error)
if not match:
return None
status = match.group(1)
if status == "409":
message = (
"The search index was busy and this change conflicted with another "
"update. Nothing is wrong with the file — retry ingestion."
)
else:
message = (
f"The search index rejected this document (OpenSearch error {status}). "
"This is an infrastructure problem, not a problem with the file. "
"Retry ingestion, and check the backend logs if it persists."
)
# RETRYABLE across the board: of the two available values, USER_ACTIONABLE
# would be actively wrong — there is nothing the uploader can change about
# their file to fix a search-index fault.
return {
"component": "opensearch",
"failure_phase": "indexing",
"user_facing_message": message,
"actionable_by": "RETRYABLE",
}
def _provider_credential_failure_metadata(error: str) -> dict | None:
"""Classify invalid/revoked provider API key failures for ingestion toasts/UI."""
if not error:
return None
from api.provider_validation import (
is_provider_credential_error,
sanitize_provider_error_content,
)
cleaned = sanitize_provider_error_content(error)
if not (is_provider_credential_error(error) or is_provider_credential_error(cleaned)):
return None
return {
"component": "openrag",
"failure_phase": "embedding",
"user_facing_message": cleaned,
"actionable_by": "USER_ACTIONABLE",
}
def _context_window_failure_metadata(error: str) -> dict | None:
"""Classify a chunk the embedding model rejected as longer than its input limit."""
from api.provider_validation import context_window_token_counts, is_context_window_error
if not is_context_window_error(error):
return None
limit, requested = context_window_token_counts(error)
if limit and requested:
size = f" ({requested} tokens; the limit is {limit})"
elif limit:
size = f" (the limit is {limit} tokens)"
else:
size = ""
return {
"component": "openrag",
"failure_phase": "embedding",
"user_facing_message": (
f"A section of this document is longer than the embedding model accepts{size}. "
"Try a smaller chunk size or an embedding model with a larger input limit, "
"then retry ingestion."
),
"actionable_by": "USER_ACTIONABLE",
}
def _is_transient_connectivity_error(error: str) -> bool:
lowered = error.lower()
return any(marker in lowered for marker in _TRANSIENT_CONNECTIVITY_ERROR_MARKERS)
def _is_docling_transient_error(error: str) -> bool:
lowered = error.lower()
# Legacy timeout messages emitted by Docling polling path.
if "docling conversion did not complete" in lowered:
return any(marker in lowered for marker in _DOCLING_TRANSIENT_ERROR_MARKERS)
# Connector-originated transient network failures can still surface while
# file phase is DOCLING (before conversion can complete).
return _is_transient_connectivity_error(error)
def _transient_connectivity_user_message(error: str) -> str:
lowered = error.lower()
if any(
marker in lowered
for marker in (
"disconnect",
"without sending a response",
"connection refused",
"connection reset",
"broken pipe",
"remote protocol",
)
):
return "Ingestion service connection was lost. Please retry ingestion."
if any(marker in lowered for marker in ("timeout", "timed out")):
return "Document processing timed out. Please retry ingestion."
return "Ingestion service was temporarily unavailable. Please retry ingestion."
def _is_langflow_transport_failure(error: str) -> bool:
"""HTTP client / Langflow transport failures on the connector ingest path."""
lowered = error.lower()
return any(
marker in lowered
for marker in (
"server disconnected",
"disconnected without sending a response",
"remote protocol error",
"read error",
"write error",
)
)
class IngestionTimeoutError(Exception):
"""Raised when file processing exceeds the configured timeout"""
pass
class TaskService:
# Cleanup interval in seconds (2 hours)
CLEANUP_INTERVAL_SECONDS = 2 * 60 * 60
def __init__(
self,
document_service=None,
models_service=None,
ingestion_timeout=3600,
docling_service=None,
docling_polling_service=None,
session_manager=None,
):
self.document_service = document_service
self.models_service = models_service
self.docling_service = docling_service
self.session_manager = session_manager
# Backend-side Docling polling coordinator. Injected by the container
# so LangflowFileProcessor receives it from the established DI chain
# rather than constructing it inline. None disables the two-phase
# flow and falls back to the legacy single-call ingestion path.
self.docling_polling_service = docling_polling_service
self.task_store: dict[str, dict[str, UploadTask]] = {} # user_id -> {task_id -> UploadTask}
self.background_tasks = set()
self.ingestion_timeout = ingestion_timeout
self._cleanup_task: asyncio.Task | None = None
# Locks for task counter updates, keyed by task_id
# Kept separate from UploadTask to maintain serialization compatibility
self._task_locks: dict[str, asyncio.Lock] = {}
# Track individual file processing tasks for cancellation: (task_id, file_path) -> asyncio.Task
self._file_tasks: dict[tuple[str, str], asyncio.Task] = {}
# Global semaphore to limit concurrent file processing across all tasks.
# TaskService is a singleton, so this limits concurrency system-wide.
self._worker_count = get_worker_count()
self._processing_semaphore = asyncio.Semaphore(self._worker_count)
def _get_task_lock(self, task_id: str) -> asyncio.Lock:
"""Get or create a lock for a specific task's counter updates"""
if task_id not in self._task_locks:
self._task_locks[task_id] = asyncio.Lock()
return self._task_locks[task_id]
def start_cleanup_scheduler(self) -> None:
"""Start the periodic cleanup background task.
Should be called once after the event loop is running (e.g., during app startup).
"""
if self._cleanup_task is None or self._cleanup_task.done():
self._cleanup_task = asyncio.create_task(self._periodic_cleanup())
logger.info(
"Started periodic task cleanup scheduler",
interval_seconds=self.CLEANUP_INTERVAL_SECONDS,
)
async def _periodic_cleanup(self) -> None:
"""Periodically clean up old completed/failed tasks."""
while True:
try:
await asyncio.sleep(self.CLEANUP_INTERVAL_SECONDS)
cleaned = await self.cleanup_old_tasks()
if cleaned > 0:
logger.debug("Periodic cleanup completed", tasks_cleaned=cleaned)
except asyncio.CancelledError:
logger.debug("Periodic cleanup task cancelled")
raise
except Exception as e:
logger.warning("Error during periodic cleanup", error=str(e))
async def exponential_backoff_delay(
self, retry_count: int, base_delay: float = 1.0, max_delay: float = 60.0
) -> None:
"""Apply exponential backoff with jitter"""
delay = min(base_delay * (2**retry_count) + random.uniform(0, 1), max_delay)
await asyncio.sleep(delay)
async def _process_with_timeout(
self, coro: Coroutine[Any, Any, T], timeout_seconds: int | None = None
) -> T:
"""Wrapper to add timeout protection to file processing
Args:
coro: Coroutine to execute with timeout
timeout_seconds: Timeout in seconds (uses self.ingestion_timeout if None)
Returns:
The result of the coroutine
Raises:
IngestionTimeoutError: If processing exceeds timeout
"""
timeout: int = timeout_seconds or self.ingestion_timeout
try:
return await asyncio.wait_for(coro, timeout=timeout)
except TimeoutError:
raise IngestionTimeoutError(
f"File processing timed out after {timeout} seconds."
) from None
async def create_upload_task(
self,
user_id: str,
file_paths: list,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
original_filenames: dict | None = None,
replace_duplicates: bool = False,
settings: dict | None = None,
preview_mode: bool = False,
*,
allow_anonymous_delete: bool,
existing_task_id: str | None = None,
) -> str:
"""Create a new upload task for bulk file processing.
``allow_anonymous_delete`` is the uploading user's resolved
``knowledge:delete:anonymous`` and has no default; see
``TaskProcessor.delete_document_by_filename``.
"""
# Use default DocumentFileProcessor with user context
from models.processors import DocumentFileProcessor
processor = DocumentFileProcessor(
self.document_service,
models_service=self.models_service,
owner_user_id=user_id,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
docling_service=self.docling_service,
replace_duplicates=replace_duplicates,
session_manager=self.session_manager,
settings=settings,
allow_anonymous_delete=allow_anonymous_delete,
)
return await self.create_custom_task(
user_id,
file_paths,
processor,
original_filenames=original_filenames,
temp_file_paths=file_paths,
preview_mode=preview_mode,
existing_task_id=existing_task_id,
)
async def create_langflow_upload_task(
self,
user_id: str,
file_paths: list,
langflow_file_service,
session_manager,
original_filenames: dict | None = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
session_id: str = None,
tweaks: dict = None,
settings: dict = None,
replace_duplicates: bool = False,
connector_type: str = "local",
existing_task_id: str = None,
temp_file_paths: list | None = None,
preview_mode: bool = False,
*,
allow_anonymous_delete: bool,
) -> str:
"""Create a new upload task for Langflow file processing with upload and ingest.
``allow_anonymous_delete`` is the uploading user's resolved
``knowledge:delete:anonymous`` and has no default; see
``TaskProcessor.delete_document_by_filename``.
"""
# Use LangflowFileProcessor with user context
from models.processors import LangflowFileProcessor
processor = LangflowFileProcessor(
langflow_file_service=langflow_file_service,
session_manager=session_manager,
owner_user_id=user_id,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
session_id=session_id,
tweaks=tweaks,
settings=settings,
replace_duplicates=replace_duplicates,
connector_type=connector_type,
docling_polling_service=self.docling_polling_service,
preview_mode=preview_mode,
allow_anonymous_delete=allow_anonymous_delete,
)
return await self.create_custom_task(
user_id,
file_paths,
processor,
original_filenames,
existing_task_id=existing_task_id,
temp_file_paths=temp_file_paths if temp_file_paths is not None else file_paths,
preview_mode=preview_mode,
)
def record_rejected_uploads(
self,
user_id: str,
rejections: list[tuple[str, int]],
*,
task_id: str | None = None,
) -> str:
"""Keep files the client did not send because they exceed the upload limit.
Attached to ``task_id`` when that task exists, already counted as
failed, so the panel shows them after the toasts are gone. Otherwise a
completed task that contains only these failures.
"""
from utils.upload_limits import max_upload_mb, upload_too_large_message
if not rejections:
raise ValueError("rejections must not be empty")
limit_mb = max_upload_mb()
store_user_id = user_id or AnonymousUser().user_id
upload_task = self._find_upload_task(task_id) if task_id else None
created = upload_task is None
if upload_task is None:
upload_task = UploadTask(
task_id=task_id or str(uuid.uuid4()),
total_files=0,
status=TaskStatus.COMPLETED,
)
if store_user_id not in self.task_store:
self.task_store[store_user_id] = {}
self.task_store[store_user_id][upload_task.task_id] = upload_task
for index, (filename, size_bytes) in enumerate(rejections):
key = f"over-limit:{upload_task.task_id}:{index}:{filename}"
upload_task.file_tasks[key] = FileTask(
file_path=key,
filename=filename,
status=TaskStatus.FAILED,
error=upload_too_large_message(filename, size_bytes, limit_mb),
)
upload_task.total_files += 1
upload_task.failed_files += 1
upload_task.processed_files += 1
upload_task.updated_at = time.time()
if created:
upload_task.status = TaskStatus.COMPLETED
return upload_task.task_id
async def create_url_upload_task(
self,
owner_user_id: str,
docs_url: str,
crawl_depth: int,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
connector_type: str = "openrag_docs",
is_sample_data: bool = False,
existing_task_id: str = None,
) -> str:
"""Create a new upload task for traditional (non-Langflow) URL ingestion."""
from models.url import UrlProcessor
processor = UrlProcessor(
document_service=self.document_service,
models_service=self.models_service,
docs_url=docs_url,
crawl_depth=crawl_depth,
owner_user_id=owner_user_id,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
connector_type=connector_type,
is_sample_data=is_sample_data,
)
return await self.create_custom_task(
owner_user_id, [docs_url], processor, existing_task_id=existing_task_id
)
async def create_langflow_url_upload_task(
self,
owner_user_id: str,
docs_url: str,
crawl_depth: int,
langflow_file_service,
session_manager,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
connector_type: str = "openrag_docs",
prevent_outside: bool = True,
tweaks: dict = None,
existing_task_id: str = None,
) -> str:
"""Create a new upload task for Langflow URL ingestion."""
from models.url import LangflowUrlProcessor
processor = LangflowUrlProcessor(
langflow_file_service=langflow_file_service,
session_manager=session_manager,
docs_url=docs_url,
crawl_depth=crawl_depth,
owner_user_id=owner_user_id,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
connector_type=connector_type,
prevent_outside=prevent_outside,
tweaks=tweaks,
)
return await self.create_custom_task(
owner_user_id, [docs_url], processor, existing_task_id=existing_task_id
)
async def create_custom_task(
self,
user_id: str,
items: list,
processor,
original_filenames: dict | None = None,
existing_task_id: str = None,
temp_file_paths: list | None = None,
preview_mode: bool = False,
) -> str:
"""Create a new task with custom processor for any type of items"""
import os
# Store anonymous tasks under a stable key so they can be retrieved later
store_user_id = user_id or AnonymousUser().user_id
task_id = existing_task_id or str(uuid.uuid4())
# Create file tasks with original filenames if provided
normalized_originals = (
{str(k): v for k, v in original_filenames.items()} if original_filenames else {}
)
file_tasks = {
str(item): FileTask(
file_path=str(item),
filename=normalized_originals.get(str(item), os.path.basename(str(item))),
)
for item in items
}
task_already_exists = (
existing_task_id
and store_user_id in self.task_store
and existing_task_id in self.task_store[store_user_id]
)
if task_already_exists:
upload_task = self.task_store[store_user_id][existing_task_id]
upload_task.file_tasks.update(file_tasks)
upload_task.total_files += len(items)
upload_task.status = TaskStatus.RUNNING
else:
upload_task = UploadTask(
task_id=task_id,
total_files=len(items),
file_tasks=file_tasks,
preview_mode=preview_mode,
)
upload_task.processor = processor
if store_user_id not in self.task_store:
self.task_store[store_user_id] = {}
self.task_store[store_user_id][task_id] = upload_task
# Store temp file paths for cleanup after processing
if temp_file_paths:
if upload_task.temp_file_paths is None:
upload_task.temp_file_paths = []
normalized_temp_paths = [str(path) for path in temp_file_paths]
unknown_temp_paths = [
path for path in normalized_temp_paths if path not in upload_task.file_tasks
]
if unknown_temp_paths:
logger.warning(
"temp_file_paths do not match file_tasks keys; retry retention may fail",
task_id=task_id,
unknown_temp_paths=unknown_temp_paths,
)
upload_task.temp_file_paths.extend(normalized_temp_paths)
# Start background processing
background_task = asyncio.create_task(
self.background_custom_processor(store_user_id, task_id, items, processor)
)
self.background_tasks.add(background_task)
background_task.add_done_callback(self.background_tasks.discard)
# Store the task whenever this call creates one. URL sources reserve an
# ID before enqueueing, so `existing_task_id` alone does not mean the
# task existed already.
if not task_already_exists:
upload_task.background_task = background_task
# Send telemetry event for task creation with metadata
asyncio.create_task(
TelemetryClient.send_event(
Category.TASK_OPERATIONS,
MessageId.ORB_TASK_CREATED,
metadata={
"total_files": len(items),
"processor_type": processor.__class__.__name__,
},
)
)
return task_id
def _get_display_filenames(self, upload_task: UploadTask) -> list[str]:
filenames: list[str] = [
task.filename or os.path.basename(task.file_path)
for task in upload_task.file_tasks.values()
]
if len(filenames) <= 3:
# e.g. ['book-1.xlsx']
# e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx']
return filenames
# e.g. ['book-1.xlsx', 'book-2.xlsx', 'book-3.xlsx', '...']
return filenames[:3] + ["..."]
def _format_duration(self, duration: float | int) -> str:
"""
Convert specified duration (seconds) into a human-readable string:
- < 60 s → "45s"
- < 60 min → "3m 42s"
- ≥ 60 min → "2h 14m 35s"
"""
total_seconds: int = max(0, int(duration))
if total_seconds < 60:
return f"{total_seconds}s"
mins, secs = divmod(total_seconds, 60)
if mins < 60:
return f"{mins}m {secs}s"
hours, mins = divmod(mins, 60)
return f"{hours}h {mins}m {secs}s"
def _find_upload_task(self, task_id: str) -> UploadTask | None:
"""The task with this id, whichever user's store holds it.
A claim conflict is not confined to one task: the registry keys names by
ownership scope, not by task, so a file in one task can lose a name to a
file in another — two uploads in flight at once, or an upload racing a
connector sync.
"""
for tasks in self.task_store.values():
found = tasks.get(task_id)
if found is not None:
return found
return None
async def _fail_files_stranded_by(self, refused_holders: set[str]) -> None:
"""Fail the files that were skipped for a name this one never indexed.
The duplicate gate decides a losing file's outcome the moment it loses
the claim, before the winner has ingested anything. When the winner then
fails, nothing landed under that name, so the loser's "skipped, a file
with this name already exists" describes something that never happened —
and it is counted as a successful file, which is a document the user
never got. SKIPPED is also not a retry candidate (retry_failed_files
takes FAILED only), so nothing would ever recover it.
Failing it instead says what happened and puts it back within reach of a
retry, which re-runs the gate and resolves correctly either way: it
ingests if the name is genuinely free, and skips again as a duplicate if
something else holds it by then.
Each loser is corrected inside its own task, which may not be this one
and may already have finished. That stays consistent: processed_files is
untouched, one successful file becomes a failed one under that task's
lock, and a COMPLETED task carrying failures is the ordinary shape retry
operates on — it refuses only while a task is still RUNNING. The
completion event for such a task was emitted with the earlier counts,
which leaves a stale statistic behind rather than inconsistent state.
"""
for holder in refused_holders:
loser_task_id, _, file_key = holder.partition(":")
if not file_key:
continue
owning_task = self._find_upload_task(loser_task_id)
if owning_task is None:
# Task already evicted from the store; nothing left to correct.
continue
stranded = owning_task.file_tasks.get(file_key)
if stranded is None or stranded.status != TaskStatus.SKIPPED:
continue
# Only the in-flight skip is ours to reverse; a file skipped against
# something already indexed is unaffected by this failure.
if (stranded.result or {}).get("reason") != "duplicate_filename":
continue
stranded.status = TaskStatus.FAILED
stranded.error = INFLIGHT_CLAIM_WINNER_FAILED_ERROR
stranded.result = None
stranded.updated_at = time.time()
async with self._get_task_lock(loser_task_id):
if owning_task.successful_files > 0:
owning_task.successful_files -= 1
owning_task.failed_files += 1
owning_task.updated_at = time.time()
logger.info(
"Reversed an in-flight duplicate skip after the ingesting file failed",
task_id=loser_task_id,
file_path=stranded.file_path,
filename=stranded.filename,
)
async def background_custom_processor(
self, user_id: str, task_id: str, items: list, processor=None
) -> None:
"""Background task to process items using custom processor"""
try:
upload_task: UploadTask = self.task_store[user_id][task_id]
upload_task.status = TaskStatus.RUNNING
upload_task.updated_at = time.time()
processor = processor or upload_task.processor
logger.info(
"Upload / ingestion task started",
task_number=upload_task.sequence_number,
task_id=task_id,
total_files=upload_task.total_files,
filenames=self._get_display_filenames(upload_task),
processor_type=processor.__class__.__name__,
user_id=user_id,
worker_count=self._worker_count,
)
# Process items with limited concurrency using the global semaphore
# - Limits concurrency across all tasks, not just within this one
# - Potential bottlenecks related to downstream Langflow / Docling capacity rather than backend I/O
async def process_with_semaphore(item, item_key: str):
async with self._processing_semaphore:
file_task = upload_task.file_tasks[item_key]
# Skip files that were already cancelled or failed before reaching the worker
if file_task.status in [
TaskStatus.FAILED,
TaskStatus.COMPLETED,
TaskStatus.SKIPPED,
]:
logger.info(
"File processing task skipped (already terminal)",
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
status=file_task.status.value,
)
# Increment processed_files for cancelled files so task can complete
async with self._get_task_lock(task_id):
upload_task.processed_files += 1
return
# Check again after acquiring semaphore - file may have been cancelled while waiting
if file_task.status in [
TaskStatus.FAILED,
TaskStatus.COMPLETED,
TaskStatus.SKIPPED,
]:
logger.info(
"File processing task skipped (cancelled while waiting for semaphore)",
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
status=file_task.status.value,
)
# Increment processed_files for cancelled files so task can complete
async with self._get_task_lock(task_id):
upload_task.processed_files += 1
return
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
logger.info(
"File processing task running",
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
)
try:
# Add timeout protection to prevent indefinite hangs
# Traces opened for this file (and its model calls)
# carry the task; an upload's files share a session.
with trace_context(
session_id=task_id, attributes={"openrag.task_id": task_id}
):
await self._process_with_timeout(
processor.process_item(upload_task, item, file_task),
timeout_seconds=self.ingestion_timeout,
)
logger.info(
"File processing task succeeded",
status="PASSED",
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
)
except asyncio.CancelledError:
# Handle cancellation explicitly
if file_task.status == TaskStatus.RUNNING:
file_task.status = TaskStatus.FAILED
file_task.error = "File processing task cancelled."
async with self._get_task_lock(task_id):
upload_task.failed_files += 1
logger.warning(
"File processing task cancelled",
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
)
raise # Re-raise to propagate cancellation
except IngestionTimeoutError as e:
# Handle timeout explicitly
if file_task.status == TaskStatus.RUNNING:
file_task.status = TaskStatus.FAILED
file_task.error = str(e)
async with self._get_task_lock(task_id):
upload_task.failed_files += 1
# Don't re-raise - treat as normal failure, not cancellation
logger.error(
"File processing task timed out",
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
exception=str(e),
)
except Exception as e:
# Note: Processors already handle incrementing failed_files and
# setting file_task status/error, so we don't duplicate that here.
# Only update timestamp if processor didn't already set it
if file_task.status == TaskStatus.RUNNING:
file_task.status = TaskStatus.FAILED
# Don't overwrite cancellation errors - preserve user cancellation message
if file_task.error != "File cancelled by user":
if not file_task.error:
file_task.error = str(e) or repr(e)
logger.error(
"File processing task exception encountered",
status="FAILED",
traceback=traceback.format_exc(),
task_number=upload_task.sequence_number,
task_id=task_id,
file_path=file_task.file_path,
exception=str(e),
)
finally:
# Hand back the filename this file held while in flight
# (utils.filename_claims), so the next task — or a retry
# of this one — can take it. Every file passes through
# here, including the ones that never claimed anything.
refused = filename_claims.release(claim_holder(task_id, item_key))
if refused and file_task.status == TaskStatus.FAILED:
await self._fail_files_stranded_by(refused)
file_task.updated_at = time.time()
# Only increment processed_files if the file reached a terminal state
# This prevents counter inconsistency on cancellation.
# SKIPPED is terminal too — e.g. source-deleted files in
# connector sync — and must count, or the upload task
# never reaches processed_files >= total_files and stays
# open forever.
if file_task.status in [
TaskStatus.COMPLETED,
TaskStatus.FAILED,
TaskStatus.SKIPPED,
]:
async with self._get_task_lock(task_id):
upload_task.processed_files += 1
upload_task.updated_at = time.time()
# Create tasks and track them for file-level cancellation
tasks = []
for item in items:
item_key = str(item)
task = asyncio.create_task(process_with_semaphore(item, item_key))
tasks.append(task)
# Track file task for cancellation (will be cleaned up in finally block)
file_task = upload_task.file_tasks.get(item_key)
if file_task:
self._file_tasks[(task_id, file_task.file_path)] = task
try:
await asyncio.gather(*tasks, return_exceptions=True)
finally:
# Clean up file task references even if parent task is cancelled
for item in items:
item_key = str(item)
file_task = upload_task.file_tasks.get(item_key)
if file_task:
self._file_tasks.pop((task_id, file_task.file_path), None)
# Mark task as completed if all files (including appended ones) are done
if upload_task.processed_files >= upload_task.total_files:
# Force an index refresh BEFORE marking the task COMPLETED so that
# callers polling for COMPLETED status can immediately query or delete
# the newly indexed chunks without hitting the near-real-time refresh window.
if upload_task.successful_files > 0:
try:
from config.settings import clients, get_index_name
await clients.opensearch.indices.refresh(index=get_index_name())
except Exception as e:
logger.debug("Index refresh after ingest failed (non-fatal)", error=str(e))
upload_task.status = TaskStatus.COMPLETED
upload_task.updated_at = time.time()
# Clean up upload temps that are not retryable; keep local RETRYABLE
# failures so retry can re-read the staged source file.
self._cleanup_upload_temp_files(upload_task)
status: str = "FAILED"
if upload_task.failed_files == 0:
status = "PASSED"
elif upload_task.successful_files > 0:
status = "FAILED (partial success)"
logger.info(
"Upload / ingestion task finished",
status=status,
task_number=upload_task.sequence_number,
task_id=task_id,
duration=self._format_duration(upload_task.duration_seconds),
total_files=upload_task.total_files,
processed_files=upload_task.processed_files,
successful_files=upload_task.successful_files,
failed_files=upload_task.failed_files,
filenames=self._get_display_filenames(upload_task),
processor_type=processor.__class__.__name__,
user_id=user_id,
worker_count=self._worker_count,
)
# Send telemetry for task completion
asyncio.create_task(
TelemetryClient.send_event(
Category.TASK_OPERATIONS,
MessageId.ORB_TASK_COMPLETE,
metadata={
"total_files": upload_task.total_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
},
)
)
except asyncio.CancelledError:
if user_id in self.task_store and task_id in self.task_store[user_id]:
# Task status and pending files already handled by cancel_task()
upload_task = self.task_store[user_id][task_id]
logger.warning(
"Upload / ingestion task cancelled",
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
duration=self._format_duration(upload_task.duration_seconds),
total_files=upload_task.total_files,
processed_files=upload_task.processed_files,
successful_files=upload_task.successful_files,
failed_files=upload_task.failed_files,
filenames=self._get_display_filenames(upload_task),
processor_type=upload_task.processor.__class__.__name__,
user_id=user_id,
worker_count=self._worker_count,
)
else:
logger.warning(
"Upload / ingestion task cancelled",
status="FAILED",
task_id=task_id,
user_id=user_id,
worker_count=self._worker_count,
)
raise # Re-raise to properly handle cancellation
except Exception as e:
if user_id in self.task_store and task_id in self.task_store[user_id]:
upload_task = self.task_store[user_id][task_id]
upload_task.status = TaskStatus.FAILED
upload_task.updated_at = time.time()
logger.error(
"Upload / ingestion task exception encountered",
status="FAILED",
task_number=upload_task.sequence_number,
task_id=task_id,
duration=self._format_duration(upload_task.duration_seconds),
total_files=upload_task.total_files,
processed_files=upload_task.processed_files,
successful_files=upload_task.successful_files,
failed_files=upload_task.failed_files,
filenames=self._get_display_filenames(upload_task),
processor_type=upload_task.processor.__class__.__name__,
user_id=user_id,
worker_count=self._worker_count,
exception=str(e),
)
# Send telemetry for task failure
asyncio.create_task(
TelemetryClient.send_event(
Category.TASK_OPERATIONS,
MessageId.ORB_TASK_FAILED,
metadata={
"total_files": upload_task.total_files,
"processed_files": upload_task.processed_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
},
)
)
else:
logger.error(
"Upload / ingestion exception encountered",
status="FAILED",
task_id=task_id,
user_id=user_id,
worker_count=self._worker_count,
exception=str(e),
)
def _resolve_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:
"""Look up a task by ID, falling back to anonymous/shared tasks."""
if not task_id:
return None
for candidate_user_id in [user_id, AnonymousUser().user_id]:
if (
candidate_user_id in self.task_store
and task_id in self.task_store[candidate_user_id]
):
return self.task_store[candidate_user_id][task_id]
return None
def get_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:
"""Public accessor for upload task lookup."""
return self._resolve_upload_task(user_id, task_id)
def _resolve_upload_task_store(
self, user_id: str, task_id: str
) -> tuple[str, UploadTask] | None:
"""Return (store_user_id, upload_task) for a task visible to this user."""
if not task_id:
return None
for candidate_user_id in [user_id, AnonymousUser().user_id]:
if (
candidate_user_id in self.task_store
and task_id in self.task_store[candidate_user_id]
):
return candidate_user_id, self.task_store[candidate_user_id][task_id]
return None
async def retry_failed_files(
self,
user_id: str,
task_id: str,
*,
file_paths: list[str] | None = None,
retryable_only: bool = True,
) -> dict | None:
"""Re-queue failed files for ingestion when their source paths still exist.
Only files classified as RETRYABLE are retried when *retryable_only* is
True (the default). When *file_paths* is set, only those task paths are
considered; paths missing from the task or not in a failed state are
reported in *skipped*. This reuses the task's original processor — it
does not accept new uploads from the client.
Connector tasks often use provider IDs (not local absolute paths) as
``file_path`` keys. For those non-absolute identifiers, skip local
filesystem existence checks and let the connector processor re-fetch.
"""
resolved = self._resolve_upload_task_store(user_id, task_id)
if resolved is None:
return None
store_user_id, upload_task = resolved
paths_to_retry: list[str] = []
skipped: list[dict] = []
requested_paths = set(file_paths) if file_paths is not None else None
# Keep status checks, candidate selection, and state transitions inside
# one lock to avoid concurrent retry requests enqueueing duplicates.
async with self._get_task_lock(task_id):
processor = upload_task.processor
if upload_task.status == TaskStatus.RUNNING:
return {
"error": "task_in_progress",
"message": "Task is still running",
"task_id": task_id,
}
if processor is None:
return {
"error": "no_processor",
"message": "Cannot retry: task processor is no longer available",
"task_id": task_id,
}
if requested_paths is not None:
for path in requested_paths:
file_task = upload_task.file_tasks.get(path)
if file_task is None:
skipped.append({"file_path": path, "reason": "file_not_in_task"})
elif file_task.status != TaskStatus.FAILED:
skipped.append(
{
"file_path": path,
"filename": file_task.filename,
"reason": "not_failed",
}
)
# Build retry candidates before mutating shared task/file state.
retry_candidates: list[tuple[str, FileTask]] = []
for file_path, file_task in list(upload_task.file_tasks.items()):
if requested_paths is not None and file_path not in requested_paths:
continue
if file_task.status != TaskStatus.FAILED:
continue
if retryable_only:
metadata = self._infer_failure_metadata(file_task)
if not metadata or metadata.get("actionable_by") != "RETRYABLE":
skipped.append(
{
"file_path": file_path,
"filename": file_task.filename,
"reason": "not_retryable",
}
)
continue
# Only enforce local source-file existence for absolute paths.
# Connector-backed tasks typically store remote IDs as file_path.
if os.path.isabs(file_path) and not os.path.isfile(file_path):
skipped.append(
{
"file_path": file_path,
"filename": file_task.filename,
"reason": "source_file_missing",
}
)
continue
retry_candidates.append((file_path, file_task))
now = time.time()
for file_path, file_task in retry_candidates:
if upload_task.failed_files > 0:
upload_task.failed_files -= 1
if upload_task.processed_files > 0:
upload_task.processed_files -= 1
file_task.status = TaskStatus.PENDING
file_task.error = None
file_task.result = None
file_task.retry_count += 1
file_task.docling_task_id = None
# Connector tasks use remote IDs as file_path and ingest via Langflow.
if os.path.isabs(file_path):
file_task.docling_status = DoclingPhaseStatus.PENDING
file_task.phase = IngestionPhase.DOCLING
else:
file_task.docling_status = DoclingPhaseStatus.PENDING
file_task.phase = IngestionPhase.LANGFLOW
file_task.updated_at = now
paths_to_retry.append(file_path)
if paths_to_retry:
upload_task.status = TaskStatus.RUNNING
upload_task.updated_at = now
if not paths_to_retry:
return {
"task_id": task_id,
"retried": 0,
"skipped": skipped,
"status": "no_op",
"message": "No retryable files with available source data",
}
background_task = asyncio.create_task(
self.background_custom_processor(store_user_id, task_id, paths_to_retry, processor)
)
upload_task.background_task = background_task
self.background_tasks.add(background_task)
def _clear_retry_background_task(done_task: asyncio.Task) -> None:
self.background_tasks.discard(done_task)
if upload_task.background_task is done_task:
upload_task.background_task = None
background_task.add_done_callback(_clear_retry_background_task)
return {
"task_id": task_id,
"retried": len(paths_to_retry),
"skipped": skipped,
"status": "accepted",
}
@staticmethod
def _keep_completed_file_in_list(file_task: FileTask) -> bool:
"""Source-deleted cleanups stay visible so the UI can show Removed.
Normal completed ingests are omitted from the task file list to keep it
tidy. A file that was removed from the index because it disappeared at
the source is also completed, but dropping it would hide the cleanup.
"""
result = file_task.result or {}
return result.get("reason") == "deleted_at_source"
def _serialize_file_task(self, file_task: FileTask) -> dict:
"""Serialize a FileTask to the standard dict shape."""
return {
"status": file_task.status.value,
"result": file_task.result,
"error": file_task.error,
"retry_count": file_task.retry_count,
"created_at": file_task.created_at,
"updated_at": file_task.updated_at,
"duration_seconds": file_task.duration_seconds,
"filename": file_task.filename,
"phase": file_task.phase.value,
"docling_status": file_task.docling_status.value,
"docling_task_id": file_task.docling_task_id,
}
def _infer_failure_metadata(self, file_task: FileTask) -> dict | None:
"""Infer structured failure metadata for a failed FileTask.
Returns a dict with component, failure_phase, user_facing_message, and
actionable_by when the failure can be classified, or None when the cause
is unknown and no fields should be emitted.
Priority order: transient / retryable docling outcomes (expired, polling
timeout) before generic docling FAILED, which indicates conversion failure.
"""
docling_status = file_task.docling_status
phase = file_task.phase
error = file_task.error or ""
if _is_task_cancellation_error(error):
return {
"failure_phase": "cancelled",
"user_facing_message": "Ingestion was cancelled.",
"actionable_by": "USER_ACTIONABLE",
}
from utils.upload_limits import is_upload_size_error, upload_size_user_message
# A size refusal is stored while docling_status is still PENDING, because
# the proxy rejects the body before a Docling task exists. Later branches
# would drop that text.
if is_upload_size_error(error):
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": upload_size_user_message(error),
"actionable_by": "USER_ACTIONABLE",
}
# Before the substring heuristics below, which would otherwise read this
# as a plain duplicate: the file was never ingested, and retrying is
# exactly the right move.
if error == INFLIGHT_CLAIM_WINNER_FAILED_ERROR:
return {
"component": "openrag",
"failure_phase": "unknown",
"user_facing_message": (
f"{INFLIGHT_CLAIM_WINNER_FAILED_ERROR} Retry to ingest it."
),
"actionable_by": "RETRYABLE",
}
# Before any substring heuristic: an OpenSearch transport failure carries a
# JSON payload whose contents would otherwise match the file-corruption and
# "already exists" markers below.
opensearch_meta = _opensearch_failure_metadata(error)
if opensearch_meta:
return opensearch_meta
# Password-protected PDFs. Newer docling no longer names the reason, so the
# backend recovers it from the file bytes (docling_service.password_protected_pdf_error).
if "incorrect password" in error.lower():
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": "This PDF is password-protected. Password-protected PDFs are not supported. Remove the password and upload the PDF again.",
"actionable_by": "USER_ACTIONABLE",
}
if docling_status == DoclingPhaseStatus.EXPIRED:
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": (
"The document processing result could not be found. "
"The task may have expired. Please retry ingestion."
),
"actionable_by": "RETRYABLE",
}
# First, check if the error is non-retryable based on common markers
if _is_non_retryable_file_error(error):
# Image files that produced no text almost certainly failed because OCR
# is disabled — not because the file is corrupted. Give a targeted tip.
filename = file_task.filename or file_task.file_path or ""
if "no text content could be extracted" in error.lower() and _is_ocr_required_file(
filename
):
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": (
"No text content could be extracted from this image file. "
"Enable OCR in Settings > Knowledge Base and retry ingestion."
),
"actionable_by": "RETRYABLE",
}
if phase == IngestionPhase.LANGFLOW:
component = "langflow"
failure_phase = "unknown"
else:
component = "docling"
failure_phase = "parsing"
# Extract detailed error message if possible, otherwise default to corrupted/invalid msg
msg = (
"The file appears corrupted or invalid and cannot be processed. "
"Upload a valid file."
)
if "Docling result unavailable after SUCCESS status: " in error:
msg = error.split("Docling result unavailable after SUCCESS status: ", 1)[1]
elif "Docling conversion did not complete" in error:
sub_msg = error.split("Docling conversion did not complete", 1)[1]
if sub_msg.startswith(" (failed): "):
msg = sub_msg[len(" (failed): ") :]
else:
msg = sub_msg.strip(" ():")
return {
"component": component,
"failure_phase": failure_phase,
"user_facing_message": msg,
"actionable_by": "USER_ACTIONABLE",
}
# Handle docling conversion completion issues (failed / timeout)
if phase == IngestionPhase.DOCLING and "Docling conversion did not complete" in error:
user_facing_message = "Document processing timed out. Please retry ingestion."
if "timeout" not in error.lower() and "expired" not in error.lower():
msg = error.split("Docling conversion did not complete", 1)[1]
if msg.startswith(" (failed): "):
user_facing_message = msg[len(" (failed): ") :]
elif msg.startswith(" (timeout): "):
user_facing_message = "Document processing timed out. Please retry ingestion."
else:
user_facing_message = msg.strip(" ():")
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": user_facing_message,
"actionable_by": "RETRYABLE"
if "timed out" in user_facing_message.lower()
else "USER_ACTIONABLE",
}
if phase == IngestionPhase.DOCLING and _is_docling_transient_error(error):
langflow_transport = _is_langflow_transport_failure(error)
return {
"component": "langflow" if langflow_transport else "docling",
"failure_phase": "unknown" if langflow_transport else "parsing",
"user_facing_message": _transient_connectivity_user_message(error),
"actionable_by": "RETRYABLE",
}
if phase == IngestionPhase.DOCLING and docling_status == DoclingPhaseStatus.PROCESSING:
# Check if this was actually a cancellation before assuming timeout
if _is_task_cancellation_error(error):
return {
"failure_phase": "cancelled",
"user_facing_message": "Ingestion was cancelled.",
"actionable_by": "USER_ACTIONABLE",
}
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": "Document processing timed out. Please retry ingestion.",
"actionable_by": "RETRYABLE",
}
if docling_status == DoclingPhaseStatus.FAILED:
msg = "The file could not be processed into readable document content."
if error:
if "Docling result unavailable after SUCCESS status: " in error:
msg = error.split("Docling result unavailable after SUCCESS status: ", 1)[1]
elif "Docling conversion did not complete" in error:
sub_msg = error.split("Docling conversion did not complete", 1)[1]
if sub_msg.startswith(" (failed): "):
msg = sub_msg[len(" (failed): ") :]
else:
msg = sub_msg.strip(" ():")
else:
msg = error
return {
"component": "docling",
"failure_phase": "parsing",
"user_facing_message": msg,
"actionable_by": "USER_ACTIONABLE",
}
if "already exists" in error:
return {
"component": "openrag",
"failure_phase": "file_validation",
"user_facing_message": "A file with this name already exists.",
"actionable_by": "USER_ACTIONABLE",
}
# Prefer credential failures over transport messages. Langflow may report
# "server disconnected" when the real cause is a revoked embedding API key;
# resolve_ingest_error_message() rewrites file_task.error in that case, and
# this check also covers errors that mention both disconnect + API key text.
credential_meta = _provider_credential_failure_metadata(error)
if credential_meta:
return credential_meta
# Before the Langflow fallbacks, which would report the provider's raw
# sentence as a retryable failure: retrying the same chunk fails again.
context_window_meta = _context_window_failure_metadata(error)
if context_window_meta:
return context_window_meta
if _is_transient_connectivity_error(error) or _is_langflow_transport_failure(error):
return {
"component": "langflow",
"failure_phase": "unknown",
"user_facing_message": _transient_connectivity_user_message(error),
"actionable_by": "RETRYABLE",
}
if phase == IngestionPhase.LANGFLOW:
error_lower = error.lower()
if any(
marker in error_lower
for marker in ("timeout", "timed out", "unavailable", "connection refused")
):
return {
"component": "langflow",
"failure_phase": "unknown",
"user_facing_message": (
"Ingestion timed out or the service was unavailable. Please retry."
),
"actionable_by": "RETRYABLE",
}
# Prefer a sanitized specific provider/Langflow message (model missing,
# project misconfig, etc.) over a generic "unexpectedly" toast.
from api.provider_validation import (
is_generic_upstream_error,
is_provider_credential_error,
sanitize_provider_error_content,
)
cleaned = sanitize_provider_error_content(error)
if (
cleaned
and not is_generic_upstream_error(cleaned)
and "{" not in cleaned
and "}" not in cleaned
):
lowered = cleaned.lower()
missing_deployment = "deployment" in lowered and any(
marker in lowered for marker in ("not found", "does not exist")
)
user_actionable = (
is_provider_credential_error(cleaned)
or missing_deployment
or any(
marker in lowered
for marker in (
"model",
"project",
"not found",
"not properly configured",
"no models",
"unauthorized",
"forbidden",
"permission",
"quota",
"rate limit",
)
)
)
return {
"component": "langflow",
"failure_phase": "unknown",
"user_facing_message": cleaned,
"actionable_by": ("USER_ACTIONABLE" if user_actionable else "RETRYABLE"),
}
return {
"component": "langflow",
"failure_phase": "unknown",
"user_facing_message": (
"Ingestion failed unexpectedly. Please retry. "
"If it fails again, contact your administrator."
),
"actionable_by": "RETRYABLE",
}
return None
def get_task_status(self, user_id: str, task_id: str) -> dict | None:
"""Get the status of a specific upload task
Includes fallback to shared tasks stored under the "anonymous" user key
so default system tasks are visible to all users.
"""
upload_task = self._resolve_upload_task(user_id, task_id)
if upload_task is None:
return None
file_statuses = {}
running_files_count = 0
pending_files_count = 0
for file_path, file_task in upload_task.file_tasks.items():
file_statuses[file_path] = self._serialize_file_task(file_task)
# Count running and pending files
if file_task.status.value == "running":
running_files_count += 1
elif file_task.status.value == "pending":
pending_files_count += 1
return {
"task_id": upload_task.task_id,
"status": upload_task.status.value,
"total_files": upload_task.total_files,
"processed_files": upload_task.processed_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
"running_files": running_files_count,
"pending_files": pending_files_count,
"created_at": upload_task.created_at,
"updated_at": upload_task.updated_at,
"duration_seconds": upload_task.duration_seconds,
"files": file_statuses,
}
def get_task_status2(self, user_id: str, task_id: str) -> dict | None:
"""Get the status of a specific upload task with structured failure metadata.
Identical to get_task_status but enriches failed file entries with
component, failure_phase, user_facing_message, and actionable_by fields
when the failure cause can be classified.
"""
upload_task = self._resolve_upload_task(user_id, task_id)
if upload_task is None:
return None
file_statuses = {}
running_files_count = 0
pending_files_count = 0
for file_path, file_task in upload_task.file_tasks.items():
entry = self._serialize_file_task(file_task)
if file_task.status == TaskStatus.FAILED:
metadata = self._infer_failure_metadata(file_task)
if metadata:
entry.update(metadata)
file_statuses[file_path] = entry
if file_task.status == TaskStatus.RUNNING:
running_files_count += 1
elif file_task.status == TaskStatus.PENDING:
pending_files_count += 1
return {
"task_id": upload_task.task_id,
"status": upload_task.status.value,
"total_files": upload_task.total_files,
"processed_files": upload_task.processed_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
"running_files": running_files_count,
"pending_files": pending_files_count,
"created_at": upload_task.created_at,
"updated_at": upload_task.updated_at,
"duration_seconds": upload_task.duration_seconds,
"files": file_statuses,
}
def get_all_tasks2(self, user_id: str) -> list:
"""Get all tasks for a user with structured failure metadata on failed files.
Identical to get_all_tasks but enriches failed file entries with
component, failure_phase, user_facing_message, and actionable_by fields
when the failure cause can be classified.
"""
tasks_by_id = {}
is_shared = False
def add_tasks_from_store(store_user_id):
if store_user_id not in self.task_store:
return
for task_id, upload_task in self.task_store[store_user_id].items():
if task_id in tasks_by_id:
continue
running_files_count = 0
pending_files_count = 0
file_statuses = {}
for file_path, file_task in upload_task.file_tasks.items():
# The task panel only surfaces in-flight/failed files to keep
# the list tidy. Preview-mode tasks instead need every file
# (including completed ones) so the live preview carousel can
# still enumerate them and render their cached Docling layout.
# Source-deleted cleanups are completed but must stay visible
# as successful removals, not vanish like a normal ingest.
if (
file_task.status != TaskStatus.COMPLETED
or upload_task.preview_mode
or self._keep_completed_file_in_list(file_task)
):
entry = self._serialize_file_task(file_task)
if file_task.status == TaskStatus.FAILED:
metadata = self._infer_failure_metadata(file_task)
if metadata:
entry.update(metadata)
file_statuses[file_path] = entry
if file_task.status == TaskStatus.RUNNING:
running_files_count += 1
elif file_task.status == TaskStatus.PENDING:
pending_files_count += 1
tasks_by_id[task_id] = {
"task_id": upload_task.task_id,
"status": upload_task.status.value,
"total_files": upload_task.total_files,
"processed_files": upload_task.processed_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
"running_files": running_files_count,
"pending_files": pending_files_count,
"created_at": upload_task.created_at,
"updated_at": upload_task.updated_at,
"duration_seconds": upload_task.duration_seconds,
"files": file_statuses,
"is_shared": is_shared,
}
add_tasks_from_store(user_id)
is_shared = True
add_tasks_from_store(AnonymousUser().user_id)
tasks = list(tasks_by_id.values())
tasks.sort(key=lambda x: x["created_at"], reverse=True)
return tasks
def get_all_tasks(self, user_id: str) -> list:
"""Get all tasks for a user
Returns the union of the user's own tasks and shared default tasks stored
under the "anonymous" user key. User-owned tasks take precedence
if a task_id overlaps.
"""
tasks_by_id = {}
is_shared = False
def add_tasks_from_store(store_user_id):
if store_user_id not in self.task_store:
return
for task_id, upload_task in self.task_store[store_user_id].items():
if task_id in tasks_by_id:
continue
# Calculate running and pending counts and build file statuses
running_files_count = 0
pending_files_count = 0
file_statuses = {}
for file_path, file_task in upload_task.file_tasks.items():
if file_task.status.value != "completed" or self._keep_completed_file_in_list(
file_task
):
file_statuses[file_path] = {
"status": file_task.status.value,
"result": file_task.result,
"error": file_task.error,
"retry_count": file_task.retry_count,
"created_at": file_task.created_at,
"updated_at": file_task.updated_at,
"duration_seconds": file_task.duration_seconds,
"filename": file_task.filename,
"phase": file_task.phase.value,
"docling_status": file_task.docling_status.value,
"docling_task_id": file_task.docling_task_id,
}
if file_task.status.value == "running":
running_files_count += 1
elif file_task.status.value == "pending":
pending_files_count += 1
tasks_by_id[task_id] = {
"task_id": upload_task.task_id,
"status": upload_task.status.value,
"total_files": upload_task.total_files,
"processed_files": upload_task.processed_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
"running_files": running_files_count,
"pending_files": pending_files_count,
"created_at": upload_task.created_at,
"updated_at": upload_task.updated_at,
"duration_seconds": upload_task.duration_seconds,
"files": file_statuses,
"is_shared": is_shared,
}
# First, add user-owned tasks; then shared anonymous;
add_tasks_from_store(user_id)
is_shared = True
add_tasks_from_store(AnonymousUser().user_id)
tasks = list(tasks_by_id.values())
tasks.sort(key=lambda x: x["created_at"], reverse=True)
return tasks
async def cleanup_old_tasks(self, max_age_seconds: int = 3600) -> int:
"""Remove completed/failed tasks older than max_age_seconds
Args:
max_age_seconds: Maximum age in seconds for completed tasks (default: 1 hour)
Returns:
Number of tasks cleaned up
"""
current_time = time.time()
cleaned_count = 0
# Complexity Analysis:
# O(n) where n = total tasks across all users
for user_id in list(self.task_store.keys()):
for task_id in list(self.task_store[user_id].keys()):
task = self.task_store[user_id][task_id]
# Only cleanup completed, failed, or cancelled tasks that are old enough
if (
task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED]
and current_time - task.updated_at > max_age_seconds
):
# Task is leaving memory; reclaim any retained upload temps
# (including RETRYABLE locals kept for in-flight retry).
self._cleanup_upload_temp_files(task, force=True)
del self.task_store[user_id][task_id]
# Clean up the associated lock
self._task_locks.pop(task_id, None)
cleaned_count += 1
logger.debug(
"Cleaned up old task",
task_id=task_id,
user_id=user_id,
age_seconds=current_time - task.updated_at,
)
# Remove empty user entries
if not self.task_store[user_id]:
del self.task_store[user_id]
if cleaned_count > 0:
logger.info("Task cleanup completed", cleaned_count=cleaned_count)
return cleaned_count
async def cancel_task(self, user_id: str, task_id: str) -> bool:
"""Cancel a task if it exists and is not already completed.
Supports cancellation of shared default tasks stored under the anonymous user.
The outer task lock is held for the entire operation so that concurrent
cancel requests (e.g. rapid UI clicks) are serialised: only the first
caller performs the actual cancellation, subsequent ones see a terminal
status and return False immediately.
"""
# Resolve which user bucket owns the task before acquiring the lock.
candidate_user_ids = [user_id, AnonymousUser().user_id]
store_user_id = None
for candidate_user_id in candidate_user_ids:
if (
candidate_user_id in self.task_store
and task_id in self.task_store[candidate_user_id]
):
store_user_id = candidate_user_id
break
if store_user_id is None:
return False
upload_task = self.task_store[store_user_id][task_id]
# Phase 1: idempotency check — hold the lock only long enough to read
# the current status and signal cancellation. We must NOT hold the lock
# while awaiting the background task: the background task's CancelledError
# handler also acquires _get_task_lock (to increment failed_files), which
# would deadlock with cancel_task holding that same lock.
async with self._get_task_lock(task_id):
# Can only cancel pending or running tasks.
if upload_task.status in [
TaskStatus.COMPLETED,
TaskStatus.FAILED,
TaskStatus.CANCELLED,
]:
return False
# Mark in-flight immediately so a concurrent cancel sees CANCELLED
# and returns False without repeating the work below.
upload_task.status = TaskStatus.CANCELLED
# Phase 2: cancel and await the background task OUTSIDE the lock so the
# background task's own lock acquisitions (failed_files increment, etc.)
# can complete without deadlocking.
if hasattr(upload_task, "background_task") and not upload_task.background_task.done():
upload_task.background_task.cancel()
try:
await upload_task.background_task
except asyncio.CancelledError:
pass # Expected when we cancel the task
except Exception:
pass # Ignore other errors during cancellation
# Phase 3: finalise state — background task is done, so no concurrent
# writer can touch the file tasks any more. Lock is still used for
# safety in case a second cancel slipped through between phases.
async with self._get_task_lock(task_id):
upload_task.updated_at = time.time()
now = time.time()
for file_task in upload_task.file_tasks.values():
if file_task.status in [TaskStatus.PENDING, TaskStatus.RUNNING]:
# PENDING files cancelled before entering process_with_semaphore
# need processed_files incremented here, since the finally block
# won't run. RUNNING files normally cancelled are already counted
# by the finally block, but incrementing here is safe (phase 3 runs
# after background task completes, so no double-counting).
upload_task.processed_files += 1
upload_task.failed_files += 1
file_task.status = TaskStatus.FAILED
file_task.error = "Task cancelled by user"
file_task.updated_at = now
self._cleanup_upload_temp_files(upload_task, force=True)
return True
async def cancel_file(self, user_id: str, task_id: str, file_path: str) -> bool:
"""Cancel a single file within a task.
Marks the file as FAILED with "cancelled by user" error. The file will be
skipped if it's still PENDING, or marked as cancelled if already RUNNING.
"""
# Check candidate user IDs first, then anonymous to find which user ID the task is mapped to
candidate_user_ids = [user_id, AnonymousUser().user_id]
store_user_id = None
for candidate_user_id in candidate_user_ids:
if (
candidate_user_id in self.task_store
and task_id in self.task_store[candidate_user_id]
):
store_user_id = candidate_user_id
break
if store_user_id is None:
return False
upload_task = self.task_store[store_user_id][task_id]
# Find the file task
file_task = upload_task.file_tasks.get(file_path)
if not file_task:
return False
# Lock to prevent race conditions
async with self._get_task_lock(task_id):
# Can only cancel pending or running files
if file_task.status not in [TaskStatus.PENDING, TaskStatus.RUNNING]:
return False
# Mark file as failed (cancelled)
was_running = file_task.status == TaskStatus.RUNNING
file_task.status = TaskStatus.FAILED
file_task.error = "File cancelled by user"
file_task.updated_at = time.time()
# Increment failed counter
upload_task.failed_files += 1
# If it was running, it will be counted in processed_files by the worker's finally block
# If it was pending, we need to count it now since it will never enter the worker
if not was_running:
upload_task.processed_files += 1
upload_task.updated_at = time.time()
# Cancel the running asyncio task if it exists (outside the lock to avoid deadlock)
file_task_key = (task_id, file_path)
if file_task_key in self._file_tasks:
asyncio_task = self._file_tasks[file_task_key]
if not asyncio_task.done():
asyncio_task.cancel()
logger.info(
"Cancelled asyncio task for file",
task_id=task_id,
file_path=file_path,
)
return True
def delete_task(self, user_id: str, task_id: str) -> TaskDeleteResult:
"""Remove a terminal (completed/failed/cancelled) task from memory.
Deletes tasks owned by the calling user or shared anonymous tasks that
are visible to the user (same set exposed by get_all_tasks).
Returns:
TaskDeleteResult.DELETED – task found and removed.
TaskDeleteResult.NOT_FOUND – task ID does not exist for this user.
TaskDeleteResult.IN_PROGRESS – task exists but is not yet terminal.
"""
resolved = self._resolve_upload_task_store(user_id, task_id)
if resolved is None:
return TaskDeleteResult.NOT_FOUND
store_user_id, upload_task = resolved
if upload_task.status not in [
TaskStatus.COMPLETED,
TaskStatus.FAILED,
TaskStatus.CANCELLED,
]:
return TaskDeleteResult.IN_PROGRESS
self._cleanup_upload_temp_files(upload_task, force=True)
del self.task_store[store_user_id][task_id]
self._task_locks.pop(task_id, None)
if not self.task_store[store_user_id]:
del self.task_store[store_user_id]
return TaskDeleteResult.DELETED
def delete_all_terminal_tasks(self, user_id: str) -> list[str]:
"""Remove all completed/failed/cancelled tasks owned by a user.
Only touches the calling user's own store. Shared tasks stored under
the anonymous key are intentionally excluded: they are visible to all
authenticated users, so a bulk clear by one user must not remove them
for everyone else. Those tasks are aged out by cleanup_old_tasks.
Returns the list of deleted task IDs.
"""
if user_id not in self.task_store:
return []
to_delete = [
tid
for tid, t in self.task_store[user_id].items()
if t.status in [TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED]
]
for tid in to_delete:
task = self.task_store[user_id][tid]
self._cleanup_upload_temp_files(task, force=True)
del self.task_store[user_id][tid]
self._task_locks.pop(tid, None)
if not self.task_store.get(user_id):
self.task_store.pop(user_id, None)
return to_delete
def _file_task_for_temp_path(self, upload_task: UploadTask, temp_path: str) -> FileTask | None:
"""Resolve the FileTask for a staged upload temp path."""
file_task = upload_task.file_tasks.get(temp_path)
if file_task is not None:
return file_task
for candidate in upload_task.file_tasks.values():
if candidate.file_path == temp_path:
return candidate
return None
def _is_retryable_local_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
"""True when a staged temp belongs to a failed local RETRYABLE upload.
Local uploads use the staged path as the file_tasks key and
FileTask.file_path. When those diverge, this falls back to matching
FileTask.file_path. Unmapped absolute temps are retained (see
_should_retain_upload_temp) rather than deleted silently.
"""
if not os.path.isabs(temp_path):
return False
file_task = self._file_task_for_temp_path(upload_task, temp_path)
if file_task is None or file_task.status != TaskStatus.FAILED:
return False
metadata = self._infer_failure_metadata(file_task)
return bool(metadata and metadata.get("actionable_by") == "RETRYABLE")
def _is_unsettled_claim_loser_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
"""True when a staged temp belongs to a file skipped for a name another
in-flight file still holds.
Such a skip is provisional: if the file holding the name fails, nothing
was indexed under it and _fail_files_stranded_by turns this file into a
retryable failure. That correction can land after this task has already
finished — the holder may belong to another task entirely — and a retry
then needs the staged source that cleanup would otherwise have deleted.
Retention is bounded the same way a retryable failure's is: whichever
way the holder ends, the temp is reclaimed when the task ages out of the
store (cleanup_old_tasks force-cleans what it evicts).
"""
if not os.path.isabs(temp_path):
return False
file_task = self._file_task_for_temp_path(upload_task, temp_path)
if file_task is None or file_task.status != TaskStatus.SKIPPED:
return False
if (file_task.result or {}).get("reason") != "duplicate_filename":
return False
# Local uploads key file_tasks by the staged path, which is what the
# claim holder was built from.
return filename_claims.is_awaiting_outcome(
claim_holder(upload_task.task_id, file_task.file_path)
)
def _should_retain_upload_temp(self, upload_task: UploadTask, temp_path: str) -> bool:
"""Return True when an upload temp should be kept after processing."""
if self._is_retryable_local_upload_temp(upload_task, temp_path):
return True
if self._is_unsettled_claim_loser_temp(upload_task, temp_path):
return True
if (
os.path.isabs(temp_path)
and self._file_task_for_temp_path(upload_task, temp_path) is None
):
logger.warning(
"Upload temp path has no matching file task; retaining staged file",
temp_path=temp_path,
task_id=upload_task.task_id,
)
return True
return False
def _cleanup_upload_temp_files(self, upload_task: UploadTask, *, force: bool = False) -> None:
"""Remove staged upload temp files that are not retryable.
Keeps temps for failed local uploads classified as RETRYABLE so retry
can reuse the original source path. Use *force* to delete all temps
(e.g. user cancelled the task).
"""
if not getattr(upload_task, "temp_file_paths", None):
return
from utils.file_utils import safe_unlink
retained: list[str] = []
for temp_path in upload_task.temp_file_paths:
if not force and self._should_retain_upload_temp(upload_task, temp_path):
retained.append(temp_path)
continue
safe_unlink(temp_path)
if os.path.exists(temp_path):
retained.append(temp_path)
logger.warning(
"Failed to clean up temp file after processing",
temp_path=temp_path,
)
else:
logger.debug("Cleaned up temp file", temp_path=temp_path)
upload_task.temp_file_paths = retained
async def shutdown(self):
"""Cleanup process pool and cancel all background tasks
Ensures graceful shutdown by:
1. Cancelling the periodic cleanup task
2. Cancelling all running background tasks
3. Waiting for cancellation to complete
4. Force-cleaning staged upload temps for all tracked tasks
5. Shutting down the process pool
"""
logger.info("Shutting down TaskService", background_tasks_count=len(self.background_tasks))
# Cancel the periodic cleanup task
if self._cleanup_task is not None and not self._cleanup_task.done():
self._cleanup_task.cancel()
try:
await self._cleanup_task
except asyncio.CancelledError:
pass
# Cancel all background tasks
for task in self.background_tasks:
if not task.done():
task.cancel()
# Wait for all tasks to complete cancellation
if self.background_tasks:
results = await asyncio.gather(*self.background_tasks, return_exceptions=True)
# Log any unexpected errors (not CancelledError)
for result in results:
if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
logger.warning(
"Background task raised exception during shutdown", error=str(result)
)
for user_tasks in self.task_store.values():
for upload_task in user_tasks.values():
self._cleanup_upload_temp_files(upload_task, force=True)