OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Openrag improved tasks logs backend (#1671)
Rico Furtadob27d45baa811
diff --git a/src/api/tasks.py b/src/api/tasks.pyindex b78b0770d..c5c779ce7 100644--- a/src/api/tasks.py+++ b/src/api/tasks.py@@ -1,9 +1,9 @@from fastapi import Dependsfrom fastapi.responses import JSONResponse-from utils.telemetry import TelemetryClient, Category, MessageId-from dependencies import get_task_service, get_current_user+from dependencies import get_current_user, get_task_servicefrom session_manager import User+from utils.telemetry import Category, MessageId, TelemetryClientasync def task_status(@@ -19,6 +19,19 @@ async def task_status(return JSONResponse(task_status_result)+async def task_status_enhanced(+ task_id: str,+ task_service=Depends(get_task_service),+ user: User = Depends(get_current_user),+):+ """Get the status of a specific task with structured failure metadata."""+ task_status_result = task_service.get_task_status2(user.user_id, task_id)+ if not task_status_result:+ return JSONResponse({"error": "Task not found"}, status_code=404)++ return JSONResponse(task_status_result)++async def all_tasks(task_service=Depends(get_task_service),user: User = Depends(get_current_user),@@ -28,6 +41,15 @@ async def all_tasks(return JSONResponse({"tasks": tasks})+async def all_tasks_enhanced(+ task_service=Depends(get_task_service),+ user: User = Depends(get_current_user),+):+ """Get all tasks with structured failure metadata on failed files."""+ tasks = task_service.get_all_tasks2(user.user_id)+ return JSONResponse({"tasks": tasks})++async def cancel_task(task_id: str,task_service=Depends(get_task_service),@@ -37,9 +59,7 @@ async def cancel_task(success = await task_service.cancel_task(user.user_id, task_id)if not success:await TelemetryClient.send_event(Category.TASK_OPERATIONS, MessageId.ORB_TASK_CANCEL_FAILED)- return JSONResponse(- {"error": "Task not found or cannot be cancelled"}, status_code=400- )+ return JSONResponse({"error": "Task not found or cannot be cancelled"}, status_code=400)await TelemetryClient.send_event(Category.TASK_OPERATIONS, MessageId.ORB_TASK_CANCELLED)return JSONResponse({"status": "cancelled", "task_id": task_id})diff --git a/src/api/v1/documents.py b/src/api/v1/documents.pyindex 5fd3027de..cd4ed381a 100644--- a/src/api/v1/documents.py+++ b/src/api/v1/documents.py@@ -4,23 +4,22 @@ Public API v1 Documents endpoint.Provides document ingestion and management.Uses API key authentication."""-from typing import List, Optionalfrom fastapi import Depends, File, Form, UploadFile-from pydantic import BaseModelfrom fastapi.responses import JSONResponse-from api.documents import delete_documents_by_filename_core+from pydantic import BaseModel+from api.documents import delete_documents_by_filename_corefrom api.router import upload_ingest_router-from utils.logging_config import get_loggerfrom dependencies import (+ get_api_key_user_async,get_document_service,- get_task_service,- get_session_manager,get_langflow_file_service,- get_api_key_user_async,+ get_session_manager,+ get_task_service,)from session_manager import User+from utils.logging_config import get_loggerlogger = get_logger(__name__)@@ -30,10 +29,10 @@ class DeleteDocV1Body(BaseModel):async def ingest_endpoint(- file: List[UploadFile] = File(...),- session_id: Optional[str] = Form(None),- settings: Optional[str] = Form(None),- tweaks: Optional[str] = Form(None),+ file: list[UploadFile] = File(...),+ session_id: str | None = Form(None),+ settings: str | None = Form(None),+ tweaks: str | None = Form(None),replace_duplicates: str = Form("true"),create_filter: str = Form("false"),document_service=Depends(get_document_service),@@ -64,6 +63,26 @@ async def ingest_endpoint()+async def all_tasks_enhanced_endpoint(+ task_service=Depends(get_task_service),+ user: User = Depends(get_api_key_user_async),+):+ """Get all ingestion tasks with structured failure metadata on failed files.++ GET /v1/tasks/enhanced++ Returns the same list as GET /v1/tasks/{task_id} would across all tasks,+ with component, failure_phase, user_facing_message, and actionable_by+ added to any failed file entry whose cause can be classified.++ Note: completed files are omitted from each task's ``files`` dict to+ reduce payload size; use GET /v1/tasks/{task_id}/enhanced for the full+ file list of a specific task.+ """+ tasks = task_service.get_all_tasks2(user.user_id)+ return JSONResponse({"tasks": tasks})++async def task_status_endpoint(task_id: str,task_service=Depends(get_task_service),@@ -76,6 +95,30 @@ async def task_status_endpoint(return JSONResponse(task_status)+async def task_status_enhanced_endpoint(+ task_id: str,+ task_service=Depends(get_task_service),+ user: User = Depends(get_api_key_user_async),+):+ """Get the status of an ingestion task with structured failure metadata.++ GET /v1/tasks/{task_id}/enhanced++ Returns the same baseline task and file status information as+ GET /v1/tasks/{task_id}, and additionally includes component,+ failure_phase, user_facing_message, and actionable_by on any file+ entry whose status is 'failed' and whose failure cause can be+ classified.++ Note: unlike GET /v1/tasks/enhanced, this endpoint includes completed+ files in the task's ``files`` dict.+ """+ task_status = task_service.get_task_status2(user.user_id, task_id)+ if not task_status:+ return JSONResponse({"error": "Task not found"}, status_code=404)+ return JSONResponse(task_status)++async def delete_document_endpoint(body: DeleteDocV1Body,session_manager=Depends(get_session_manager),diff --git a/src/app/routes/internal.py b/src/app/routes/internal.pyindex dd44b3d11..86b13e80e 100644--- a/src/app/routes/internal.py+++ b/src/app/routes/internal.py@@ -78,7 +78,18 @@ def register_internal_routes(app: FastAPI):app.add_api_route("/upload_bucket", upload.upload_bucket, methods=["POST"], tags=["internal"])# Task endpoints+ # Literal sub-paths must be registered before the parameterised /{task_id}+ # so Starlette does not absorb "enhanced" as a task_id value.+ app.add_api_route(+ "/tasks/enhanced", tasks.all_tasks_enhanced, methods=["GET"], tags=["internal"]+ )app.add_api_route("/tasks/{task_id}", tasks.task_status, methods=["GET"], tags=["internal"])+ app.add_api_route(+ "/tasks/{task_id}/enhanced",+ tasks.task_status_enhanced,+ methods=["GET"],+ tags=["internal"],+ )app.add_api_route("/tasks", tasks.all_tasks, methods=["GET"], tags=["internal"])app.add_api_route("/tasks/{task_id}/cancel",diff --git a/src/app/routes/public_v1.py b/src/app/routes/public_v1.pyindex d7e735fb9..557745e2c 100644--- a/src/app/routes/public_v1.py+++ b/src/app/routes/public_v1.py@@ -50,12 +50,26 @@ def register_public_v1_routes(app: FastAPI):methods=["POST"],tags=["public"],)+ # Literal sub-paths must be registered before the parameterised /{task_id}+ # so Starlette does not absorb "enhanced" as a task_id value.+ app.add_api_route(+ "/v1/tasks/enhanced",+ v1_documents.all_tasks_enhanced_endpoint,+ methods=["GET"],+ tags=["public"],+ )app.add_api_route("/v1/tasks/{task_id}",v1_documents.task_status_endpoint,methods=["GET"],tags=["public"],)+ app.add_api_route(+ "/v1/tasks/{task_id}/enhanced",+ v1_documents.task_status_enhanced_endpoint,+ methods=["GET"],+ tags=["public"],+ )app.add_api_route("/v1/documents",v1_documents.delete_document_endpoint,diff --git a/src/services/docling_polling_service.py b/src/services/docling_polling_service.pyindex a8cc83a07..9067db47c 100644--- a/src/services/docling_polling_service.py+++ b/src/services/docling_polling_service.py@@ -9,20 +9,21 @@ conversion (which can be many minutes for large or OCR-heavy documents).import asyncioimport timefrom dataclasses import dataclass-from enum import Enum-from typing import Optional+from enum import StrEnumfrom services.docling_service import (+ DoclingServeError,DoclingService,DoclingStatusSnapshot,DoclingTaskState,+ DoclingTransientError,)from utils.logging_config import get_loggerlogger = get_logger(__name__)-class PollOutcome(str, Enum):+class PollOutcome(StrEnum):SUCCESS = "success"FAILED = "failed"EXPIRED = "expired"@@ -32,8 +33,8 @@ class PollOutcome(str, Enum):@dataclassclass DoclingPollResult:outcome: PollOutcome- detail: Optional[str] = None- last_snapshot: Optional[DoclingStatusSnapshot] = None+ detail: str | None = None+ last_snapshot: DoclingStatusSnapshot | None = Noneelapsed_seconds: float = 0.0@@ -51,9 +52,15 @@ class DoclingPollingService:max_interval: float = 30.0,backoff_factor: float = 1.5,transient_retry_budget: int = 5,+ result_fetch_retry_budget: int = 3,) -> DoclingPollResult:"""Loop on Docling status until terminal or until max_seconds elapses.+ A SUCCESS status is treated as ready only after the result endpoint+ returns a payload with usable ``document.json_content``. This prevents+ handing Langflow a task that Docling accepted but failed to convert+ into a consumable document.+Transient errors (network, 5xx, NOT_FOUND seen briefly before the taskis registered server-side) are absorbed up to ``transient_retry_budget``before being surfaced as failures. The interval grows by@@ -69,7 +76,7 @@ class DoclingPollingService:deadline = start + max_secondsinterval = poll_intervalconsecutive_not_found = 0- last_snapshot: Optional[DoclingStatusSnapshot] = None+ last_snapshot: DoclingStatusSnapshot | None = Nonelogger.debug("Starting Docling polling", task_id=task_id)@@ -81,8 +88,75 @@ class DoclingPollingService:elapsed = time.monotonic() - startif snapshot.state == DoclingTaskState.SUCCESS:+ result_fetch_errors = 0+ while True:+ now = time.monotonic()+ elapsed = now - start+ remaining = deadline - now+ if remaining <= 0:+ detail = (+ "Docling result fetch timed out after SUCCESS status "+ f"({max_seconds}s budget reached)"+ )+ logger.warning(+ "Docling result fetch exceeded max_seconds after SUCCESS status",+ task_id=task_id,+ detail=detail,+ elapsed_seconds=round(elapsed, 2),+ max_seconds=max_seconds,+ )+ return DoclingPollResult(+ outcome=PollOutcome.FAILED,+ detail=detail,+ last_snapshot=snapshot,+ elapsed_seconds=elapsed,+ )+ try:+ await self.docling_service.fetch_task_result(task_id)+ break+ except DoclingTransientError as e:+ result_fetch_errors += 1+ if result_fetch_errors > result_fetch_retry_budget:+ detail = (+ f"Docling result fetch failed after {result_fetch_errors} "+ f"transient errors: {str(e)}"+ )+ logger.warning(+ "Docling result fetch exceeded transient retry budget",+ task_id=task_id,+ detail=detail,+ elapsed_seconds=round(elapsed, 2),+ )+ return DoclingPollResult(+ outcome=PollOutcome.FAILED,+ detail=detail,+ last_snapshot=snapshot,+ elapsed_seconds=elapsed,+ )+ logger.debug(+ "Transient error fetching Docling result, retrying",+ task_id=task_id,+ attempt=result_fetch_errors,+ error=str(e),+ )+ await asyncio.sleep(min(interval, remaining))+ except DoclingServeError as e:+ detail = f"Docling result unavailable after SUCCESS status: {str(e)}"+ logger.warning(+ "Docling task reached SUCCESS but result fetch failed",+ task_id=task_id,+ detail=detail,+ elapsed_seconds=round(elapsed, 2),+ )+ return DoclingPollResult(+ outcome=PollOutcome.FAILED,+ detail=detail,+ last_snapshot=snapshot,+ elapsed_seconds=elapsed,+ )+logger.debug(- "Docling task reached SUCCESS",+ "Docling task reached SUCCESS and result is available",task_id=task_id,elapsed_seconds=round(elapsed, 2),)diff --git a/src/services/docling_service.py b/src/services/docling_service.pyindex 7c1b2db7e..9831e89ab 100644--- a/src/services/docling_service.py+++ b/src/services/docling_service.py@@ -33,6 +33,10 @@ class DoclingServeError(Exception):"""Raised when docling-serve conversion fails."""+class DoclingTransientError(DoclingServeError):+ """Raised for errors that may resolve on retry (network failures, 5xx)."""++class DoclingTaskState(StrEnum):"""Result of a single status check against Docling Serve."""@@ -281,8 +285,12 @@ class DoclingService:try:response = await client.get(url)except httpx.RequestError as e:- raise DoclingServeError(f"Network error fetching docling result: {str(e)}") from e+ raise DoclingTransientError(f"Network error fetching docling result: {str(e)}") from e+ if response.status_code >= 500:+ raise DoclingTransientError(+ f"Docling result fetch failed with HTTP {response.status_code}: {response.text[:300]}"+ )if response.status_code == 404:raise DoclingServeError(f"Docling result not found for task {task_id} (task expired or unknown)"diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 25d197dac..20604ff84 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -7,7 +7,7 @@ import uuidfrom collections.abc import Coroutinefrom typing import Any, TypeVar-from models.tasks import FileTask, TaskStatus, UploadTask+from models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTaskfrom session_manager import AnonymousUserfrom utils.gpu_detection import get_worker_countfrom utils.logging_config import get_logger@@ -578,26 +578,113 @@ class TaskService:exception=str(e),)- def get_task_status(self, user_id: str, task_id: str) -> dict | None:- """Get the status of a specific upload task-- Includes fallback to shared tasks stored under the "anonymous" user key- so default system tasks are visible to all users.- """+ def _resolve_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:+ """Look up a task by ID, falling back to anonymous/shared tasks."""if not task_id:return None-- # Prefer the caller's user_id; otherwise check shared/anonymous tasks- candidate_user_ids = [user_id, AnonymousUser().user_id]-- upload_task = None- for candidate_user_id in candidate_user_ids:+ for candidate_user_id in [user_id, AnonymousUser().user_id]:if (candidate_user_id in self.task_storeand task_id in self.task_store[candidate_user_id]):- upload_task = self.task_store[candidate_user_id][task_id]- break+ return self.task_store[candidate_user_id][task_id]+ return None++ def _serialize_file_task(self, file_task: FileTask) -> dict:+ """Serialize a FileTask to the standard dict shape."""+ return {+ "status": file_task.status.value,+ "result": file_task.result,+ "error": file_task.error,+ "retry_count": file_task.retry_count,+ "created_at": file_task.created_at,+ "updated_at": file_task.updated_at,+ "duration_seconds": file_task.duration_seconds,+ "filename": file_task.filename,+ "phase": file_task.phase.value,+ "docling_status": file_task.docling_status.value,+ "docling_task_id": file_task.docling_task_id,+ }++ def _infer_failure_metadata(self, file_task: FileTask) -> dict | None:+ """Infer structured failure metadata for a failed FileTask.++ Returns a dict with component, failure_phase, user_facing_message, and+ actionable_by when the failure can be classified, or None when the cause+ is unknown and no fields should be emitted.++ Priority order: docling_status enum first (stable), error string patterns+ second (fallback for edge cases like polling timeout).+ """+ docling_status = file_task.docling_status+ phase = file_task.phase+ error = file_task.error or ""++ if docling_status == DoclingPhaseStatus.FAILED:+ return {+ "component": "docling",+ "failure_phase": "parsing",+ "user_facing_message": (+ "The file could not be processed into readable document content."+ ),+ "actionable_by": "USER_ACTIONABLE",+ }++ if docling_status == DoclingPhaseStatus.EXPIRED:+ return {+ "component": "docling",+ "failure_phase": "parsing",+ "user_facing_message": (+ "The document processing result could not be found. "+ "The task may have expired. Please retry ingestion."+ ),+ "actionable_by": "RETRYABLE",+ }++ if phase == IngestionPhase.DOCLING and "Docling conversion did not complete" in error:+ return {+ "component": "docling",+ "failure_phase": "parsing",+ "user_facing_message": "Document processing timed out. Please retry ingestion.",+ "actionable_by": "RETRYABLE",+ }++ if phase == IngestionPhase.DOCLING and docling_status == DoclingPhaseStatus.PROCESSING:+ return {+ "component": "docling",+ "failure_phase": "parsing",+ "user_facing_message": "Document processing timed out. Please retry ingestion.",+ "actionable_by": "RETRYABLE",+ }++ if "already exists" in error:+ return {+ "component": "openrag",+ "failure_phase": "file_validation",+ "user_facing_message": "A file with this name already exists.",+ "actionable_by": "USER_ACTIONABLE",+ }++ if phase == IngestionPhase.LANGFLOW:+ return {+ "component": "langflow",+ "failure_phase": "unknown",+ "user_facing_message": (+ "Ingestion failed unexpectedly. Please retry. "+ "If it fails again, contact your administrator."+ ),+ "actionable_by": "RETRYABLE",+ }++ return None++ def get_task_status(self, user_id: str, task_id: str) -> dict | None:+ """Get the status of a specific upload task++ Includes fallback to shared tasks stored under the "anonymous" user key+ so default system tasks are visible to all users.+ """+ upload_task = self._resolve_upload_task(user_id, task_id)if upload_task is None:return None@@ -607,19 +694,7 @@ class TaskService:pending_files_count = 0for file_path, file_task in upload_task.file_tasks.items():- file_statuses[file_path] = {- "status": file_task.status.value,- "result": file_task.result,- "error": file_task.error,- "retry_count": file_task.retry_count,- "created_at": file_task.created_at,- "updated_at": file_task.updated_at,- "duration_seconds": file_task.duration_seconds,- "filename": file_task.filename,- "phase": file_task.phase.value,- "docling_status": file_task.docling_status.value,- "docling_task_id": file_task.docling_task_id,- }+ file_statuses[file_path] = self._serialize_file_task(file_task)# Count running and pending filesif file_task.status.value == "running":@@ -642,6 +717,106 @@ class TaskService:"files": file_statuses,}+ def get_task_status2(self, user_id: str, task_id: str) -> dict | None:+ """Get the status of a specific upload task with structured failure metadata.++ Identical to get_task_status but enriches failed file entries with+ component, failure_phase, user_facing_message, and actionable_by fields+ when the failure cause can be classified.+ """+ upload_task = self._resolve_upload_task(user_id, task_id)++ if upload_task is None:+ return None++ file_statuses = {}+ running_files_count = 0+ pending_files_count = 0++ for file_path, file_task in upload_task.file_tasks.items():+ entry = self._serialize_file_task(file_task)+ if file_task.status == TaskStatus.FAILED:+ metadata = self._infer_failure_metadata(file_task)+ if metadata:+ entry.update(metadata)+ file_statuses[file_path] = entry++ if file_task.status == TaskStatus.RUNNING:+ running_files_count += 1+ elif file_task.status == TaskStatus.PENDING:+ pending_files_count += 1++ return {+ "task_id": upload_task.task_id,+ "status": upload_task.status.value,+ "total_files": upload_task.total_files,+ "processed_files": upload_task.processed_files,+ "successful_files": upload_task.successful_files,+ "failed_files": upload_task.failed_files,+ "running_files": running_files_count,+ "pending_files": pending_files_count,+ "created_at": upload_task.created_at,+ "updated_at": upload_task.updated_at,+ "duration_seconds": upload_task.duration_seconds,+ "files": file_statuses,+ }++ def get_all_tasks2(self, user_id: str) -> list:+ """Get all tasks for a user with structured failure metadata on failed files.++ Identical to get_all_tasks but enriches failed file entries with+ component, failure_phase, user_facing_message, and actionable_by fields+ when the failure cause can be classified.+ """+ tasks_by_id = {}++ def add_tasks_from_store(store_user_id):+ if store_user_id not in self.task_store:+ return+ for task_id, upload_task in self.task_store[store_user_id].items():+ if task_id in tasks_by_id:+ continue++ running_files_count = 0+ pending_files_count = 0+ file_statuses = {}++ for file_path, file_task in upload_task.file_tasks.items():+ if file_task.status != TaskStatus.COMPLETED:+ entry = self._serialize_file_task(file_task)+ if file_task.status == TaskStatus.FAILED:+ metadata = self._infer_failure_metadata(file_task)+ if metadata:+ entry.update(metadata)+ file_statuses[file_path] = entry++ if file_task.status == TaskStatus.RUNNING:+ running_files_count += 1+ elif file_task.status == TaskStatus.PENDING:+ pending_files_count += 1++ tasks_by_id[task_id] = {+ "task_id": upload_task.task_id,+ "status": upload_task.status.value,+ "total_files": upload_task.total_files,+ "processed_files": upload_task.processed_files,+ "successful_files": upload_task.successful_files,+ "failed_files": upload_task.failed_files,+ "running_files": running_files_count,+ "pending_files": pending_files_count,+ "created_at": upload_task.created_at,+ "updated_at": upload_task.updated_at,+ "duration_seconds": upload_task.duration_seconds,+ "files": file_statuses,+ }++ add_tasks_from_store(user_id)+ add_tasks_from_store(AnonymousUser().user_id)++ tasks = list(tasks_by_id.values())+ tasks.sort(key=lambda x: x["created_at"], reverse=True)+ return tasks+def get_all_tasks(self, user_id: str) -> list:"""Get all tasks for a userdiff --git a/tests/unit/test_docling_polling_service.py b/tests/unit/test_docling_polling_service.pyindex 803e01bb8..300a9a3b4 100644--- a/tests/unit/test_docling_polling_service.py+++ b/tests/unit/test_docling_polling_service.py@@ -6,14 +6,20 @@ Docling state without ever invoking Langflow, so that Langflow executionslots are reserved for chunking / embedding / indexing only."""-import pytestfrom unittest.mock import AsyncMock, patch+import pytest+from services.docling_polling_service import (DoclingPollingService,PollOutcome,)-from services.docling_service import DoclingStatusSnapshot, DoclingTaskState+from services.docling_service import (+ DoclingServeError,+ DoclingStatusSnapshot,+ DoclingTaskState,+ DoclingTransientError,+)def _snap(state: DoclingTaskState, detail: str | None = None) -> DoclingStatusSnapshot:@@ -42,6 +48,7 @@ def no_sleep():@pytest.mark.asyncioasync def test_returns_success_immediately_when_already_done(polling_service, mock_docling_service):mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}result = await polling_service.poll_until_ready(task_id="t1", poll_interval=1.0, max_seconds=10.0@@ -49,6 +56,7 @@ async def test_returns_success_immediately_when_already_done(polling_service, moassert result.outcome == PollOutcome.SUCCESSassert mock_docling_service.check_task_status.call_count == 1+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")@pytest.mark.asyncio@@ -62,6 +70,8 @@ async def test_loops_through_processing_then_success(_snap(DoclingTaskState.SUCCESS),]+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}+result = await polling_service.poll_until_ready(task_id="t1", poll_interval=1.0, max_seconds=60.0)@@ -69,6 +79,23 @@ async def test_loops_through_processing_then_success(assert result.outcome == PollOutcome.SUCCESSassert mock_docling_service.check_task_status.call_count == 4assert no_sleep.call_count == 3+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")+++async def test_success_status_requires_fetchable_result(polling_service, mock_docling_service):+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)+ mock_docling_service.fetch_task_result.side_effect = DoclingServeError(+ "missing document.json_content"+ )++ result = await polling_service.poll_until_ready(+ task_id="t1", poll_interval=1.0, max_seconds=10.0+ )++ assert result.outcome == PollOutcome.FAILED+ assert "missing document.json_content" in (result.detail or "")+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")@pytest.mark.asyncio@@ -96,6 +123,7 @@ async def test_tolerates_brief_not_found_then_succeeds(_snap(DoclingTaskState.PROCESSING),_snap(DoclingTaskState.SUCCESS),]+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}result = await polling_service.poll_until_ready(task_id="t1",@@ -105,6 +133,7 @@ async def test_tolerates_brief_not_found_then_succeeds()assert result.outcome == PollOutcome.SUCCESS+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")@pytest.mark.asyncio@@ -170,6 +199,7 @@ async def test_backoff_grows_interval_up_to_cap(polling_service, mock_docling_se_snap(DoclingTaskState.PROCESSING),_snap(DoclingTaskState.SUCCESS),]+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}await polling_service.poll_until_ready(task_id="t1",@@ -184,3 +214,67 @@ async def test_backoff_grows_interval_up_to_cap(polling_service, mock_docling_se# see the raw progression.sleeps = [call.args[0] for call in no_sleep.call_args_list]assert sleeps == [1.0, 2.0, 4.0, 4.0]+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")+++async def test_transient_result_fetch_error_retries_then_succeeds(+ polling_service, mock_docling_service, no_sleep+):+ """A transient network error on fetch_task_result is retried, not failed immediately."""+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)+ mock_docling_service.fetch_task_result.side_effect = [+ DoclingTransientError("connection reset"),+ DoclingTransientError("connection reset"),+ {"body": "ok"},+ ]++ result = await polling_service.poll_until_ready(+ task_id="t1",+ poll_interval=1.0,+ max_seconds=60.0,+ result_fetch_retry_budget=3,+ )++ assert result.outcome == PollOutcome.SUCCESS+ assert mock_docling_service.fetch_task_result.await_count == 3+++async def test_transient_result_fetch_error_exceeds_budget_returns_failed(+ polling_service, mock_docling_service, no_sleep+):+ """Exhausting result_fetch_retry_budget on transient errors yields FAILED."""+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)+ mock_docling_service.fetch_task_result.side_effect = DoclingTransientError("timeout")++ result = await polling_service.poll_until_ready(+ task_id="t1",+ poll_interval=1.0,+ max_seconds=60.0,+ result_fetch_retry_budget=2,+ )++ assert result.outcome == PollOutcome.FAILED+ assert "transient" in (result.detail or "").lower()+ assert mock_docling_service.fetch_task_result.await_count == 3 # budget + 1+++async def test_permanent_result_fetch_error_fails_immediately(+ polling_service, mock_docling_service+):+ """A non-transient DoclingServeError (e.g. 404) is not retried."""+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)+ mock_docling_service.fetch_task_result.side_effect = DoclingServeError("task expired")++ result = await polling_service.poll_until_ready(+ task_id="t1",+ poll_interval=1.0,+ max_seconds=60.0,+ result_fetch_retry_budget=3,+ )++ assert result.outcome == PollOutcome.FAILED+ assert "task expired" in (result.detail or "")+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.pynew file mode 100644index 000000000..01b54321b--- /dev/null+++ b/tests/unit/test_task_service_get_task_status2.py@@ -0,0 +1,489 @@+"""+Unit tests for TaskService.get_task_status2 and its supporting helpers.++Tests verify that:+- Failed file entries receive structured failure metadata (component,+ failure_phase, user_facing_message, actionable_by) when the cause is known.+- Non-failed entries (completed, running, pending) are never decorated.+- Unknown failures produce no extra fields.+- get_task_status (the existing method) is unaffected by the refactor.+"""++from unittest.mock import Mock++import pytest++from models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask+from services.task_service import TaskService++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def task_service():+ return TaskService(document_service=Mock(), ingestion_timeout=2)+++def _make_upload_task(task_id: str, file_tasks: dict[str, FileTask]) -> UploadTask:+ return UploadTask(task_id=task_id, total_files=len(file_tasks), file_tasks=file_tasks)+++def _make_file_task(+ *,+ file_path: str = "test.pdf",+ status: TaskStatus = TaskStatus.FAILED,+ phase: IngestionPhase = IngestionPhase.DOCLING,+ docling_status: DoclingPhaseStatus = DoclingPhaseStatus.PENDING,+ error: str | None = None,+ filename: str | None = "test.pdf",+) -> FileTask:+ ft = FileTask(file_path=file_path, filename=filename)+ ft.status = status+ ft.phase = phase+ ft.docling_status = docling_status+ ft.error = error+ return ft+++def _store_task(task_service: TaskService, user_id: str, upload_task: UploadTask) -> None:+ task_service.task_store.setdefault(user_id, {})[upload_task.task_id] = upload_task+++# ---------------------------------------------------------------------------+# _infer_failure_metadata+# ---------------------------------------------------------------------------+++class TestInferFailureMetadata:+ def test_docling_failed_status(self, task_service):+ ft = _make_file_task(docling_status=DoclingPhaseStatus.FAILED)+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "docling"+ assert meta["failure_phase"] == "parsing"+ assert meta["actionable_by"] == "USER_ACTIONABLE"+ assert meta["user_facing_message"]++ def test_docling_expired_status(self, task_service):+ ft = _make_file_task(docling_status=DoclingPhaseStatus.EXPIRED)+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "docling"+ assert meta["failure_phase"] == "parsing"+ assert meta["actionable_by"] == "RETRYABLE"+ assert meta["user_facing_message"]++ def test_docling_timeout_via_error_string(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PENDING,+ error="Docling conversion did not complete (timeout): polling exceeded 300s",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "docling"+ assert meta["failure_phase"] == "parsing"+ assert meta["actionable_by"] == "RETRYABLE"++ def test_docling_phase_still_processing(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PROCESSING,+ error="some generic failure",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "docling"+ assert meta["failure_phase"] == "parsing"+ assert meta["actionable_by"] == "RETRYABLE"++ def test_langflow_phase_failure(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error="Langflow run timed out",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "langflow"+ assert meta["failure_phase"] == "unknown"+ assert meta["actionable_by"] == "RETRYABLE"++ def test_duplicate_file_error(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PENDING,+ error="File with name 'report.pdf' already exists",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "openrag"+ assert meta["failure_phase"] == "file_validation"+ assert meta["actionable_by"] == "USER_ACTIONABLE"++ def test_langflow_duplicate_file_error_prefers_file_validation(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error="File with name 'report.pdf' already exists",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is not None+ assert meta["component"] == "openrag"+ assert meta["failure_phase"] == "file_validation"+ assert meta["actionable_by"] == "USER_ACTIONABLE"++ def test_unknown_failure_returns_none(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PENDING,+ error="some completely unknown error",+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is None++ def test_no_error_string_unknown_phase(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PENDING,+ error=None,+ )+ meta = task_service._infer_failure_metadata(ft)+ assert meta is None+++# ---------------------------------------------------------------------------+# get_task_status2 — task not found+# ---------------------------------------------------------------------------+++class TestGetTaskStatus2NotFound:+ def test_missing_task_id_returns_none(self, task_service):+ assert task_service.get_task_status2("user1", "") is None++ def test_unknown_task_id_returns_none(self, task_service):+ assert task_service.get_task_status2("user1", "does-not-exist") is None+++# ---------------------------------------------------------------------------+# get_task_status2 — failure metadata on failed files+# ---------------------------------------------------------------------------+++class TestGetTaskStatus2FailureMetadata:+ def test_docling_failed_adds_metadata(self, task_service):+ ft = _make_file_task(+ docling_status=DoclingPhaseStatus.FAILED,+ error="Docling conversion did not complete (failed): Docling reported failure",+ )+ task = _make_upload_task("t1", {"test.pdf": ft})+ task.status = TaskStatus.FAILED+ task.failed_files = 1+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t1")+ file_entry = result["files"]["test.pdf"]++ assert file_entry["status"] == "failed"+ assert file_entry["component"] == "docling"+ assert file_entry["failure_phase"] == "parsing"+ assert file_entry["actionable_by"] == "USER_ACTIONABLE"+ assert file_entry["user_facing_message"]++ def test_docling_expired_adds_metadata(self, task_service):+ ft = _make_file_task(docling_status=DoclingPhaseStatus.EXPIRED)+ task = _make_upload_task("t2", {"test.pdf": ft})+ task.status = TaskStatus.FAILED+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t2")+ file_entry = result["files"]["test.pdf"]++ assert file_entry["component"] == "docling"+ assert file_entry["failure_phase"] == "parsing"+ assert file_entry["actionable_by"] == "RETRYABLE"++ def test_langflow_failure_adds_metadata(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.LANGFLOW,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error="Langflow run failed",+ )+ task = _make_upload_task("t3", {"doc.pdf": ft})+ task.status = TaskStatus.FAILED+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t3")+ file_entry = result["files"]["doc.pdf"]++ assert file_entry["component"] == "langflow"+ assert file_entry["failure_phase"] == "unknown"+ assert file_entry["actionable_by"] == "RETRYABLE"++ def test_duplicate_file_failure_adds_metadata(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PENDING,+ error="File with name 'report.pdf' already exists",+ )+ task = _make_upload_task("t4", {"report.pdf": ft})+ task.status = TaskStatus.FAILED+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t4")+ file_entry = result["files"]["report.pdf"]++ assert file_entry["component"] == "openrag"+ assert file_entry["failure_phase"] == "file_validation"+ assert file_entry["actionable_by"] == "USER_ACTIONABLE"++ def test_unknown_failure_adds_no_metadata_fields(self, task_service):+ ft = _make_file_task(+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PENDING,+ error="some unrecognised error",+ )+ task = _make_upload_task("t5", {"doc.pdf": ft})+ task.status = TaskStatus.FAILED+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t5")+ file_entry = result["files"]["doc.pdf"]++ assert "component" not in file_entry+ assert "failure_phase" not in file_entry+ assert "user_facing_message" not in file_entry+ assert "actionable_by" not in file_entry+++# ---------------------------------------------------------------------------+# get_task_status2 — non-failed files get no metadata+# ---------------------------------------------------------------------------+++class TestGetTaskStatus2NonFailedFiles:+ def test_completed_file_has_no_metadata(self, task_service):+ ft = _make_file_task(status=TaskStatus.COMPLETED, phase=IngestionPhase.COMPLETE)+ ft.docling_status = DoclingPhaseStatus.SUCCESS+ ft.error = None+ task = _make_upload_task("t6", {"doc.pdf": ft})+ task.status = TaskStatus.COMPLETED+ task.successful_files = 1+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t6")+ file_entry = result["files"]["doc.pdf"]++ assert "component" not in file_entry+ assert "failure_phase" not in file_entry+ assert "user_facing_message" not in file_entry+ assert "actionable_by" not in file_entry++ def test_running_file_has_no_metadata(self, task_service):+ ft = _make_file_task(+ status=TaskStatus.RUNNING,+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PROCESSING,+ error=None,+ )+ task = _make_upload_task("t7", {"doc.pdf": ft})+ task.status = TaskStatus.RUNNING+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t7")+ file_entry = result["files"]["doc.pdf"]++ assert "component" not in file_entry+ assert "failure_phase" not in file_entry++ def test_pending_file_has_no_metadata(self, task_service):+ ft = _make_file_task(status=TaskStatus.PENDING, error=None)+ task = _make_upload_task("t8", {"doc.pdf": ft})+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t8")+ file_entry = result["files"]["doc.pdf"]++ assert "component" not in file_entry+ assert "failure_phase" not in file_entry+++# ---------------------------------------------------------------------------+# get_task_status2 — mixed files+# ---------------------------------------------------------------------------+++class TestGetTaskStatus2Mixed:+ def test_only_failed_files_get_metadata(self, task_service):+ failed_ft = _make_file_task(+ file_path="bad.pdf",+ filename="bad.pdf",+ docling_status=DoclingPhaseStatus.FAILED,+ error="Docling conversion did not complete (failed): Docling reported failure",+ )+ ok_ft = _make_file_task(+ file_path="good.pdf",+ filename="good.pdf",+ status=TaskStatus.COMPLETED,+ phase=IngestionPhase.COMPLETE,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error=None,+ )+ running_ft = _make_file_task(+ file_path="wip.pdf",+ filename="wip.pdf",+ status=TaskStatus.RUNNING,+ phase=IngestionPhase.DOCLING,+ docling_status=DoclingPhaseStatus.PROCESSING,+ error=None,+ )+ task = _make_upload_task(+ "t9", {"bad.pdf": failed_ft, "good.pdf": ok_ft, "wip.pdf": running_ft}+ )+ task.status = TaskStatus.RUNNING+ task.successful_files = 1+ task.failed_files = 1+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t9")++ bad = result["files"]["bad.pdf"]+ assert bad["component"] == "docling"+ assert bad["failure_phase"] == "parsing"++ good = result["files"]["good.pdf"]+ assert "component" not in good++ wip = result["files"]["wip.pdf"]+ assert "component" not in wip++ def test_top_level_fields_present(self, task_service):+ ft = _make_file_task(docling_status=DoclingPhaseStatus.FAILED)+ task = _make_upload_task("t10", {"doc.pdf": ft})+ task.status = TaskStatus.FAILED+ task.failed_files = 1+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status2("user1", "t10")++ for key in (+ "task_id",+ "status",+ "total_files",+ "processed_files",+ "successful_files",+ "failed_files",+ "running_files",+ "pending_files",+ "created_at",+ "updated_at",+ "duration_seconds",+ "files",+ ):+ assert key in result, f"missing key: {key}"+++# ---------------------------------------------------------------------------+# get_task_status2 — anonymous user fallback+# ---------------------------------------------------------------------------+++class TestGetTaskStatus2AnonymousFallback:+ def test_resolves_anonymous_task(self, task_service):+ from session_manager import AnonymousUser++ ft = _make_file_task(docling_status=DoclingPhaseStatus.FAILED)+ task = _make_upload_task("anon-task", {"doc.pdf": ft})+ task.status = TaskStatus.FAILED+ anon_id = AnonymousUser().user_id+ task_service.task_store[anon_id] = {"anon-task": task}++ result = task_service.get_task_status2("some-other-user", "anon-task")+ assert result is not None+ assert result["task_id"] == "anon-task"+ assert result["files"]["doc.pdf"]["component"] == "docling"+++# ---------------------------------------------------------------------------+# Regression: get_task_status remains unchanged+# ---------------------------------------------------------------------------+++class TestGetTaskStatusRegression:+ def test_no_metadata_fields_in_original_method(self, task_service):+ ft = _make_file_task(+ docling_status=DoclingPhaseStatus.FAILED,+ error="Docling conversion did not complete (failed): Docling reported failure",+ )+ task = _make_upload_task("reg1", {"doc.pdf": ft})+ task.status = TaskStatus.FAILED+ task.failed_files = 1+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status("user1", "reg1")+ file_entry = result["files"]["doc.pdf"]++ assert file_entry["status"] == "failed"+ assert "component" not in file_entry+ assert "failure_phase" not in file_entry+ assert "user_facing_message" not in file_entry+ assert "actionable_by" not in file_entry++ def test_original_method_baseline_shape_unchanged(self, task_service):+ ft = _make_file_task(+ status=TaskStatus.COMPLETED,+ phase=IngestionPhase.COMPLETE,+ docling_status=DoclingPhaseStatus.SUCCESS,+ error=None,+ )+ task = _make_upload_task("reg2", {"doc.pdf": ft})+ task.status = TaskStatus.COMPLETED+ task.successful_files = 1+ _store_task(task_service, "user1", task)++ result = task_service.get_task_status("user1", "reg2")+ file_entry = result["files"]["doc.pdf"]++ expected_keys = {+ "status",+ "result",+ "error",+ "retry_count",+ "created_at",+ "updated_at",+ "duration_seconds",+ "filename",+ "phase",+ "docling_status",+ "docling_task_id",+ }+ assert set(file_entry.keys()) == expected_keys++ def test_both_methods_return_same_baseline_for_failed_file(self, task_service):+ """get_task_status2 must include all fields from get_task_status plus any extras."""+ ft = _make_file_task(+ docling_status=DoclingPhaseStatus.FAILED,+ error="Docling conversion did not complete (failed): Docling reported failure",+ )+ task = _make_upload_task("reg3", {"doc.pdf": ft})+ task.status = TaskStatus.FAILED+ task.failed_files = 1+ _store_task(task_service, "user1", task)++ v1 = task_service.get_task_status("user1", "reg3")+ v2 = task_service.get_task_status2("user1", "reg3")++ # All v1 top-level keys must appear in v2+ for key in v1:+ assert key in v2, f"get_task_status2 missing top-level key: {key}"+ if key != "files":+ assert v1[key] == v2[key], f"top-level key '{key}' differs"++ # All baseline file entry keys must survive in get_task_status2+ v1_file = v1["files"]["doc.pdf"]+ v2_file = v2["files"]["doc.pdf"]+ for key in v1_file:+ assert key in v2_file, f"get_task_status2 file entry missing key: {key}"+ assert v1_file[key] == v2_file[key], f"file entry key '{key}' differs"