Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Openrag improved tasks logs backend (#1671)

Rico Furtadob27d45baa811
diff --git a/src/api/tasks.py b/src/api/tasks.py
index b78b0770d..c5c779ce7 100644
--- a/src/api/tasks.py
+++ b/src/api/tasks.py
@@ -1,9 +1,9 @@
from fastapi import Depends
from fastapi.responses import JSONResponse
-from utils.telemetry import TelemetryClient, Category, MessageId
-from dependencies import get_task_service, get_current_user
+from dependencies import get_current_user, get_task_service
from session_manager import User
+from utils.telemetry import Category, MessageId, TelemetryClient
async def task_status(
@@ -19,6 +19,19 @@ async def task_status(
return JSONResponse(task_status_result)
+async def task_status_enhanced(
+ task_id: str,
+ task_service=Depends(get_task_service),
+ user: User = Depends(get_current_user),
+):
+ """Get the status of a specific task with structured failure metadata."""
+ task_status_result = task_service.get_task_status2(user.user_id, task_id)
+ if not task_status_result:
+ return JSONResponse({"error": "Task not found"}, status_code=404)
+
+ return JSONResponse(task_status_result)
+
+
async def all_tasks(
task_service=Depends(get_task_service),
user: User = Depends(get_current_user),
@@ -28,6 +41,15 @@ async def all_tasks(
return JSONResponse({"tasks": tasks})
+async def all_tasks_enhanced(
+ task_service=Depends(get_task_service),
+ user: User = Depends(get_current_user),
+):
+ """Get all tasks with structured failure metadata on failed files."""
+ tasks = task_service.get_all_tasks2(user.user_id)
+ return JSONResponse({"tasks": tasks})
+
+
async def cancel_task(
task_id: str,
task_service=Depends(get_task_service),
@@ -37,9 +59,7 @@ async def cancel_task(
success = await task_service.cancel_task(user.user_id, task_id)
if not success:
await TelemetryClient.send_event(Category.TASK_OPERATIONS, MessageId.ORB_TASK_CANCEL_FAILED)
- return JSONResponse(
- {"error": "Task not found or cannot be cancelled"}, status_code=400
- )
+ return JSONResponse({"error": "Task not found or cannot be cancelled"}, status_code=400)
await TelemetryClient.send_event(Category.TASK_OPERATIONS, MessageId.ORB_TASK_CANCELLED)
return JSONResponse({"status": "cancelled", "task_id": task_id})
diff --git a/src/api/v1/documents.py b/src/api/v1/documents.py
index 5fd3027de..cd4ed381a 100644
--- a/src/api/v1/documents.py
+++ b/src/api/v1/documents.py
@@ -4,23 +4,22 @@ Public API v1 Documents endpoint.
Provides document ingestion and management.
Uses API key authentication.
"""
-from typing import List, Optional
from fastapi import Depends, File, Form, UploadFile
-from pydantic import BaseModel
from fastapi.responses import JSONResponse
-from api.documents import delete_documents_by_filename_core
+from pydantic import BaseModel
+from api.documents import delete_documents_by_filename_core
from api.router import upload_ingest_router
-from utils.logging_config import get_logger
from dependencies import (
+ get_api_key_user_async,
get_document_service,
- get_task_service,
- get_session_manager,
get_langflow_file_service,
- get_api_key_user_async,
+ get_session_manager,
+ get_task_service,
)
from session_manager import User
+from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -30,10 +29,10 @@ class DeleteDocV1Body(BaseModel):
async def ingest_endpoint(
- file: List[UploadFile] = File(...),
- session_id: Optional[str] = Form(None),
- settings: Optional[str] = Form(None),
- tweaks: Optional[str] = Form(None),
+ file: list[UploadFile] = File(...),
+ session_id: str | None = Form(None),
+ settings: str | None = Form(None),
+ tweaks: str | None = Form(None),
replace_duplicates: str = Form("true"),
create_filter: str = Form("false"),
document_service=Depends(get_document_service),
@@ -64,6 +63,26 @@ async def ingest_endpoint(
)
+async def all_tasks_enhanced_endpoint(
+ task_service=Depends(get_task_service),
+ user: User = Depends(get_api_key_user_async),
+):
+ """Get all ingestion tasks with structured failure metadata on failed files.
+
+ GET /v1/tasks/enhanced
+
+ Returns the same list as GET /v1/tasks/{task_id} would across all tasks,
+ with component, failure_phase, user_facing_message, and actionable_by
+ added to any failed file entry whose cause can be classified.
+
+ Note: completed files are omitted from each task's ``files`` dict to
+ reduce payload size; use GET /v1/tasks/{task_id}/enhanced for the full
+ file list of a specific task.
+ """
+ tasks = task_service.get_all_tasks2(user.user_id)
+ return JSONResponse({"tasks": tasks})
+
+
async def task_status_endpoint(
task_id: str,
task_service=Depends(get_task_service),
@@ -76,6 +95,30 @@ async def task_status_endpoint(
return JSONResponse(task_status)
+async def task_status_enhanced_endpoint(
+ task_id: str,
+ task_service=Depends(get_task_service),
+ user: User = Depends(get_api_key_user_async),
+):
+ """Get the status of an ingestion task with structured failure metadata.
+
+ GET /v1/tasks/{task_id}/enhanced
+
+ Returns the same baseline task and file status information as
+ GET /v1/tasks/{task_id}, and additionally includes component,
+ failure_phase, user_facing_message, and actionable_by on any file
+ entry whose status is 'failed' and whose failure cause can be
+ classified.
+
+ Note: unlike GET /v1/tasks/enhanced, this endpoint includes completed
+ files in the task's ``files`` dict.
+ """
+ task_status = task_service.get_task_status2(user.user_id, task_id)
+ if not task_status:
+ return JSONResponse({"error": "Task not found"}, status_code=404)
+ return JSONResponse(task_status)
+
+
async def delete_document_endpoint(
body: DeleteDocV1Body,
session_manager=Depends(get_session_manager),
diff --git a/src/app/routes/internal.py b/src/app/routes/internal.py
index dd44b3d11..86b13e80e 100644
--- a/src/app/routes/internal.py
+++ b/src/app/routes/internal.py
@@ -78,7 +78,18 @@ def register_internal_routes(app: FastAPI):
app.add_api_route("/upload_bucket", upload.upload_bucket, methods=["POST"], tags=["internal"])
# Task endpoints
+ # Literal sub-paths must be registered before the parameterised /{task_id}
+ # so Starlette does not absorb "enhanced" as a task_id value.
+ app.add_api_route(
+ "/tasks/enhanced", tasks.all_tasks_enhanced, methods=["GET"], tags=["internal"]
+ )
app.add_api_route("/tasks/{task_id}", tasks.task_status, methods=["GET"], tags=["internal"])
+ app.add_api_route(
+ "/tasks/{task_id}/enhanced",
+ tasks.task_status_enhanced,
+ methods=["GET"],
+ tags=["internal"],
+ )
app.add_api_route("/tasks", tasks.all_tasks, methods=["GET"], tags=["internal"])
app.add_api_route(
"/tasks/{task_id}/cancel",
diff --git a/src/app/routes/public_v1.py b/src/app/routes/public_v1.py
index d7e735fb9..557745e2c 100644
--- a/src/app/routes/public_v1.py
+++ b/src/app/routes/public_v1.py
@@ -50,12 +50,26 @@ def register_public_v1_routes(app: FastAPI):
methods=["POST"],
tags=["public"],
)
+ # Literal sub-paths must be registered before the parameterised /{task_id}
+ # so Starlette does not absorb "enhanced" as a task_id value.
+ app.add_api_route(
+ "/v1/tasks/enhanced",
+ v1_documents.all_tasks_enhanced_endpoint,
+ methods=["GET"],
+ tags=["public"],
+ )
app.add_api_route(
"/v1/tasks/{task_id}",
v1_documents.task_status_endpoint,
methods=["GET"],
tags=["public"],
)
+ app.add_api_route(
+ "/v1/tasks/{task_id}/enhanced",
+ v1_documents.task_status_enhanced_endpoint,
+ methods=["GET"],
+ tags=["public"],
+ )
app.add_api_route(
"/v1/documents",
v1_documents.delete_document_endpoint,
diff --git a/src/services/docling_polling_service.py b/src/services/docling_polling_service.py
index a8cc83a07..9067db47c 100644
--- a/src/services/docling_polling_service.py
+++ b/src/services/docling_polling_service.py
@@ -9,20 +9,21 @@ conversion (which can be many minutes for large or OCR-heavy documents).
import asyncio
import time
from dataclasses import dataclass
-from enum import Enum
-from typing import Optional
+from enum import StrEnum
from services.docling_service import (
+ DoclingServeError,
DoclingService,
DoclingStatusSnapshot,
DoclingTaskState,
+ DoclingTransientError,
)
from utils.logging_config import get_logger
logger = get_logger(__name__)
-class PollOutcome(str, Enum):
+class PollOutcome(StrEnum):
SUCCESS = "success"
FAILED = "failed"
EXPIRED = "expired"
@@ -32,8 +33,8 @@ class PollOutcome(str, Enum):
@dataclass
class DoclingPollResult:
outcome: PollOutcome
- detail: Optional[str] = None
- last_snapshot: Optional[DoclingStatusSnapshot] = None
+ detail: str | None = None
+ last_snapshot: DoclingStatusSnapshot | None = None
elapsed_seconds: float = 0.0
@@ -51,9 +52,15 @@ class DoclingPollingService:
max_interval: float = 30.0,
backoff_factor: float = 1.5,
transient_retry_budget: int = 5,
+ result_fetch_retry_budget: int = 3,
) -> DoclingPollResult:
"""Loop on Docling status until terminal or until max_seconds elapses.
+ A SUCCESS status is treated as ready only after the result endpoint
+ returns a payload with usable ``document.json_content``. This prevents
+ handing Langflow a task that Docling accepted but failed to convert
+ into a consumable document.
+
Transient errors (network, 5xx, NOT_FOUND seen briefly before the task
is registered server-side) are absorbed up to ``transient_retry_budget``
before being surfaced as failures. The interval grows by
@@ -69,7 +76,7 @@ class DoclingPollingService:
deadline = start + max_seconds
interval = poll_interval
consecutive_not_found = 0
- last_snapshot: Optional[DoclingStatusSnapshot] = None
+ last_snapshot: DoclingStatusSnapshot | None = None
logger.debug("Starting Docling polling", task_id=task_id)
@@ -81,8 +88,75 @@ class DoclingPollingService:
elapsed = time.monotonic() - start
if snapshot.state == DoclingTaskState.SUCCESS:
+ result_fetch_errors = 0
+ while True:
+ now = time.monotonic()
+ elapsed = now - start
+ remaining = deadline - now
+ if remaining <= 0:
+ detail = (
+ "Docling result fetch timed out after SUCCESS status "
+ f"({max_seconds}s budget reached)"
+ )
+ logger.warning(
+ "Docling result fetch exceeded max_seconds after SUCCESS status",
+ task_id=task_id,
+ detail=detail,
+ elapsed_seconds=round(elapsed, 2),
+ max_seconds=max_seconds,
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.FAILED,
+ detail=detail,
+ last_snapshot=snapshot,
+ elapsed_seconds=elapsed,
+ )
+ try:
+ await self.docling_service.fetch_task_result(task_id)
+ break
+ except DoclingTransientError as e:
+ result_fetch_errors += 1
+ if result_fetch_errors > result_fetch_retry_budget:
+ detail = (
+ f"Docling result fetch failed after {result_fetch_errors} "
+ f"transient errors: {str(e)}"
+ )
+ logger.warning(
+ "Docling result fetch exceeded transient retry budget",
+ task_id=task_id,
+ detail=detail,
+ elapsed_seconds=round(elapsed, 2),
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.FAILED,
+ detail=detail,
+ last_snapshot=snapshot,
+ elapsed_seconds=elapsed,
+ )
+ logger.debug(
+ "Transient error fetching Docling result, retrying",
+ task_id=task_id,
+ attempt=result_fetch_errors,
+ error=str(e),
+ )
+ await asyncio.sleep(min(interval, remaining))
+ except DoclingServeError as e:
+ detail = f"Docling result unavailable after SUCCESS status: {str(e)}"
+ logger.warning(
+ "Docling task reached SUCCESS but result fetch failed",
+ task_id=task_id,
+ detail=detail,
+ elapsed_seconds=round(elapsed, 2),
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.FAILED,
+ detail=detail,
+ last_snapshot=snapshot,
+ elapsed_seconds=elapsed,
+ )
+
logger.debug(
- "Docling task reached SUCCESS",
+ "Docling task reached SUCCESS and result is available",
task_id=task_id,
elapsed_seconds=round(elapsed, 2),
)
diff --git a/src/services/docling_service.py b/src/services/docling_service.py
index 7c1b2db7e..9831e89ab 100644
--- a/src/services/docling_service.py
+++ b/src/services/docling_service.py
@@ -33,6 +33,10 @@ class DoclingServeError(Exception):
"""Raised when docling-serve conversion fails."""
+class DoclingTransientError(DoclingServeError):
+ """Raised for errors that may resolve on retry (network failures, 5xx)."""
+
+
class DoclingTaskState(StrEnum):
"""Result of a single status check against Docling Serve."""
@@ -281,8 +285,12 @@ class DoclingService:
try:
response = await client.get(url)
except httpx.RequestError as e:
- raise DoclingServeError(f"Network error fetching docling result: {str(e)}") from e
+ raise DoclingTransientError(f"Network error fetching docling result: {str(e)}") from e
+ if response.status_code >= 500:
+ raise DoclingTransientError(
+ f"Docling result fetch failed with HTTP {response.status_code}: {response.text[:300]}"
+ )
if response.status_code == 404:
raise DoclingServeError(
f"Docling result not found for task {task_id} (task expired or unknown)"
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 25d197dac..20604ff84 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -7,7 +7,7 @@ import uuid
from collections.abc import Coroutine
from typing import Any, TypeVar
-from models.tasks import FileTask, TaskStatus, UploadTask
+from models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask
from session_manager import AnonymousUser
from utils.gpu_detection import get_worker_count
from utils.logging_config import get_logger
@@ -578,26 +578,113 @@ class TaskService:
exception=str(e),
)
- def get_task_status(self, user_id: str, task_id: str) -> dict | None:
- """Get the status of a specific upload task
-
- Includes fallback to shared tasks stored under the "anonymous" user key
- so default system tasks are visible to all users.
- """
+ def _resolve_upload_task(self, user_id: str, task_id: str) -> UploadTask | None:
+ """Look up a task by ID, falling back to anonymous/shared tasks."""
if not task_id:
return None
-
- # Prefer the caller's user_id; otherwise check shared/anonymous tasks
- candidate_user_ids = [user_id, AnonymousUser().user_id]
-
- upload_task = None
- for candidate_user_id in candidate_user_ids:
+ for candidate_user_id in [user_id, AnonymousUser().user_id]:
if (
candidate_user_id in self.task_store
and task_id in self.task_store[candidate_user_id]
):
- upload_task = self.task_store[candidate_user_id][task_id]
- break
+ return self.task_store[candidate_user_id][task_id]
+ return None
+
+ def _serialize_file_task(self, file_task: FileTask) -> dict:
+ """Serialize a FileTask to the standard dict shape."""
+ return {
+ "status": file_task.status.value,
+ "result": file_task.result,
+ "error": file_task.error,
+ "retry_count": file_task.retry_count,
+ "created_at": file_task.created_at,
+ "updated_at": file_task.updated_at,
+ "duration_seconds": file_task.duration_seconds,
+ "filename": file_task.filename,
+ "phase": file_task.phase.value,
+ "docling_status": file_task.docling_status.value,
+ "docling_task_id": file_task.docling_task_id,
+ }
+
+ def _infer_failure_metadata(self, file_task: FileTask) -> dict | None:
+ """Infer structured failure metadata for a failed FileTask.
+
+ Returns a dict with component, failure_phase, user_facing_message, and
+ actionable_by when the failure can be classified, or None when the cause
+ is unknown and no fields should be emitted.
+
+ Priority order: docling_status enum first (stable), error string patterns
+ second (fallback for edge cases like polling timeout).
+ """
+ docling_status = file_task.docling_status
+ phase = file_task.phase
+ error = file_task.error or ""
+
+ if docling_status == DoclingPhaseStatus.FAILED:
+ return {
+ "component": "docling",
+ "failure_phase": "parsing",
+ "user_facing_message": (
+ "The file could not be processed into readable document content."
+ ),
+ "actionable_by": "USER_ACTIONABLE",
+ }
+
+ if docling_status == DoclingPhaseStatus.EXPIRED:
+ return {
+ "component": "docling",
+ "failure_phase": "parsing",
+ "user_facing_message": (
+ "The document processing result could not be found. "
+ "The task may have expired. Please retry ingestion."
+ ),
+ "actionable_by": "RETRYABLE",
+ }
+
+ if phase == IngestionPhase.DOCLING and "Docling conversion did not complete" in error:
+ return {
+ "component": "docling",
+ "failure_phase": "parsing",
+ "user_facing_message": "Document processing timed out. Please retry ingestion.",
+ "actionable_by": "RETRYABLE",
+ }
+
+ if phase == IngestionPhase.DOCLING and docling_status == DoclingPhaseStatus.PROCESSING:
+ return {
+ "component": "docling",
+ "failure_phase": "parsing",
+ "user_facing_message": "Document processing timed out. Please retry ingestion.",
+ "actionable_by": "RETRYABLE",
+ }
+
+ if "already exists" in error:
+ return {
+ "component": "openrag",
+ "failure_phase": "file_validation",
+ "user_facing_message": "A file with this name already exists.",
+ "actionable_by": "USER_ACTIONABLE",
+ }
+
+ if phase == IngestionPhase.LANGFLOW:
+ return {
+ "component": "langflow",
+ "failure_phase": "unknown",
+ "user_facing_message": (
+ "Ingestion failed unexpectedly. Please retry. "
+ "If it fails again, contact your administrator."
+ ),
+ "actionable_by": "RETRYABLE",
+ }
+
+ return None
+
+ def get_task_status(self, user_id: str, task_id: str) -> dict | None:
+ """Get the status of a specific upload task
+
+ Includes fallback to shared tasks stored under the "anonymous" user key
+ so default system tasks are visible to all users.
+ """
+ upload_task = self._resolve_upload_task(user_id, task_id)
if upload_task is None:
return None
@@ -607,19 +694,7 @@ class TaskService:
pending_files_count = 0
for file_path, file_task in upload_task.file_tasks.items():
- file_statuses[file_path] = {
- "status": file_task.status.value,
- "result": file_task.result,
- "error": file_task.error,
- "retry_count": file_task.retry_count,
- "created_at": file_task.created_at,
- "updated_at": file_task.updated_at,
- "duration_seconds": file_task.duration_seconds,
- "filename": file_task.filename,
- "phase": file_task.phase.value,
- "docling_status": file_task.docling_status.value,
- "docling_task_id": file_task.docling_task_id,
- }
+ file_statuses[file_path] = self._serialize_file_task(file_task)
# Count running and pending files
if file_task.status.value == "running":
@@ -642,6 +717,106 @@ class TaskService:
"files": file_statuses,
}
+ def get_task_status2(self, user_id: str, task_id: str) -> dict | None:
+ """Get the status of a specific upload task with structured failure metadata.
+
+ Identical to get_task_status but enriches failed file entries with
+ component, failure_phase, user_facing_message, and actionable_by fields
+ when the failure cause can be classified.
+ """
+ upload_task = self._resolve_upload_task(user_id, task_id)
+
+ if upload_task is None:
+ return None
+
+ file_statuses = {}
+ running_files_count = 0
+ pending_files_count = 0
+
+ for file_path, file_task in upload_task.file_tasks.items():
+ entry = self._serialize_file_task(file_task)
+ if file_task.status == TaskStatus.FAILED:
+ metadata = self._infer_failure_metadata(file_task)
+ if metadata:
+ entry.update(metadata)
+ file_statuses[file_path] = entry
+
+ if file_task.status == TaskStatus.RUNNING:
+ running_files_count += 1
+ elif file_task.status == TaskStatus.PENDING:
+ pending_files_count += 1
+
+ return {
+ "task_id": upload_task.task_id,
+ "status": upload_task.status.value,
+ "total_files": upload_task.total_files,
+ "processed_files": upload_task.processed_files,
+ "successful_files": upload_task.successful_files,
+ "failed_files": upload_task.failed_files,
+ "running_files": running_files_count,
+ "pending_files": pending_files_count,
+ "created_at": upload_task.created_at,
+ "updated_at": upload_task.updated_at,
+ "duration_seconds": upload_task.duration_seconds,
+ "files": file_statuses,
+ }
+
+ def get_all_tasks2(self, user_id: str) -> list:
+ """Get all tasks for a user with structured failure metadata on failed files.
+
+ Identical to get_all_tasks but enriches failed file entries with
+ component, failure_phase, user_facing_message, and actionable_by fields
+ when the failure cause can be classified.
+ """
+ tasks_by_id = {}
+
+ def add_tasks_from_store(store_user_id):
+ if store_user_id not in self.task_store:
+ return
+ for task_id, upload_task in self.task_store[store_user_id].items():
+ if task_id in tasks_by_id:
+ continue
+
+ running_files_count = 0
+ pending_files_count = 0
+ file_statuses = {}
+
+ for file_path, file_task in upload_task.file_tasks.items():
+ if file_task.status != TaskStatus.COMPLETED:
+ entry = self._serialize_file_task(file_task)
+ if file_task.status == TaskStatus.FAILED:
+ metadata = self._infer_failure_metadata(file_task)
+ if metadata:
+ entry.update(metadata)
+ file_statuses[file_path] = entry
+
+ if file_task.status == TaskStatus.RUNNING:
+ running_files_count += 1
+ elif file_task.status == TaskStatus.PENDING:
+ pending_files_count += 1
+
+ tasks_by_id[task_id] = {
+ "task_id": upload_task.task_id,
+ "status": upload_task.status.value,
+ "total_files": upload_task.total_files,
+ "processed_files": upload_task.processed_files,
+ "successful_files": upload_task.successful_files,
+ "failed_files": upload_task.failed_files,
+ "running_files": running_files_count,
+ "pending_files": pending_files_count,
+ "created_at": upload_task.created_at,
+ "updated_at": upload_task.updated_at,
+ "duration_seconds": upload_task.duration_seconds,
+ "files": file_statuses,
+ }
+
+ add_tasks_from_store(user_id)
+ add_tasks_from_store(AnonymousUser().user_id)
+
+ tasks = list(tasks_by_id.values())
+ tasks.sort(key=lambda x: x["created_at"], reverse=True)
+ return tasks
+
def get_all_tasks(self, user_id: str) -> list:
"""Get all tasks for a user
diff --git a/tests/unit/test_docling_polling_service.py b/tests/unit/test_docling_polling_service.py
index 803e01bb8..300a9a3b4 100644
--- a/tests/unit/test_docling_polling_service.py
+++ b/tests/unit/test_docling_polling_service.py
@@ -6,14 +6,20 @@ Docling state without ever invoking Langflow, so that Langflow execution
slots are reserved for chunking / embedding / indexing only.
"""
-import pytest
from unittest.mock import AsyncMock, patch
+import pytest
+
from services.docling_polling_service import (
DoclingPollingService,
PollOutcome,
)
-from services.docling_service import DoclingStatusSnapshot, DoclingTaskState
+from services.docling_service import (
+ DoclingServeError,
+ DoclingStatusSnapshot,
+ DoclingTaskState,
+ DoclingTransientError,
+)
def _snap(state: DoclingTaskState, detail: str | None = None) -> DoclingStatusSnapshot:
@@ -42,6 +48,7 @@ def no_sleep():
@pytest.mark.asyncio
async def test_returns_success_immediately_when_already_done(polling_service, mock_docling_service):
mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)
+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}
result = await polling_service.poll_until_ready(
task_id="t1", poll_interval=1.0, max_seconds=10.0
@@ -49,6 +56,7 @@ async def test_returns_success_immediately_when_already_done(polling_service, mo
assert result.outcome == PollOutcome.SUCCESS
assert mock_docling_service.check_task_status.call_count == 1
+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")
@pytest.mark.asyncio
@@ -62,6 +70,8 @@ async def test_loops_through_processing_then_success(
_snap(DoclingTaskState.SUCCESS),
]
+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}
+
result = await polling_service.poll_until_ready(
task_id="t1", poll_interval=1.0, max_seconds=60.0
)
@@ -69,6 +79,23 @@ async def test_loops_through_processing_then_success(
assert result.outcome == PollOutcome.SUCCESS
assert mock_docling_service.check_task_status.call_count == 4
assert no_sleep.call_count == 3
+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")
+
+
+async def test_success_status_requires_fetchable_result(polling_service, mock_docling_service):
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)
+ mock_docling_service.fetch_task_result.side_effect = DoclingServeError(
+ "missing document.json_content"
+ )
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1", poll_interval=1.0, max_seconds=10.0
+ )
+
+ assert result.outcome == PollOutcome.FAILED
+ assert "missing document.json_content" in (result.detail or "")
+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")
@pytest.mark.asyncio
@@ -96,6 +123,7 @@ async def test_tolerates_brief_not_found_then_succeeds(
_snap(DoclingTaskState.PROCESSING),
_snap(DoclingTaskState.SUCCESS),
]
+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}
result = await polling_service.poll_until_ready(
task_id="t1",
@@ -105,6 +133,7 @@ async def test_tolerates_brief_not_found_then_succeeds(
)
assert result.outcome == PollOutcome.SUCCESS
+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")
@pytest.mark.asyncio
@@ -170,6 +199,7 @@ async def test_backoff_grows_interval_up_to_cap(polling_service, mock_docling_se
_snap(DoclingTaskState.PROCESSING),
_snap(DoclingTaskState.SUCCESS),
]
+ mock_docling_service.fetch_task_result.return_value = {"body": "ok"}
await polling_service.poll_until_ready(
task_id="t1",
@@ -184,3 +214,67 @@ async def test_backoff_grows_interval_up_to_cap(polling_service, mock_docling_se
# see the raw progression.
sleeps = [call.args[0] for call in no_sleep.call_args_list]
assert sleeps == [1.0, 2.0, 4.0, 4.0]
+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")
+
+
+async def test_transient_result_fetch_error_retries_then_succeeds(
+ polling_service, mock_docling_service, no_sleep
+):
+ """A transient network error on fetch_task_result is retried, not failed immediately."""
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)
+ mock_docling_service.fetch_task_result.side_effect = [
+ DoclingTransientError("connection reset"),
+ DoclingTransientError("connection reset"),
+ {"body": "ok"},
+ ]
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1",
+ poll_interval=1.0,
+ max_seconds=60.0,
+ result_fetch_retry_budget=3,
+ )
+
+ assert result.outcome == PollOutcome.SUCCESS
+ assert mock_docling_service.fetch_task_result.await_count == 3
+
+
+async def test_transient_result_fetch_error_exceeds_budget_returns_failed(
+ polling_service, mock_docling_service, no_sleep
+):
+ """Exhausting result_fetch_retry_budget on transient errors yields FAILED."""
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)
+ mock_docling_service.fetch_task_result.side_effect = DoclingTransientError("timeout")
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1",
+ poll_interval=1.0,
+ max_seconds=60.0,
+ result_fetch_retry_budget=2,
+ )
+
+ assert result.outcome == PollOutcome.FAILED
+ assert "transient" in (result.detail or "").lower()
+ assert mock_docling_service.fetch_task_result.await_count == 3 # budget + 1
+
+
+async def test_permanent_result_fetch_error_fails_immediately(
+ polling_service, mock_docling_service
+):
+ """A non-transient DoclingServeError (e.g. 404) is not retried."""
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)
+ mock_docling_service.fetch_task_result.side_effect = DoclingServeError("task expired")
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1",
+ poll_interval=1.0,
+ max_seconds=60.0,
+ result_fetch_retry_budget=3,
+ )
+
+ assert result.outcome == PollOutcome.FAILED
+ assert "task expired" in (result.detail or "")
+ mock_docling_service.fetch_task_result.assert_awaited_once_with("t1")
diff --git a/tests/unit/test_task_service_get_task_status2.py b/tests/unit/test_task_service_get_task_status2.py
new file mode 100644
index 000000000..01b54321b
--- /dev/null
+++ b/tests/unit/test_task_service_get_task_status2.py
@@ -0,0 +1,489 @@
+"""
+Unit tests for TaskService.get_task_status2 and its supporting helpers.
+
+Tests verify that:
+- Failed file entries receive structured failure metadata (component,
+ failure_phase, user_facing_message, actionable_by) when the cause is known.
+- Non-failed entries (completed, running, pending) are never decorated.
+- Unknown failures produce no extra fields.
+- get_task_status (the existing method) is unaffected by the refactor.
+"""
+
+from unittest.mock import Mock
+
+import pytest
+
+from models.tasks import DoclingPhaseStatus, FileTask, IngestionPhase, TaskStatus, UploadTask
+from services.task_service import TaskService
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def task_service():
+ return TaskService(document_service=Mock(), ingestion_timeout=2)
+
+
+def _make_upload_task(task_id: str, file_tasks: dict[str, FileTask]) -> UploadTask:
+ return UploadTask(task_id=task_id, total_files=len(file_tasks), file_tasks=file_tasks)
+
+
+def _make_file_task(
+ *,
+ file_path: str = "test.pdf",
+ status: TaskStatus = TaskStatus.FAILED,
+ phase: IngestionPhase = IngestionPhase.DOCLING,
+ docling_status: DoclingPhaseStatus = DoclingPhaseStatus.PENDING,
+ error: str | None = None,
+ filename: str | None = "test.pdf",
+) -> FileTask:
+ ft = FileTask(file_path=file_path, filename=filename)
+ ft.status = status
+ ft.phase = phase
+ ft.docling_status = docling_status
+ ft.error = error
+ return ft
+
+
+def _store_task(task_service: TaskService, user_id: str, upload_task: UploadTask) -> None:
+ task_service.task_store.setdefault(user_id, {})[upload_task.task_id] = upload_task
+
+
+# ---------------------------------------------------------------------------
+# _infer_failure_metadata
+# ---------------------------------------------------------------------------
+
+
+class TestInferFailureMetadata:
+ def test_docling_failed_status(self, task_service):
+ ft = _make_file_task(docling_status=DoclingPhaseStatus.FAILED)
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "docling"
+ assert meta["failure_phase"] == "parsing"
+ assert meta["actionable_by"] == "USER_ACTIONABLE"
+ assert meta["user_facing_message"]
+
+ def test_docling_expired_status(self, task_service):
+ ft = _make_file_task(docling_status=DoclingPhaseStatus.EXPIRED)
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "docling"
+ assert meta["failure_phase"] == "parsing"
+ assert meta["actionable_by"] == "RETRYABLE"
+ assert meta["user_facing_message"]
+
+ def test_docling_timeout_via_error_string(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PENDING,
+ error="Docling conversion did not complete (timeout): polling exceeded 300s",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "docling"
+ assert meta["failure_phase"] == "parsing"
+ assert meta["actionable_by"] == "RETRYABLE"
+
+ def test_docling_phase_still_processing(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PROCESSING,
+ error="some generic failure",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "docling"
+ assert meta["failure_phase"] == "parsing"
+ assert meta["actionable_by"] == "RETRYABLE"
+
+ def test_langflow_phase_failure(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error="Langflow run timed out",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "langflow"
+ assert meta["failure_phase"] == "unknown"
+ assert meta["actionable_by"] == "RETRYABLE"
+
+ def test_duplicate_file_error(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PENDING,
+ error="File with name 'report.pdf' already exists",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "openrag"
+ assert meta["failure_phase"] == "file_validation"
+ assert meta["actionable_by"] == "USER_ACTIONABLE"
+
+ def test_langflow_duplicate_file_error_prefers_file_validation(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error="File with name 'report.pdf' already exists",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is not None
+ assert meta["component"] == "openrag"
+ assert meta["failure_phase"] == "file_validation"
+ assert meta["actionable_by"] == "USER_ACTIONABLE"
+
+ def test_unknown_failure_returns_none(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PENDING,
+ error="some completely unknown error",
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is None
+
+ def test_no_error_string_unknown_phase(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PENDING,
+ error=None,
+ )
+ meta = task_service._infer_failure_metadata(ft)
+ assert meta is None
+
+
+# ---------------------------------------------------------------------------
+# get_task_status2 — task not found
+# ---------------------------------------------------------------------------
+
+
+class TestGetTaskStatus2NotFound:
+ def test_missing_task_id_returns_none(self, task_service):
+ assert task_service.get_task_status2("user1", "") is None
+
+ def test_unknown_task_id_returns_none(self, task_service):
+ assert task_service.get_task_status2("user1", "does-not-exist") is None
+
+
+# ---------------------------------------------------------------------------
+# get_task_status2 — failure metadata on failed files
+# ---------------------------------------------------------------------------
+
+
+class TestGetTaskStatus2FailureMetadata:
+ def test_docling_failed_adds_metadata(self, task_service):
+ ft = _make_file_task(
+ docling_status=DoclingPhaseStatus.FAILED,
+ error="Docling conversion did not complete (failed): Docling reported failure",
+ )
+ task = _make_upload_task("t1", {"test.pdf": ft})
+ task.status = TaskStatus.FAILED
+ task.failed_files = 1
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t1")
+ file_entry = result["files"]["test.pdf"]
+
+ assert file_entry["status"] == "failed"
+ assert file_entry["component"] == "docling"
+ assert file_entry["failure_phase"] == "parsing"
+ assert file_entry["actionable_by"] == "USER_ACTIONABLE"
+ assert file_entry["user_facing_message"]
+
+ def test_docling_expired_adds_metadata(self, task_service):
+ ft = _make_file_task(docling_status=DoclingPhaseStatus.EXPIRED)
+ task = _make_upload_task("t2", {"test.pdf": ft})
+ task.status = TaskStatus.FAILED
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t2")
+ file_entry = result["files"]["test.pdf"]
+
+ assert file_entry["component"] == "docling"
+ assert file_entry["failure_phase"] == "parsing"
+ assert file_entry["actionable_by"] == "RETRYABLE"
+
+ def test_langflow_failure_adds_metadata(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.LANGFLOW,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error="Langflow run failed",
+ )
+ task = _make_upload_task("t3", {"doc.pdf": ft})
+ task.status = TaskStatus.FAILED
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t3")
+ file_entry = result["files"]["doc.pdf"]
+
+ assert file_entry["component"] == "langflow"
+ assert file_entry["failure_phase"] == "unknown"
+ assert file_entry["actionable_by"] == "RETRYABLE"
+
+ def test_duplicate_file_failure_adds_metadata(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PENDING,
+ error="File with name 'report.pdf' already exists",
+ )
+ task = _make_upload_task("t4", {"report.pdf": ft})
+ task.status = TaskStatus.FAILED
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t4")
+ file_entry = result["files"]["report.pdf"]
+
+ assert file_entry["component"] == "openrag"
+ assert file_entry["failure_phase"] == "file_validation"
+ assert file_entry["actionable_by"] == "USER_ACTIONABLE"
+
+ def test_unknown_failure_adds_no_metadata_fields(self, task_service):
+ ft = _make_file_task(
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PENDING,
+ error="some unrecognised error",
+ )
+ task = _make_upload_task("t5", {"doc.pdf": ft})
+ task.status = TaskStatus.FAILED
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t5")
+ file_entry = result["files"]["doc.pdf"]
+
+ assert "component" not in file_entry
+ assert "failure_phase" not in file_entry
+ assert "user_facing_message" not in file_entry
+ assert "actionable_by" not in file_entry
+
+
+# ---------------------------------------------------------------------------
+# get_task_status2 — non-failed files get no metadata
+# ---------------------------------------------------------------------------
+
+
+class TestGetTaskStatus2NonFailedFiles:
+ def test_completed_file_has_no_metadata(self, task_service):
+ ft = _make_file_task(status=TaskStatus.COMPLETED, phase=IngestionPhase.COMPLETE)
+ ft.docling_status = DoclingPhaseStatus.SUCCESS
+ ft.error = None
+ task = _make_upload_task("t6", {"doc.pdf": ft})
+ task.status = TaskStatus.COMPLETED
+ task.successful_files = 1
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t6")
+ file_entry = result["files"]["doc.pdf"]
+
+ assert "component" not in file_entry
+ assert "failure_phase" not in file_entry
+ assert "user_facing_message" not in file_entry
+ assert "actionable_by" not in file_entry
+
+ def test_running_file_has_no_metadata(self, task_service):
+ ft = _make_file_task(
+ status=TaskStatus.RUNNING,
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PROCESSING,
+ error=None,
+ )
+ task = _make_upload_task("t7", {"doc.pdf": ft})
+ task.status = TaskStatus.RUNNING
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t7")
+ file_entry = result["files"]["doc.pdf"]
+
+ assert "component" not in file_entry
+ assert "failure_phase" not in file_entry
+
+ def test_pending_file_has_no_metadata(self, task_service):
+ ft = _make_file_task(status=TaskStatus.PENDING, error=None)
+ task = _make_upload_task("t8", {"doc.pdf": ft})
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t8")
+ file_entry = result["files"]["doc.pdf"]
+
+ assert "component" not in file_entry
+ assert "failure_phase" not in file_entry
+
+
+# ---------------------------------------------------------------------------
+# get_task_status2 — mixed files
+# ---------------------------------------------------------------------------
+
+
+class TestGetTaskStatus2Mixed:
+ def test_only_failed_files_get_metadata(self, task_service):
+ failed_ft = _make_file_task(
+ file_path="bad.pdf",
+ filename="bad.pdf",
+ docling_status=DoclingPhaseStatus.FAILED,
+ error="Docling conversion did not complete (failed): Docling reported failure",
+ )
+ ok_ft = _make_file_task(
+ file_path="good.pdf",
+ filename="good.pdf",
+ status=TaskStatus.COMPLETED,
+ phase=IngestionPhase.COMPLETE,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error=None,
+ )
+ running_ft = _make_file_task(
+ file_path="wip.pdf",
+ filename="wip.pdf",
+ status=TaskStatus.RUNNING,
+ phase=IngestionPhase.DOCLING,
+ docling_status=DoclingPhaseStatus.PROCESSING,
+ error=None,
+ )
+ task = _make_upload_task(
+ "t9", {"bad.pdf": failed_ft, "good.pdf": ok_ft, "wip.pdf": running_ft}
+ )
+ task.status = TaskStatus.RUNNING
+ task.successful_files = 1
+ task.failed_files = 1
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t9")
+
+ bad = result["files"]["bad.pdf"]
+ assert bad["component"] == "docling"
+ assert bad["failure_phase"] == "parsing"
+
+ good = result["files"]["good.pdf"]
+ assert "component" not in good
+
+ wip = result["files"]["wip.pdf"]
+ assert "component" not in wip
+
+ def test_top_level_fields_present(self, task_service):
+ ft = _make_file_task(docling_status=DoclingPhaseStatus.FAILED)
+ task = _make_upload_task("t10", {"doc.pdf": ft})
+ task.status = TaskStatus.FAILED
+ task.failed_files = 1
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status2("user1", "t10")
+
+ for key in (
+ "task_id",
+ "status",
+ "total_files",
+ "processed_files",
+ "successful_files",
+ "failed_files",
+ "running_files",
+ "pending_files",
+ "created_at",
+ "updated_at",
+ "duration_seconds",
+ "files",
+ ):
+ assert key in result, f"missing key: {key}"
+
+
+# ---------------------------------------------------------------------------
+# get_task_status2 — anonymous user fallback
+# ---------------------------------------------------------------------------
+
+
+class TestGetTaskStatus2AnonymousFallback:
+ def test_resolves_anonymous_task(self, task_service):
+ from session_manager import AnonymousUser
+
+ ft = _make_file_task(docling_status=DoclingPhaseStatus.FAILED)
+ task = _make_upload_task("anon-task", {"doc.pdf": ft})
+ task.status = TaskStatus.FAILED
+ anon_id = AnonymousUser().user_id
+ task_service.task_store[anon_id] = {"anon-task": task}
+
+ result = task_service.get_task_status2("some-other-user", "anon-task")
+ assert result is not None
+ assert result["task_id"] == "anon-task"
+ assert result["files"]["doc.pdf"]["component"] == "docling"
+
+
+# ---------------------------------------------------------------------------
+# Regression: get_task_status remains unchanged
+# ---------------------------------------------------------------------------
+
+
+class TestGetTaskStatusRegression:
+ def test_no_metadata_fields_in_original_method(self, task_service):
+ ft = _make_file_task(
+ docling_status=DoclingPhaseStatus.FAILED,
+ error="Docling conversion did not complete (failed): Docling reported failure",
+ )
+ task = _make_upload_task("reg1", {"doc.pdf": ft})
+ task.status = TaskStatus.FAILED
+ task.failed_files = 1
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status("user1", "reg1")
+ file_entry = result["files"]["doc.pdf"]
+
+ assert file_entry["status"] == "failed"
+ assert "component" not in file_entry
+ assert "failure_phase" not in file_entry
+ assert "user_facing_message" not in file_entry
+ assert "actionable_by" not in file_entry
+
+ def test_original_method_baseline_shape_unchanged(self, task_service):
+ ft = _make_file_task(
+ status=TaskStatus.COMPLETED,
+ phase=IngestionPhase.COMPLETE,
+ docling_status=DoclingPhaseStatus.SUCCESS,
+ error=None,
+ )
+ task = _make_upload_task("reg2", {"doc.pdf": ft})
+ task.status = TaskStatus.COMPLETED
+ task.successful_files = 1
+ _store_task(task_service, "user1", task)
+
+ result = task_service.get_task_status("user1", "reg2")
+ file_entry = result["files"]["doc.pdf"]
+
+ expected_keys = {
+ "status",
+ "result",
+ "error",
+ "retry_count",
+ "created_at",
+ "updated_at",
+ "duration_seconds",
+ "filename",
+ "phase",
+ "docling_status",
+ "docling_task_id",
+ }
+ assert set(file_entry.keys()) == expected_keys
+
+ def test_both_methods_return_same_baseline_for_failed_file(self, task_service):
+ """get_task_status2 must include all fields from get_task_status plus any extras."""
+ ft = _make_file_task(
+ docling_status=DoclingPhaseStatus.FAILED,
+ error="Docling conversion did not complete (failed): Docling reported failure",
+ )
+ task = _make_upload_task("reg3", {"doc.pdf": ft})
+ task.status = TaskStatus.FAILED
+ task.failed_files = 1
+ _store_task(task_service, "user1", task)
+
+ v1 = task_service.get_task_status("user1", "reg3")
+ v2 = task_service.get_task_status2("user1", "reg3")
+
+ # All v1 top-level keys must appear in v2
+ for key in v1:
+ assert key in v2, f"get_task_status2 missing top-level key: {key}"
+ if key != "files":
+ assert v1[key] == v2[key], f"top-level key '{key}' differs"
+
+ # All baseline file entry keys must survive in get_task_status2
+ v1_file = v1["files"]["doc.pdf"]
+ v2_file = v2["files"]["doc.pdf"]
+ for key in v1_file:
+ assert key in v2_file, f"get_task_status2 file entry missing key: {key}"
+ assert v1_file[key] == v2_file[key], f"file entry key '{key}' differs"