Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Implement two-phase ingestion flow with Docling polling (#1572)

Rico Furtado5598705b7266
diff --git a/src/app/container.py b/src/app/container.py
index 8cbe394cc..1f20d4f4a 100644
--- a/src/app/container.py
+++ b/src/app/container.py
@@ -5,6 +5,7 @@ Returns a dict consumed by routes (via FastAPI Depends) and the lifespan hook.
from api.connector_router import ConnectorRouter
from config.settings import (
+ ENABLE_BACKEND_DOCLING_POLLING,
INGESTION_TIMEOUT,
JWT_SIGNING_KEY,
SESSION_SECRET,
@@ -19,6 +20,7 @@ from connectors.service import ConnectorService
from services.api_key_service import APIKeyService
from services.auth_service import AuthService
from services.chat_service import ChatService
+from services.docling_polling_service import DoclingPollingService
from services.document_service import DocumentService
from services.flows_service import FlowsService
from services.knowledge_filter_service import KnowledgeFilterService
@@ -69,11 +71,24 @@ async def initialize_services():
)
search_service = SearchService(session_manager, models_service)
register_search_service(search_service)
+
+ # Backend-side Docling polling coordinator. Constructed once as a
+ # singleton (it is stateless) and gated by ENABLE_BACKEND_DOCLING_POLLING
+ # so operators can roll back to the legacy single-call ingestion path
+ # without code changes. When disabled, downstream callers receive None
+ # and fall through to the legacy flow.
+ docling_polling_service = (
+ DoclingPollingService(clients.docling_service)
+ if ENABLE_BACKEND_DOCLING_POLLING and clients.docling_service is not None
+ else None
+ )
+
task_service = TaskService(
document_service,
models_service,
ingestion_timeout=INGESTION_TIMEOUT,
docling_service=clients.docling_service,
+ docling_polling_service=docling_polling_service,
)
flows_service = FlowsService()
chat_service = ChatService(flows_service=flows_service)
@@ -197,6 +212,7 @@ async def initialize_services():
"api_key_service": api_key_service,
"langflow_mcp_service": langflow_mcp_service,
"docling_service": clients.docling_service,
+ "docling_polling_service": docling_polling_service,
"rbac_service": rbac_service,
"workspace_config_service": workspace_config_service,
}
diff --git a/src/config/settings.py b/src/config/settings.py
index a4bd232a6..df311e481 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -177,6 +177,24 @@ LANGFLOW_CONNECT_TIMEOUT = get_env_float("LANGFLOW_CONNECT_TIMEOUT", 30.0) # 30
# Default: 3600 seconds (60 minutes)
INGESTION_TIMEOUT = get_env_int("INGESTION_TIMEOUT", 3600)
+# Two-phase ingestion: backend-side Docling polling configuration.
+# Controls how the OpenRAG backend waits for Docling Serve to finish converting
+# a document before invoking the Langflow ingestion flow. Decoupling this poll
+# from Langflow keeps Langflow execution slots free during long Docling jobs.
+# When ENABLE_BACKEND_DOCLING_POLLING is false, the backend submits to Docling
+# and immediately invokes Langflow with the task_id; Langflow's DoclingRemote
+# component then polls Docling itself (legacy single-call behavior).
+ENABLE_BACKEND_DOCLING_POLLING = os.getenv("ENABLE_BACKEND_DOCLING_POLLING", "true").lower() in (
+ "true",
+ "1",
+ "yes",
+)
+DOCLING_POLL_INTERVAL_SECONDS = get_env_float("DOCLING_POLL_INTERVAL_SECONDS", 3.0)
+DOCLING_POLL_MAX_SECONDS = get_env_int("DOCLING_POLL_MAX_SECONDS", 1800)
+DOCLING_POLL_MAX_INTERVAL_SECONDS = get_env_float("DOCLING_POLL_MAX_INTERVAL_SECONDS", 30.0)
+DOCLING_POLL_BACKOFF_FACTOR = get_env_float("DOCLING_POLL_BACKOFF_FACTOR", 1.5)
+DOCLING_POLL_TRANSIENT_RETRIES = get_env_int("DOCLING_POLL_TRANSIENT_RETRIES", 5)
+
def is_no_auth_mode():
"""Check if we're running in no-auth mode (OAuth credentials missing)"""
diff --git a/src/dependencies.py b/src/dependencies.py
index 864a51bf6..5a12c0f09 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -133,6 +133,10 @@ def get_docling_service(services: dict = Depends(get_services)):
return services["docling_service"]
+def get_docling_polling_service(services: dict = Depends(get_services)):
+ return services["docling_polling_service"]
+
+
def get_rbac_service(services: dict = Depends(get_services)):
return services["rbac_service"]
diff --git a/src/models/processors.py b/src/models/processors.py
index 51cb655a7..d9e225d2b 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -735,6 +735,9 @@ class S3FileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Download an S3 object and process it using DocumentService"""
import time
+ import asyncio
+ import datetime
+ from config.settings import clients, get_embedding_model, get_index_name
from models.tasks import TaskStatus
@@ -787,7 +790,7 @@ class S3FileProcessor(TaskProcessor):
class LangflowFileProcessor(TaskProcessor):
- """Processor for Langflow file uploads with upload and ingest"""
+ """Processor for Langflow file uploads with two-phase Docling + Langflow ingestion."""
def __init__(
self,
@@ -802,6 +805,7 @@ class LangflowFileProcessor(TaskProcessor):
settings: dict = None,
replace_duplicates: bool = False,
connector_type: str = "local",
+ docling_polling_service=None,
):
super().__init__()
self.langflow_file_service = langflow_file_service
@@ -815,6 +819,11 @@ class LangflowFileProcessor(TaskProcessor):
self.settings = settings
self.replace_duplicates = replace_duplicates
self.connector_type = connector_type
+ # Backend-side Docling polling coordinator. Injected by TaskService
+ # from the container; gating by ENABLE_BACKEND_DOCLING_POLLING happens
+ # at construction time in app.container. When None, the legacy
+ # single-call ingestion path is used.
+ self.docling_polling_service = docling_polling_service
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process a file path using LangflowFileService upload_and_ingest_file"""
@@ -886,7 +895,10 @@ class LangflowFileProcessor(TaskProcessor):
# Prepare metadata tweaks similar to API endpoint
final_tweaks = self.tweaks.copy() if self.tweaks else {}
- # Process file using langflow service
+ # Process file using langflow service. Passing the polling
+ # service triggers the two-phase model: backend polls Docling,
+ # then invokes Langflow only after SUCCESS. file_task is passed
+ # so phase / docling_status are tracked on the task record.
result = await self.langflow_file_service.upload_and_ingest_file(
file_tuple=file_tuple,
session_id=self.session_id,
@@ -897,6 +909,8 @@ class LangflowFileProcessor(TaskProcessor):
owner_name=self.owner_name,
owner_email=self.owner_email,
connector_type=self.connector_type,
+ docling_polling_service=self.docling_polling_service,
+ file_task=file_task,
)
# Update task with success
diff --git a/src/models/tasks.py b/src/models/tasks.py
index af0466587..40bda40ff 100644
--- a/src/models/tasks.py
+++ b/src/models/tasks.py
@@ -12,6 +12,30 @@ class TaskStatus(Enum):
FAILED = "failed"
+class DoclingPhaseStatus(Enum):
+ """Tracks the state of the Docling conversion sub-phase for a single file."""
+
+ PENDING = "pending"
+ PROCESSING = "processing"
+ SUCCESS = "success"
+ FAILED = "failed"
+ EXPIRED = "expired"
+
+
+class IngestionPhase(Enum):
+ """Tracks which phase of the two-phase ingestion pipeline a file is in.
+
+ DOCLING: file has been submitted to Docling Serve and the backend is
+ polling for conversion completion. Langflow has not been called.
+ LANGFLOW: Docling conversion succeeded; Langflow ingestion flow is running.
+ COMPLETE: Langflow ingestion finished and the document is indexed.
+ """
+
+ DOCLING = "docling"
+ LANGFLOW = "langflow"
+ COMPLETE = "complete"
+
+
@dataclass
class FileTask:
file_path: str
@@ -22,6 +46,11 @@ class FileTask:
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
filename: Optional[str] = None # Original filename for display
+ # Two-phase ingestion fields. Only meaningful for processors that submit
+ # files to Docling Serve and then trigger Langflow (i.e. LangflowFileProcessor).
+ docling_task_id: Optional[str] = None
+ docling_status: DoclingPhaseStatus = DoclingPhaseStatus.PENDING
+ phase: IngestionPhase = IngestionPhase.DOCLING
@property
def duration_seconds(self) -> float:
diff --git a/src/services/docling_polling_service.py b/src/services/docling_polling_service.py
new file mode 100644
index 000000000..a8cc83a07
--- /dev/null
+++ b/src/services/docling_polling_service.py
@@ -0,0 +1,144 @@
+"""Backend-side Docling polling coordinator.
+
+Owns the wait loop that previously lived inside the Langflow ingestion flow's
+DoclingRemote component. Keeping the loop here means a Langflow execution slot
+is held only for the brief chunk/embed/index phase, not for the entire Docling
+conversion (which can be many minutes for large or OCR-heavy documents).
+"""
+
+import asyncio
+import time
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+from services.docling_service import (
+ DoclingService,
+ DoclingStatusSnapshot,
+ DoclingTaskState,
+)
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class PollOutcome(str, Enum):
+ SUCCESS = "success"
+ FAILED = "failed"
+ EXPIRED = "expired"
+ TIMEOUT = "timeout"
+
+
+@dataclass
+class DoclingPollResult:
+ outcome: PollOutcome
+ detail: Optional[str] = None
+ last_snapshot: Optional[DoclingStatusSnapshot] = None
+ elapsed_seconds: float = 0.0
+
+
+class DoclingPollingService:
+ """Polls a Docling task to terminal state without involving Langflow."""
+
+ def __init__(self, docling_service: DoclingService):
+ self.docling_service = docling_service
+
+ async def poll_until_ready(
+ self,
+ task_id: str,
+ poll_interval: float,
+ max_seconds: float,
+ max_interval: float = 30.0,
+ backoff_factor: float = 1.5,
+ transient_retry_budget: int = 5,
+ ) -> DoclingPollResult:
+ """Loop on Docling status until terminal or until max_seconds elapses.
+
+ Transient errors (network, 5xx, NOT_FOUND seen briefly before the task
+ is registered server-side) are absorbed up to ``transient_retry_budget``
+ before being surfaced as failures. The interval grows by
+ ``backoff_factor`` after each non-success poll, capped at
+ ``max_interval``, so we don't hammer Docling for slow conversions.
+ """
+ if poll_interval <= 0:
+ raise ValueError("poll_interval must be > 0")
+ if max_seconds <= 0:
+ raise ValueError("max_seconds must be > 0")
+
+ start = time.monotonic()
+ deadline = start + max_seconds
+ interval = poll_interval
+ consecutive_not_found = 0
+ last_snapshot: Optional[DoclingStatusSnapshot] = None
+
+ logger.debug("Starting Docling polling", task_id=task_id)
+
+ while True:
+ logger.debug("Docling polling", task_id=task_id)
+ snapshot = await self.docling_service.check_task_status(task_id)
+ last_snapshot = snapshot
+ logger.debug("Snapshot received", task_id=task_id, snapshot=last_snapshot)
+ elapsed = time.monotonic() - start
+
+ if snapshot.state == DoclingTaskState.SUCCESS:
+ logger.debug(
+ "Docling task reached SUCCESS",
+ task_id=task_id,
+ elapsed_seconds=round(elapsed, 2),
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.SUCCESS,
+ last_snapshot=snapshot,
+ elapsed_seconds=elapsed,
+ )
+
+ if snapshot.state == DoclingTaskState.FAILED:
+ logger.warning(
+ "Docling task reported FAILED",
+ task_id=task_id,
+ detail=snapshot.detail,
+ elapsed_seconds=round(elapsed, 2),
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.FAILED,
+ detail=snapshot.detail or "Docling reported failure",
+ last_snapshot=snapshot,
+ elapsed_seconds=elapsed,
+ )
+
+ if snapshot.state == DoclingTaskState.NOT_FOUND:
+ # NOT_FOUND immediately after submission can be a brief window
+ # before the task is visible. Tolerate up to the transient
+ # budget; beyond that, treat as expired/unknown.
+ consecutive_not_found += 1
+ if consecutive_not_found > transient_retry_budget:
+ logger.warning(
+ "Docling task NOT_FOUND past retry budget",
+ task_id=task_id,
+ elapsed_seconds=round(elapsed, 2),
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.EXPIRED,
+ detail="Docling task not found (expired or unknown task_id)",
+ last_snapshot=snapshot,
+ elapsed_seconds=elapsed,
+ )
+ else:
+ consecutive_not_found = 0
+
+ # Compute remaining time before deadline; sleep at most that long.
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ logger.warning(
+ "Docling polling exceeded max_seconds",
+ task_id=task_id,
+ max_seconds=max_seconds,
+ )
+ return DoclingPollResult(
+ outcome=PollOutcome.TIMEOUT,
+ detail=f"Docling polling timed out after {max_seconds}s",
+ last_snapshot=snapshot,
+ elapsed_seconds=time.monotonic() - start,
+ )
+ await asyncio.sleep(min(interval, remaining))
+ interval = min(interval * backoff_factor, max_interval)
diff --git a/src/services/docling_service.py b/src/services/docling_service.py
index 8c971332f..98ffbf627 100644
--- a/src/services/docling_service.py
+++ b/src/services/docling_service.py
@@ -1,5 +1,7 @@
import asyncio
import json
+from dataclasses import dataclass
+from enum import Enum
import platform
from pathlib import Path
from typing import Any
@@ -27,10 +29,30 @@ class DoclingConfig(BaseModel):
picture_description_local: dict | None = None
+
class DoclingServeError(Exception):
"""Raised when docling-serve conversion fails."""
+class DoclingTaskState(str, Enum):
+ """Result of a single status check against Docling Serve."""
+
+ PENDING = "pending"
+ PROCESSING = "processing"
+ SUCCESS = "success"
+ FAILED = "failed"
+ NOT_FOUND = "not_found"
+
+
+@dataclass
+class DoclingStatusSnapshot:
+ """Single-point-in-time view of a Docling task's state."""
+
+ state: DoclingTaskState
+ detail: Optional[str] = None
+ raw: Optional[dict] = None
+
+
def get_docling_preset_configs(
table_structure=False, ocr=False, picture_descriptions=False
) -> dict[str, Any]:
@@ -56,7 +78,7 @@ class DoclingService:
_default_client: httpx.AsyncClient | None = None
def __init__(
- self, docling_url: str | None = None, httpx_client: httpx.AsyncClient | None = None
+ self, docling_url: Optional[str] = None, httpx_client: Optional[httpx.AsyncClient] = None
):
"""
Initialize the DoclingService.
@@ -189,6 +211,98 @@ class DoclingService:
logger.error("Docling result retrieval failed", task_id=task_id, error=str(e))
raise
+ async def check_task_status(self, task_id: str) -> DoclingStatusSnapshot:
+ """
+ Single (non-blocking) status check against Docling Serve.
+
+ Used by the backend polling coordinator so that the polling loop lives
+ in OpenRAG and not inside Langflow. Maps the Docling Serve response
+ into a DoclingStatusSnapshot regardless of HTTP outcome.
+ """
+ client = self._get_client()
+ url = f"{self.docling_url}/v1/status/poll/{task_id}"
+ try:
+ response = await client.get(url)
+ except httpx.RequestError as e:
+ # Transient network error — surface as PROCESSING so caller can
+ # retry without prematurely failing the file.
+ logger.debug("Transient error checking docling status", task_id=task_id, error=str(e))
+ return DoclingStatusSnapshot(state=DoclingTaskState.PROCESSING, detail=str(e))
+
+ if response.status_code == 404:
+ return DoclingStatusSnapshot(state=DoclingTaskState.NOT_FOUND, detail="Task not found")
+ if response.status_code >= 500:
+ logger.debug(
+ "Transient HTTP error from docling status endpoint",
+ task_id=task_id,
+ status_code=response.status_code,
+ )
+ return DoclingStatusSnapshot(
+ state=DoclingTaskState.PROCESSING,
+ detail=f"HTTP {response.status_code}",
+ )
+ if response.status_code >= 400:
+ return DoclingStatusSnapshot(
+ state=DoclingTaskState.FAILED,
+ detail=f"HTTP {response.status_code}: {response.text[:300]}",
+ )
+
+ try:
+ payload = response.json()
+ except ValueError as e:
+ return DoclingStatusSnapshot(
+ state=DoclingTaskState.FAILED,
+ detail=f"Malformed status response: {str(e)}",
+ )
+
+ status = payload.get("task_status")
+ if status == "success":
+ return DoclingStatusSnapshot(state=DoclingTaskState.SUCCESS, raw=payload)
+ if status == "failure":
+ return DoclingStatusSnapshot(
+ state=DoclingTaskState.FAILED,
+ detail=str(payload),
+ raw=payload,
+ )
+ if status in ("started", "processing", "running"):
+ return DoclingStatusSnapshot(state=DoclingTaskState.PROCESSING, raw=payload)
+ return DoclingStatusSnapshot(state=DoclingTaskState.PENDING, raw=payload)
+
+ async def fetch_task_result(self, task_id: str) -> Dict[str, Any]:
+ """
+ Fetch the converted document for a Docling task that is already SUCCESS.
+
+ Raises:
+ DoclingServeError: if the result endpoint returns 404 (task expired
+ or unknown), an unexpected status code, or a payload missing
+ document.json_content.
+ """
+ client = self._get_client()
+ url = f"{self.docling_url}/v1/result/{task_id}"
+ try:
+ response = await client.get(url)
+ except httpx.RequestError as e:
+ raise DoclingServeError(f"Network error fetching docling result: {str(e)}") from e
+
+ if response.status_code == 404:
+ raise DoclingServeError(
+ f"Docling result not found for task {task_id} (task expired or unknown)"
+ )
+ if response.status_code >= 400:
+ raise DoclingServeError(
+ f"Docling result fetch failed with HTTP {response.status_code}: {response.text[:300]}"
+ )
+
+ try:
+ payload = response.json()
+ except ValueError as e:
+ raise DoclingServeError(f"Malformed docling result payload: {str(e)}") from e
+
+ doc_content = payload.get("document", {}).get("json_content")
+ if doc_content is None:
+ raise DoclingServeError("docling-serve response missing document.json_content")
+ return doc_content
+
async def _poll_result(
self,
client: httpx.AsyncClient,
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index eeb69ba0a..d56c76dec 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -71,7 +71,7 @@ class LangflowFileService:
return final_tweaks
- async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:
+ async def upload_user_file(self, file_tuple, jwt_token: Optional[str] = None) -> Dict[str, Any]:
"""Upload a file using Langflow Files API v2: POST /api/v2/files.
Returns JSON with keys: id, name, path, size, provider.
"""
@@ -515,63 +515,141 @@ class LangflowFileService:
raise last_error
raise RuntimeError("Unable to validate/import URL ingest flow")
+ async def submit_to_docling(self, filename: str, content: bytes, jwt_token: Optional[str] = None,
+ owner: Optional[str] = None,) -> str:
+ """Upload a file to Docling Serve and return the task_id immediately.
+
+ Phase 1 of the two-phase ingestion model. The caller is responsible
+ for polling Docling (typically via DoclingPollingService) and only
+ invoking Langflow once Docling reports SUCCESS.
+ """
+ if self.docling_service is None:
+ raise RuntimeError(
+ "DoclingService is not configured. Ensure DOCLING_SERVE_URL is set "
+ "and the service was injected correctly."
+ )
+ try:
+ task_id = await self.docling_service.upload_to_docling_direct_async(
+ filename, content, user_id=owner, auth_header=jwt_token
+ logger.debug(
+ "[LF] Docling submission accepted",
+ extra={"task_id": task_id, "filename": filename},
+ )
+ return task_id
+ except Exception as e:
+ logger.error(
+ "[LF] Docling submission failed",
+ extra={"error": str(e), "filename": filename},
+ )
+ raise Exception(f"Docling upload failed: {str(e)}")
+
async def upload_and_ingest_file(
self,
file_tuple,
- session_id: str | None = None,
- tweaks: dict[str, Any] | None = None,
- settings: dict[str, Any] | None = None,
- jwt_token: str | None = None,
- owner: str | None = None,
- owner_name: str | None = None,
- owner_email: str | None = None,
- connector_type: str | None = None,
- ) -> dict[str, Any]:
+ session_id: Optional[str] = None,
+ tweaks: Optional[Dict[str, Any]] = None,
+ settings: Optional[Dict[str, Any]] = None,
+ jwt_token: Optional[str] = None,
+ owner: Optional[str] = None,
+ owner_name: Optional[str] = None,
+ owner_email: Optional[str] = None,
+ connector_type: Optional[str] = None,
+ docling_polling_service: Optional[Any] = None,
+ file_task: Optional[Any] = None,
+ ) -> Dict[str, Any]:
"""
- Combined Docling upload and Langflow ingest operation.
- First uploads the file to Docling, then runs ingestion on it via Langflow
- using the Docling task ID.
+ Two-phase Docling upload + Langflow ingest operation.
+
+ Phase 1: submit the file to Docling, receive a task_id. If a
+ ``docling_polling_service`` is provided, poll the backend for Docling
+ completion before invoking Langflow. This keeps Langflow execution
+ slots free during long Docling conversions.
+
+ Phase 2: trigger the Langflow ingestion flow once Docling has
+ succeeded. The task_id is forwarded so the flow's DoclingRemote
+ component fetches the already-completed result instead of re-uploading
+ or re-polling.
+
+ When ``docling_polling_service`` is None, falls back to the legacy
+ single-step behavior (Langflow polls Docling itself), preserving
+ backward compatibility.
Args:
- file_tuple: File tuple (filename, content, content_type)
- session_id: Optional session ID for the ingestion flow
- tweaks: Optional tweaks for the ingestion flow
- settings: Optional UI settings to convert to component tweaks
- jwt_token: Optional JWT token for authentication
-
- Returns:
- Combined result with Docling task info and ingestion result
+ file_tuple: (filename, content, content_type)
+ docling_polling_service: optional DoclingPollingService for the
+ two-phase flow. When None, Langflow handles polling.
+ file_task: optional FileTask for phase / status tracking.
"""
- logger.debug("[LF] Starting Docling-based upload and ingest operation")
+ from models.tasks import DoclingPhaseStatus, IngestionPhase
- if self.docling_service is None:
- raise RuntimeError(
- "DoclingService is not configured. Ensure DOCLING_SERVE_URL is set "
- "and the service was injected correctly."
- )
+ logger.debug("[LF] Starting two-phase Docling+Langflow ingest")
filename, content, _ = file_tuple
- # Step 1: Upload the file to Docling
- try:
- task_id = await self.docling_service.upload_to_docling_direct_async(
- filename, content, user_id=owner, auth_header=jwt_token
+ # ── Phase 1: submit to Docling ──────────────────────────────────
+ if file_task is not None:
+ file_task.phase = IngestionPhase.DOCLING
+ file_task.docling_status = DoclingPhaseStatus.PENDING
+
+ task_id = await self.submit_to_docling(filename, content,user_id=owner, auth_header=jwt_token)
+
+ if file_task is not None:
+ file_task.docling_task_id = task_id
+ file_task.docling_status = DoclingPhaseStatus.PROCESSING
+
+ # ── Phase 1b: backend-side polling (optional) ───────────────────
+ if docling_polling_service is not None:
+ from config.settings import (
+ DOCLING_POLL_BACKOFF_FACTOR,
+ DOCLING_POLL_INTERVAL_SECONDS,
+ DOCLING_POLL_MAX_INTERVAL_SECONDS,
+ DOCLING_POLL_MAX_SECONDS,
+ DOCLING_POLL_TRANSIENT_RETRIES,
)
- logger.debug(
- "[LF] Docling upload completed successfully",
+ from services.docling_polling_service import PollOutcome
+
+ poll_result = await docling_polling_service.poll_until_ready(
+ task_id=task_id,
+ poll_interval=DOCLING_POLL_INTERVAL_SECONDS,
+ max_seconds=DOCLING_POLL_MAX_SECONDS,
+ max_interval=DOCLING_POLL_MAX_INTERVAL_SECONDS,
+ backoff_factor=DOCLING_POLL_BACKOFF_FACTOR,
+ transient_retry_budget=DOCLING_POLL_TRANSIENT_RETRIES,
+ )
+
+ if poll_result.outcome != PollOutcome.SUCCESS:
+ if file_task is not None:
+ if poll_result.outcome == PollOutcome.EXPIRED:
+ file_task.docling_status = DoclingPhaseStatus.EXPIRED
+ else:
+ file_task.docling_status = DoclingPhaseStatus.FAILED
+ logger.error(
+ "[LF] Docling polling did not reach SUCCESS; skipping Langflow",
+ extra={
+ "task_id": task_id,
+ "filename": filename,
+ "outcome": poll_result.outcome.value,
+ "detail": poll_result.detail,
+ "elapsed_seconds": round(poll_result.elapsed_seconds, 2),
+ },
+ )
+ raise Exception(
+ f"Docling conversion did not complete ({poll_result.outcome.value}): "
+ f"{poll_result.detail or 'no detail provided'}"
+ )
+
+ if file_task is not None:
+ file_task.docling_status = DoclingPhaseStatus.SUCCESS
+ logger.info(
+ "[LF] Docling conversion ready; proceeding to Langflow",
extra={
"task_id": task_id,
"filename": filename,
+ "elapsed_seconds": round(poll_result.elapsed_seconds, 2),
},
)
- except Exception as e:
- logger.error(
- "[LF] Docling upload failed during combined operation",
- extra={"error": str(e)},
- )
- raise Exception(f"Docling upload failed: {str(e)}") from e
- # Step 2: Prepare for ingestion
+ # ── Phase 2: trigger Langflow ingestion ─────────────────────────
final_tweaks = LangflowFileService.merge_ui_ingest_settings_into_tweaks(tweaks, settings)
if settings:
logger.debug(
@@ -579,7 +657,9 @@ class LangflowFileService:
extra={"settings": settings, "tweaks": final_tweaks},
)
- # Step 3: Run ingestion via Langflow
+ if file_task is not None:
+ file_task.phase = IngestionPhase.LANGFLOW
+
try:
total_start_time = time.time()
ingest_result = await self.run_ingestion_flow(
@@ -601,10 +681,18 @@ class LangflowFileService:
"[LF] Ingestion failed during combined operation",
extra={"error": str(e), "filename": filename},
)
- # Can't cancel Docling task because API for that does not exist
+ # Docling Serve has no cancel endpoint; let any orphan task expire.
raise
- # Return combined result
+ if file_task is not None:
+ file_task.phase = IngestionPhase.COMPLETE
+ # Legacy path leaves docling_status at PROCESSING because the
+ # backend never observed Docling completion directly. Langflow
+ # returning success implies its DoclingRemote component consumed
+ # the task, so Docling succeeded — mark SUCCESS to keep status
+ # fields coherent. Idempotent for the polling path.
+ file_task.docling_status = DoclingPhaseStatus.SUCCESS
+
return {
"status": "success",
"docling_task_id": task_id,
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 85116af0f..4c7176990 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -19,6 +19,7 @@ logger = get_logger(__name__)
class IngestionTimeoutError(Exception):
"""Raised when file processing exceeds the configured timeout"""
+
pass
@@ -26,13 +27,23 @@ class TaskService:
# Cleanup interval in seconds (2 hours)
CLEANUP_INTERVAL_SECONDS = 2 * 60 * 60
- def __init__(self, document_service=None, models_service=None, ingestion_timeout=3600, docling_service=None):
+ def __init__(
+ self,
+ document_service=None,
+ models_service=None,
+ ingestion_timeout=3600,
+ docling_service=None,
+ docling_polling_service=None,
+ ):
self.document_service = document_service
self.models_service = models_service
self.docling_service = docling_service
- self.task_store: dict[
- str, dict[str, UploadTask]
- ] = {} # user_id -> {task_id -> UploadTask}
+ # Backend-side Docling polling coordinator. Injected by the container
+ # so LangflowFileProcessor receives it from the established DI chain
+ # rather than constructing it inline. None disables the two-phase
+ # flow and falls back to the legacy single-call ingestion path.
+ self.docling_polling_service = docling_polling_service
+ self.task_store: dict[str, dict[str, UploadTask]] = {} # user_id -> {task_id -> UploadTask}
self.background_tasks = set()
self.ingestion_timeout = ingestion_timeout
self._cleanup_task: asyncio.Task | None = None
@@ -102,7 +113,9 @@ class TaskService:
try:
return await asyncio.wait_for(coro, timeout=timeout)
except asyncio.TimeoutError:
- raise IngestionTimeoutError(f"File processing timed out after {timeout} seconds.") from None
+ raise IngestionTimeoutError(
+ f"File processing timed out after {timeout} seconds."
+ ) from None
async def create_upload_task(
self,
@@ -160,8 +173,11 @@ class TaskService:
settings=settings,
replace_duplicates=replace_duplicates,
connector_type=connector_type,
+ docling_polling_service=self.docling_polling_service,
+ )
+ return await self.create_custom_task(
+ user_id, file_paths, processor, original_filenames, existing_task_id=existing_task_id
)
- return await self.create_custom_task(user_id, file_paths, processor, original_filenames, existing_task_id=existing_task_id)
async def create_langflow_url_upload_task(
self,
@@ -194,11 +210,21 @@ class TaskService:
prevent_outside=prevent_outside,
tweaks=tweaks,
)
- return await self.create_custom_task(owner_user_id, [docs_url], processor, existing_task_id=existing_task_id)
+ return await self.create_custom_task(
+ owner_user_id, [docs_url], processor, existing_task_id=existing_task_id
+ )
- async def create_custom_task(self, user_id: str, items: list, processor, original_filenames: dict | None = None, existing_task_id: str = None) -> str:
+ async def create_custom_task(
+ self,
+ user_id: str,
+ items: list,
+ processor,
+ original_filenames: dict | None = None,
+ existing_task_id: str = None,
+ ) -> str:
"""Create a new task with custom processor for any type of items"""
import os
+
# Store anonymous tasks under a stable key so they can be retrieved later
store_user_id = user_id or AnonymousUser().user_id
task_id = existing_task_id or str(uuid.uuid4())
@@ -210,14 +236,16 @@ class TaskService:
file_tasks = {
str(item): FileTask(
file_path=str(item),
- filename=normalized_originals.get(
- str(item), os.path.basename(str(item))
- ),
+ filename=normalized_originals.get(str(item), os.path.basename(str(item))),
)
for item in items
}
- if existing_task_id and store_user_id in self.task_store and existing_task_id in self.task_store[store_user_id]:
+ if (
+ existing_task_id
+ and store_user_id in self.task_store
+ and existing_task_id in self.task_store[store_user_id]
+ ):
upload_task = self.task_store[store_user_id][existing_task_id]
upload_task.file_tasks.update(file_tasks)
upload_task.total_files += len(items)
@@ -252,7 +280,7 @@ class TaskService:
metadata={
"total_files": len(items),
"processor_type": processor.__class__.__name__,
- }
+ },
)
)
@@ -334,7 +362,7 @@ class TaskService:
# Add timeout protection to prevent indefinite hangs
await self._process_with_timeout(
processor.process_item(upload_task, item, file_task),
- timeout_seconds=self.ingestion_timeout
+ timeout_seconds=self.ingestion_timeout,
)
logger.info(
@@ -421,6 +449,7 @@ class TaskService:
if upload_task.successful_files > 0:
try:
from config.settings import clients, get_index_name
+
await clients.opensearch.indices.refresh(index=get_index_name())
except Exception as e:
logger.debug("Index refresh after ingest failed (non-fatal)", error=str(e))
@@ -460,7 +489,7 @@ class TaskService:
"total_files": upload_task.total_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
- }
+ },
)
)
@@ -527,7 +556,7 @@ class TaskService:
"processed_files": upload_task.processed_files,
"successful_files": upload_task.successful_files,
"failed_files": upload_task.failed_files,
- }
+ },
)
)
else:
@@ -578,6 +607,9 @@ class TaskService:
"updated_at": file_task.updated_at,
"duration_seconds": file_task.duration_seconds,
"filename": file_task.filename,
+ "phase": file_task.phase.value,
+ "docling_status": file_task.docling_status.value,
+ "docling_task_id": file_task.docling_task_id,
}
# Count running and pending files
@@ -633,6 +665,9 @@ class TaskService:
"updated_at": file_task.updated_at,
"duration_seconds": file_task.duration_seconds,
"filename": file_task.filename,
+ "phase": file_task.phase.value,
+ "docling_status": file_task.docling_status.value,
+ "docling_task_id": file_task.docling_task_id,
}
if file_task.status.value == "running":
@@ -682,8 +717,10 @@ class TaskService:
for task_id in list(self.task_store[user_id].keys()):
task = self.task_store[user_id][task_id]
# Only cleanup completed or failed tasks that are old enough
- if (task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED] and
- current_time - task.updated_at > max_age_seconds):
+ if (
+ task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]
+ and current_time - task.updated_at > max_age_seconds
+ ):
del self.task_store[user_id][task_id]
# Clean up the associated lock
self._task_locks.pop(task_id, None)
@@ -692,7 +729,7 @@ class TaskService:
"Cleaned up old task",
task_id=task_id,
user_id=user_id,
- age_seconds=current_time - task.updated_at
+ age_seconds=current_time - task.updated_at,
)
# Remove empty user entries
@@ -731,10 +768,7 @@ class TaskService:
return False
# Cancel the background task to stop scheduling new work
- if (
- hasattr(upload_task, "background_task")
- and not upload_task.background_task.done()
- ):
+ if hasattr(upload_task, "background_task") and not upload_task.background_task.done():
upload_task.background_task.cancel()
# Wait for the background task to actually stop to avoid race conditions
try:
@@ -792,5 +826,6 @@ class TaskService:
# Log any unexpected errors (not CancelledError)
for i, result in enumerate(results):
if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
- logger.warning("Background task raised exception during shutdown", error=str(result))
-
+ logger.warning(
+ "Background task raised exception during shutdown", error=str(result)
+ )
diff --git a/tests/unit/test_docling_polling_service.py b/tests/unit/test_docling_polling_service.py
new file mode 100644
index 000000000..803e01bb8
--- /dev/null
+++ b/tests/unit/test_docling_polling_service.py
@@ -0,0 +1,186 @@
+"""Unit tests for DoclingPollingService.
+
+Verifies the backend-side polling coordinator that replaces Langflow's
+in-flow Docling polling. The coordinator's job is to wait for terminal
+Docling state without ever invoking Langflow, so that Langflow execution
+slots are reserved for chunking / embedding / indexing only.
+"""
+
+import pytest
+from unittest.mock import AsyncMock, patch
+
+from services.docling_polling_service import (
+ DoclingPollingService,
+ PollOutcome,
+)
+from services.docling_service import DoclingStatusSnapshot, DoclingTaskState
+
+
+def _snap(state: DoclingTaskState, detail: str | None = None) -> DoclingStatusSnapshot:
+ return DoclingStatusSnapshot(state=state, detail=detail)
+
+
+def mock_docling_service():
+ svc = AsyncMock()
+ return svc
+
+
+def polling_service(mock_docling_service):
+ return DoclingPollingService(mock_docling_service)
+
+
[email protected](autouse=True)
+def no_sleep():
+ with patch(
+ "services.docling_polling_service.asyncio.sleep", new_callable=AsyncMock
+ ) as mock_sleep:
+ yield mock_sleep
+
+
+async def test_returns_success_immediately_when_already_done(polling_service, mock_docling_service):
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.SUCCESS)
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1", poll_interval=1.0, max_seconds=10.0
+ )
+
+ assert result.outcome == PollOutcome.SUCCESS
+ assert mock_docling_service.check_task_status.call_count == 1
+
+
+async def test_loops_through_processing_then_success(
+ polling_service, mock_docling_service, no_sleep
+):
+ mock_docling_service.check_task_status.side_effect = [
+ _snap(DoclingTaskState.PENDING),
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.SUCCESS),
+ ]
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1", poll_interval=1.0, max_seconds=60.0
+ )
+
+ assert result.outcome == PollOutcome.SUCCESS
+ assert mock_docling_service.check_task_status.call_count == 4
+ assert no_sleep.call_count == 3
+
+
+async def test_returns_failed_on_docling_failure(polling_service, mock_docling_service):
+ mock_docling_service.check_task_status.return_value = _snap(
+ DoclingTaskState.FAILED, detail="conversion error"
+ )
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1", poll_interval=1.0, max_seconds=10.0
+ )
+
+ assert result.outcome == PollOutcome.FAILED
+ assert "conversion error" in (result.detail or "")
+
+
+async def test_tolerates_brief_not_found_then_succeeds(
+ polling_service, mock_docling_service, no_sleep
+):
+ """NOT_FOUND immediately after submit is a known race; absorb it."""
+ mock_docling_service.check_task_status.side_effect = [
+ _snap(DoclingTaskState.NOT_FOUND),
+ _snap(DoclingTaskState.NOT_FOUND),
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.SUCCESS),
+ ]
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1",
+ poll_interval=1.0,
+ max_seconds=60.0,
+ transient_retry_budget=5,
+ )
+
+ assert result.outcome == PollOutcome.SUCCESS
+
+
+async def test_returns_expired_when_not_found_exceeds_budget(
+ polling_service, mock_docling_service, no_sleep
+):
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.NOT_FOUND)
+
+ result = await polling_service.poll_until_ready(
+ task_id="t1",
+ poll_interval=1.0,
+ max_seconds=60.0,
+ transient_retry_budget=2,
+ )
+
+ assert result.outcome == PollOutcome.EXPIRED
+ assert "not found" in (result.detail or "").lower()
+
+
+async def test_returns_timeout_when_max_seconds_exceeded(
+ polling_service, mock_docling_service, no_sleep
+):
+ mock_docling_service.check_task_status.return_value = _snap(DoclingTaskState.PROCESSING)
+
+ # Loop counter — break ourselves after enough iterations to simulate
+ # crossing the deadline. We achieve this by having the patched
+ # asyncio.sleep advance time past the deadline.
+ import time
+
+ real_monotonic = time.monotonic
+ base = real_monotonic()
+ counter = {"n": 0}
+
+ def fake_monotonic():
+ counter["n"] += 1
+ # First call is "start"; subsequent calls advance fast.
+ return base + counter["n"] * 5.0
+
+ with patch("services.docling_polling_service.time.monotonic", fake_monotonic):
+ result = await polling_service.poll_until_ready(
+ task_id="t1", poll_interval=1.0, max_seconds=2.0
+ )
+
+ assert result.outcome == PollOutcome.TIMEOUT
+
+
+async def test_invalid_arguments(polling_service):
+ with pytest.raises(ValueError):
+ await polling_service.poll_until_ready(task_id="t1", poll_interval=0, max_seconds=10)
+ with pytest.raises(ValueError):
+ await polling_service.poll_until_ready(task_id="t1", poll_interval=1.0, max_seconds=0)
+
+
+async def test_backoff_grows_interval_up_to_cap(polling_service, mock_docling_service, no_sleep):
+ """Verify the sleep interval grows until it hits max_interval."""
+ mock_docling_service.check_task_status.side_effect = [
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.PROCESSING),
+ _snap(DoclingTaskState.SUCCESS),
+ ]
+
+ await polling_service.poll_until_ready(
+ task_id="t1",
+ poll_interval=1.0,
+ max_seconds=120.0,
+ max_interval=4.0,
+ backoff_factor=2.0,
+ )
+
+ # Sleeps after 4 non-success checks: 1, 2, 4, 4 (capped). Actual sleep
+ # call args are min(interval, remaining) — remaining is large, so we
+ # see the raw progression.
+ sleeps = [call.args[0] for call in no_sleep.call_args_list]
+ assert sleeps == [1.0, 2.0, 4.0, 4.0]
diff --git a/tests/unit/test_docling_service_status_check.py b/tests/unit/test_docling_service_status_check.py
new file mode 100644
index 000000000..98fea2c5a
--- /dev/null
+++ b/tests/unit/test_docling_service_status_check.py
@@ -0,0 +1,158 @@
+"""Unit tests for DoclingService.check_task_status and fetch_task_result.
+
+These are the single-poll primitives that the backend's polling coordinator
+uses instead of the legacy in-method polling loop.
+"""
+
+import pytest
+from unittest.mock import AsyncMock, MagicMock
+import httpx
+from services.docling_service import (
+ DoclingService,
+ DoclingServeError,
+ DoclingTaskState,
+)
+
+
+def _resp(status_code: int, json_data: dict | None = None, text: str = "") -> MagicMock:
+ r = MagicMock(spec=httpx.Response)
+ r.status_code = status_code
+ r.text = text
+ if json_data is not None:
+ r.json.return_value = json_data
+ else:
+ r.json.side_effect = ValueError("no json")
+ return r
+
+
+def mock_client():
+ c = AsyncMock(spec=httpx.AsyncClient)
+ c.__aenter__.return_value = c
+ return c
+
+
+def docling_service(mock_client):
+ return DoclingService(docling_url="http://docling:8000", httpx_client=mock_client)
+
+
+# ── check_task_status ─────────────────────────────────────────────────
+
+
+async def test_check_status_success(docling_service, mock_client):
+ mock_client.get.return_value = _resp(200, {"task_status": "success"})
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.SUCCESS
+
+
+async def test_check_status_failure(docling_service, mock_client):
+ mock_client.get.return_value = _resp(200, {"task_status": "failure", "error": "boom"})
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.FAILED
+ assert "boom" in (snap.detail or "")
+
+
+async def test_check_status_processing(docling_service, mock_client):
+ mock_client.get.return_value = _resp(200, {"task_status": "started"})
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.PROCESSING
+
+
+async def test_check_status_unknown_state_treated_as_pending(docling_service, mock_client):
+ mock_client.get.return_value = _resp(200, {"task_status": "queued"})
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.PENDING
+
+
+async def test_check_status_404_returns_not_found(docling_service, mock_client):
+ mock_client.get.return_value = _resp(404, text="not found")
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.NOT_FOUND
+
+
+async def test_check_status_5xx_treated_as_processing(docling_service, mock_client):
+ """5xx is transient — surface as PROCESSING so the polling loop retries."""
+ mock_client.get.return_value = _resp(503, text="bad gateway")
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.PROCESSING
+
+
+async def test_check_status_4xx_other_treated_as_failed(docling_service, mock_client):
+ mock_client.get.return_value = _resp(400, text="bad request")
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.FAILED
+
+
+async def test_check_status_network_error_treated_as_processing(docling_service, mock_client):
+ mock_client.get.side_effect = httpx.ConnectError("connection refused")
+
+ snap = await docling_service.check_task_status("t1")
+
+ assert snap.state == DoclingTaskState.PROCESSING
+
+
+# ── fetch_task_result ─────────────────────────────────────────────────
+
+
+async def test_fetch_result_success(docling_service, mock_client):
+ mock_client.get.return_value = _resp(200, {"document": {"json_content": {"k": "v"}}})
+
+ out = await docling_service.fetch_task_result("t1")
+
+ assert out == {"k": "v"}
+
+
+async def test_fetch_result_404_raises(docling_service, mock_client):
+ mock_client.get.return_value = _resp(404, text="not found")
+
+ with pytest.raises(DoclingServeError, match="not found"):
+ await docling_service.fetch_task_result("t1")
+
+
+async def test_fetch_result_5xx_raises(docling_service, mock_client):
+ mock_client.get.return_value = _resp(500, text="internal")
+
+ with pytest.raises(DoclingServeError, match="HTTP 500"):
+ await docling_service.fetch_task_result("t1")
+
+
+async def test_fetch_result_missing_json_content_raises(docling_service, mock_client):
+ mock_client.get.return_value = _resp(200, {"document": {}})
+
+ with pytest.raises(DoclingServeError, match="missing document.json_content"):
+ await docling_service.fetch_task_result("t1")
+
+
+async def test_fetch_result_network_error_raises(docling_service, mock_client):
+ mock_client.get.side_effect = httpx.ConnectError("dns error")
+
+ with pytest.raises(DoclingServeError, match="Network error"):
+ await docling_service.fetch_task_result("t1")
diff --git a/tests/unit/test_langflow_file_service_two_phase.py b/tests/unit/test_langflow_file_service_two_phase.py
new file mode 100644
index 000000000..5e2a6217e
--- /dev/null
+++ b/tests/unit/test_langflow_file_service_two_phase.py
@@ -0,0 +1,276 @@
+"""Unit tests for the two-phase Docling + Langflow ingestion flow.
+
+Verifies the core architectural property: when a polling service is provided,
+``LangflowFileService.upload_and_ingest_file`` must NOT invoke the Langflow
+ingestion flow until Docling reports SUCCESS, and must NEVER invoke it when
+Docling fails / expires / times out.
+"""
+
+import pytest
+from unittest.mock import AsyncMock, patch
+
+from models.tasks import (
+ DoclingPhaseStatus,
+ FileTask,
+ IngestionPhase,
+)
+from services.docling_polling_service import DoclingPollResult, PollOutcome
+from services.langflow_file_service import LangflowFileService
+
+
+def file_tuple():
+ return ("test.pdf", b"PDFDATA", "application/pdf")
+
+
+def file_task():
+ return FileTask(file_path="/tmp/test.pdf", filename="test.pdf")
+
+
+def mock_docling_service():
+ svc = AsyncMock()
+ svc.upload_to_docling_direct_async.return_value = "task-abc-123"
+ return svc
+
+
+def mock_polling_service():
+ return AsyncMock()
+
+
+def langflow_service(mock_docling_service):
+ svc = LangflowFileService(docling_service=mock_docling_service)
+ # Stub the actual Langflow HTTP call — those have their own coverage.
+ svc.run_ingestion_flow = AsyncMock(return_value={"status": "ok"})
+ return svc
+
+
+async def test_two_phase_success_invokes_langflow_with_task_id(
+ langflow_service, mock_polling_service, file_tuple, file_task
+):
+ mock_polling_service.poll_until_ready.return_value = DoclingPollResult(
+ outcome=PollOutcome.SUCCESS, elapsed_seconds=2.5
+ )
+
+ result = await langflow_service.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=mock_polling_service,
+ file_task=file_task,
+ )
+
+ # Langflow was invoked exactly once, with the docling_task_id forwarded.
+ assert langflow_service.run_ingestion_flow.call_count == 1
+ kwargs = langflow_service.run_ingestion_flow.call_args.kwargs
+ assert kwargs["docling_task_id"] == "task-abc-123"
+
+ # FileTask reflects full lifecycle.
+ assert file_task.docling_task_id == "task-abc-123"
+ assert file_task.docling_status == DoclingPhaseStatus.SUCCESS
+ assert file_task.phase == IngestionPhase.COMPLETE
+
+ # Result envelope.
+ assert result["status"] == "success"
+ assert result["docling_task_id"] == "task-abc-123"
+
+
+async def test_langflow_not_invoked_on_docling_failure(
+ langflow_service, mock_polling_service, file_tuple, file_task
+):
+ mock_polling_service.poll_until_ready.return_value = DoclingPollResult(
+ outcome=PollOutcome.FAILED, detail="OCR engine crashed"
+ )
+
+ with pytest.raises(Exception, match="OCR engine crashed"):
+ await langflow_service.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=mock_polling_service,
+ file_task=file_task,
+ )
+
+ # Crucial assertion — Langflow must never run when Docling failed.
+ assert langflow_service.run_ingestion_flow.call_count == 0
+ assert file_task.docling_status == DoclingPhaseStatus.FAILED
+ # Phase remains DOCLING (never advanced to LANGFLOW).
+ assert file_task.phase == IngestionPhase.DOCLING
+
+
+async def test_langflow_not_invoked_on_docling_expired(
+ langflow_service, mock_polling_service, file_tuple, file_task
+):
+ mock_polling_service.poll_until_ready.return_value = DoclingPollResult(
+ outcome=PollOutcome.EXPIRED, detail="task not found"
+ )
+
+ with pytest.raises(Exception, match="expired"):
+ await langflow_service.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=mock_polling_service,
+ file_task=file_task,
+ )
+
+ assert langflow_service.run_ingestion_flow.call_count == 0
+ assert file_task.docling_status == DoclingPhaseStatus.EXPIRED
+
+
+async def test_langflow_not_invoked_on_polling_timeout(
+ langflow_service, mock_polling_service, file_tuple, file_task
+):
+ mock_polling_service.poll_until_ready.return_value = DoclingPollResult(
+ outcome=PollOutcome.TIMEOUT, detail="exceeded 1800s"
+ )
+
+ with pytest.raises(Exception, match="timeout"):
+ await langflow_service.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=mock_polling_service,
+ file_task=file_task,
+ )
+
+ assert langflow_service.run_ingestion_flow.call_count == 0
+ assert file_task.docling_status == DoclingPhaseStatus.FAILED
+
+
+async def test_phase_progresses_only_after_polling_succeeds(
+ langflow_service, mock_polling_service, file_tuple, file_task
+):
+ """Phase must be DOCLING during polling, then LANGFLOW, then COMPLETE."""
+ observed_phases = []
+
+ async def record_then_succeed(*args, **kwargs):
+ observed_phases.append(file_task.phase)
+ return DoclingPollResult(outcome=PollOutcome.SUCCESS)
+
+ mock_polling_service.poll_until_ready.side_effect = record_then_succeed
+
+ async def record_then_run(*args, **kwargs):
+ observed_phases.append(file_task.phase)
+ return {"status": "ok"}
+
+ langflow_service.run_ingestion_flow = AsyncMock(side_effect=record_then_run)
+
+ await langflow_service.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=mock_polling_service,
+ file_task=file_task,
+ )
+
+ assert observed_phases == [IngestionPhase.DOCLING, IngestionPhase.LANGFLOW]
+ assert file_task.phase == IngestionPhase.COMPLETE
+
+
+async def test_legacy_path_without_polling_service_calls_langflow_directly(
+ langflow_service, file_tuple, file_task
+):
+ """Backward compatibility: when no polling service is provided, Langflow
+ is invoked immediately after Docling submission (Langflow handles polling).
+ On success the file_task must end with docling_status=SUCCESS so the
+ status fields stay coherent — Langflow returning success implies its
+ DoclingRemote component consumed the task.
+ """
+ result = await langflow_service.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=None,
+ file_task=file_task,
+ )
+
+ assert langflow_service.run_ingestion_flow.call_count == 1
+ kwargs = langflow_service.run_ingestion_flow.call_args.kwargs
+ assert kwargs["docling_task_id"] == "task-abc-123"
+ assert result["status"] == "success"
+ # Status fields must not leave the task stuck in PROCESSING.
+ assert file_task.docling_status == DoclingPhaseStatus.SUCCESS
+ assert file_task.phase == IngestionPhase.COMPLETE
+ assert file_task.docling_task_id == "task-abc-123"
+
+
+def test_processor_default_polling_service_is_none():
+ """LangflowFileProcessor no longer constructs a polling service inline.
+ The container injects it; absent injection, the processor falls back
+ to the legacy single-call path.
+ """
+ from models.processors import LangflowFileProcessor
+
+ lf_svc = LangflowFileService(docling_service=AsyncMock())
+ processor = LangflowFileProcessor(
+ langflow_file_service=lf_svc,
+ session_manager=None,
+ )
+ assert processor.docling_polling_service is None
+
+
+def test_processor_accepts_injected_polling_service():
+ """The polling service is passed through unchanged when injected."""
+ from models.processors import LangflowFileProcessor
+
+ lf_svc = LangflowFileService(docling_service=AsyncMock())
+ injected = AsyncMock()
+ processor = LangflowFileProcessor(
+ langflow_file_service=lf_svc,
+ session_manager=None,
+ docling_polling_service=injected,
+ )
+ assert processor.docling_polling_service is injected
+
+
+async def test_task_service_threads_polling_service_to_processor(monkeypatch):
+ """TaskService.create_langflow_upload_task must forward its injected
+ polling service to the LangflowFileProcessor it constructs. This is the
+ DI contract that replaces the processor's inline construction.
+ """
+ from services.task_service import TaskService
+
+ injected = AsyncMock()
+ captured: dict = {}
+
+ class FakeProcessor:
+ def __init__(self, **kwargs):
+ captured.update(kwargs)
+
+ monkeypatch.setattr("models.processors.LangflowFileProcessor", FakeProcessor)
+
+ async def fake_create_custom_task(*args, **kwargs):
+ return "task-id"
+
+ svc = TaskService(docling_polling_service=injected)
+ monkeypatch.setattr(svc, "create_custom_task", fake_create_custom_task)
+
+ await svc.create_langflow_upload_task(
+ user_id="u1",
+ file_paths=["/tmp/x.pdf"],
+ langflow_file_service=AsyncMock(),
+ session_manager=AsyncMock(),
+ )
+
+ assert captured["docling_polling_service"] is injected
+
+
+async def test_docling_submit_failure_skips_polling_and_langflow(
+ mock_docling_service, mock_polling_service, file_tuple, file_task
+):
+ mock_docling_service.upload_to_docling_direct_async.side_effect = Exception(
+ "docling unreachable"
+ )
+ svc = LangflowFileService(docling_service=mock_docling_service)
+ svc.run_ingestion_flow = AsyncMock()
+
+ with pytest.raises(Exception, match="Docling upload failed"):
+ await svc.upload_and_ingest_file(
+ file_tuple=file_tuple,
+ docling_polling_service=mock_polling_service,
+ file_task=file_task,
+ )
+
+ assert mock_polling_service.poll_until_ready.call_count == 0
+ assert svc.run_ingestion_flow.call_count == 0
+ assert file_task.docling_task_id is None