LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Workflow Orchestration Utilities for Worker Tasks
This module provides standardized workflow orchestration patterns,
chord execution, batch processing, and task coordination utilities.
"""
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any
from queue_backend import BarrierHandle, FairnessKey
from queue_backend.pg_barrier import PgBarrier
from unstract.core.data_models import ExecutionStatus
from ...enums import FileDestinationType, PipelineType
from ...enums.worker_enums import QueueName
from ...infrastructure.logging import WorkerLogger
if TYPE_CHECKING:
from celery.canvas import Signature
logger = WorkerLogger.get_logger(__name__)
class WorkflowOrchestrationUtils:
"""Centralized workflow orchestration patterns and utilities."""
@staticmethod
def record_pg_orchestration_failure(
*,
api_client: Any,
execution_id: str,
total_files: int,
error_message: str,
logger: Any,
workflow_logger: Any | None = None,
) -> None:
"""Record an orchestration failure: surface the error to the UI and
reconcile the file counters, both best-effort.
Two things happen, neither of which may disturb the caller's control flow
(it still has the original orchestration exception to log / re-raise):
* **(C)** if a ``workflow_logger`` is given, publish the error to the UI /
WebSocket logs (guarded — a logging hiccup must not abort the rest).
* **(B)** mark the attempted files failed via ``update_status`` so the run
reads "N failed" not "N in progress" (UI derives in-progress as
``total - successful - failed``). ``total_files`` is sent alongside the
aggregates because the backend serializer requires it ("total_files is
required when file aggregates are provided") and enforces
``successful + failed <= total`` — both hold here (0 + N <= N). The call
is wrapped: a status-update failure (e.g. a serializer/validation error)
is logged but **not** re-raised, so it can never mask the real
orchestration error or skip the caller's re-raise (greptile P1).
"""
if workflow_logger:
try:
workflow_logger.log_error(
logger, f"❌ Workflow orchestration failed: {error_message}"
)
except Exception as log_error:
logger.warning(f"Failed to publish error to UI logs: {log_error}")
try:
api_client.update_workflow_execution_status(
execution_id=execution_id,
status=ExecutionStatus.ERROR.value,
error_message=error_message,
total_files=total_files,
successful_files=0,
failed_files=total_files,
)
except Exception as status_error:
logger.error(
f"Failed to write ERROR status + file counts for execution "
f"{execution_id}: {status_error}",
exc_info=True,
)
@staticmethod
def create_chord_execution(
batch_tasks: list[Signature],
callback_task_name: str,
callback_kwargs: dict[str, Any],
callback_queue: str,
app_instance: Any,
*,
fairness: FairnessKey | None = None,
) -> BarrierHandle | None:
"""Standardized fan-out + callback pattern.
Routes through :class:`PgBarrier`, which arms a ``pg_barrier_state`` row
and dispatches the header tasks onto the PG queue. A fresh instance per
call: the barrier holds no cross-execution state, and the substrate is no
longer selectable, so there is nothing for a module-level singleton to
memoise.
Args:
batch_tasks: List of batch task signatures
callback_task_name: Name of callback task
callback_kwargs: Keyword arguments for callback
callback_queue: Queue name for callback task
app_instance: Celery app instance
fairness: Optional ``FairnessKey`` attached as a message
header on every header task and the callback — closes
the chord-fairness gap that Phase 5.1 (``dispatch()``)
deliberately scoped out. See ``queue_backend.fairness``
for the slot name constant.
Returns:
Barrier handle (``.id`` exposed for logging) or None if no
batch tasks.
Note:
CRITICAL: Returns None for zero batch tasks, signaling to
parent that direct pipeline status updates should be
handled instead.
"""
return PgBarrier().enqueue(
batch_tasks,
callback_task_name=callback_task_name,
callback_kwargs=callback_kwargs,
callback_queue=callback_queue,
app_instance=app_instance,
fairness=fairness,
)
@staticmethod
def determine_manual_review_routing(
files: dict[str, Any],
manual_review_config: dict[str, Any] | None = None,
default_destination: str = FileDestinationType.DESTINATION.value,
) -> dict[str, str]:
"""Determine manual review routing for files based on configuration.
Args:
files: Dictionary of files to route
manual_review_config: Manual review configuration
default_destination: Default destination if no manual review
Returns:
Dictionary mapping file keys to destinations
Note:
This consolidates the complex manual review decision logic found
across multiple workers.
"""
routing = {}
manual_review_required = False
# Check if manual review is globally enabled
if manual_review_config:
manual_review_required = manual_review_config.get("enabled", False)
for file_key, file_data in files.items():
if manual_review_required:
# Additional per-file checks could go here
routing[file_key] = FileDestinationType.MANUALREVIEW.value
logger.debug(f"File {file_key} routed to manual review")
else:
routing[file_key] = default_destination
logger.debug(f"File {file_key} routed to {default_destination}")
if manual_review_required:
logger.info(f"Manual review routing: {len(files)} files routed for review")
return routing
@staticmethod
def create_batch_task_signatures(
batch_files: list[Any],
task_name: str,
base_kwargs: dict[str, Any],
queue_name: str,
app_instance: Any,
) -> list[Any]:
"""Create standardized batch task signatures.
Args:
batch_files: List of file batches
task_name: Name of task to execute
base_kwargs: Base keyword arguments for all tasks
queue_name: Queue name for task execution
app_instance: Celery app instance
Returns:
List of task signatures
Note:
This standardizes batch task signature creation across workers.
"""
signatures = []
for batch_index, batch in enumerate(batch_files):
batch_kwargs = base_kwargs.copy()
batch_kwargs.update(
{
"batch_files": batch,
"batch_index": batch_index,
"total_batches": len(batch_files),
}
)
signature = app_instance.signature(
task_name, kwargs=batch_kwargs, queue=queue_name
)
signatures.append(signature)
logger.info(
f"Created {len(signatures)} batch task signatures for {task_name} "
f"on queue {queue_name}"
)
return signatures
@staticmethod
def calculate_batch_processing_metrics(
total_files: int, batch_size: int, processing_time_seconds: float | None = None
) -> dict[str, int | float]:
"""Calculate batch processing metrics for monitoring and optimization.
Args:
total_files: Total number of files processed
batch_size: Size of each batch
processing_time_seconds: Total processing time
Returns:
Dictionary of metrics
Note:
This provides consistent metrics calculation across orchestration.
"""
num_batches = (total_files + batch_size - 1) // batch_size # Ceiling division
avg_files_per_batch = total_files / num_batches if num_batches > 0 else 0
metrics = {
"total_files": total_files,
"batch_size": batch_size,
"num_batches": num_batches,
"avg_files_per_batch": avg_files_per_batch,
}
if processing_time_seconds is not None:
metrics.update(
{
"processing_time_seconds": processing_time_seconds,
"avg_time_per_batch": processing_time_seconds / num_batches
if num_batches > 0
else 0,
"avg_time_per_file": processing_time_seconds / total_files
if total_files > 0
else 0,
}
)
return metrics
@staticmethod
def determine_callback_queue(
workflow_type: str, default_queue: str = "celery"
) -> str:
"""Determine appropriate callback queue based on workflow type.
Args:
workflow_type: Type of workflow being processed
default_queue: Default queue if no specific mapping
Returns:
Queue name for callback processing
Note:
This centralizes queue determination logic found across workers.
"""
# Map workflow types to specific queues using enums
queue_mapping = {
PipelineType.API.value: QueueName.API_DEPLOYMENTS.value,
PipelineType.ETL.value: QueueName.GENERAL.value,
PipelineType.TASK.value: QueueName.GENERAL.value,
PipelineType.APP.value: QueueName.GENERAL.value,
}
# Check for environment-specific overrides
env_queue = os.getenv(f"CALLBACK_QUEUE_{workflow_type}")
if env_queue:
logger.info(
f"Using environment-specified queue for {workflow_type}: {env_queue}"
)
return env_queue
queue = queue_mapping.get(workflow_type.upper(), default_queue)
logger.debug(f"Determined callback queue for {workflow_type}: {queue}")
return queue
@staticmethod
def create_callback_signature_data(
execution_id: str,
workflow_id: str,
organization_id: str,
additional_context: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Create standardized callback signature data.
Args:
execution_id: Workflow execution ID
workflow_id: Workflow ID
organization_id: Organization ID
additional_context: Additional context data
Returns:
Dictionary of callback signature data
Note:
This standardizes callback signature creation across workers.
"""
callback_data = {
"execution_id": execution_id,
"workflow_id": workflow_id,
"organization_id": organization_id,
"callback_metadata": {
"created_at": None, # Will be set by callback handler
"orchestrator": "WorkflowOrchestrationUtils",
},
}
if additional_context:
callback_data.update(additional_context)
return callback_data
@staticmethod
def validate_orchestration_parameters(
execution_id: str,
workflow_id: str,
organization_id: str,
files: dict[str, Any] | None = None,
) -> None:
"""Validate common orchestration parameters.
Args:
execution_id: Workflow execution ID
workflow_id: Workflow ID
organization_id: Organization ID
files: Optional files dictionary
Raises:
ValueError: If validation fails
Note:
This provides consistent parameter validation across orchestration.
"""
if not execution_id:
raise ValueError("execution_id is required for workflow orchestration")
if not workflow_id:
raise ValueError("workflow_id is required for workflow orchestration")
if not organization_id:
raise ValueError("organization_id is required for workflow orchestration")
if files is not None and not isinstance(files, dict):
raise ValueError("files must be a dictionary when provided")
logger.debug(
f"Orchestration parameters validated - "
f"exec_id={execution_id}, workflow_id={workflow_id}, org_id={organization_id}"
)
class WorkflowOrchestrationMixin:
"""Mixin class to add orchestration utilities to worker tasks."""
def create_chord(
self,
batch_tasks,
callback_task_name,
callback_kwargs,
callback_queue,
*,
fairness: FairnessKey | None = None,
):
"""Create chord using standardized pattern.
Forwards ``fairness`` to ``create_chord_execution`` so mixin
callers can opt into the chord-fairness plumbing that bare
``dispatch()`` sites get via Phase 5.1. ``None`` keeps the
pre-Barrier wire shape exactly.
"""
# Get app instance from task context
app_instance = getattr(self, "app", None)
if not app_instance:
raise RuntimeError("Celery app instance not available in task context")
return WorkflowOrchestrationUtils.create_chord_execution(
batch_tasks,
callback_task_name,
callback_kwargs,
callback_queue,
app_instance,
fairness=fairness,
)
def determine_manual_review_routing(self, files, manual_review_config=None):
"""Determine manual review routing using standardized logic."""
return WorkflowOrchestrationUtils.determine_manual_review_routing(
files, manual_review_config
)
def create_batch_signatures(self, batch_files, task_name, base_kwargs, queue_name):
"""Create batch task signatures using standardized pattern."""
app_instance = getattr(self, "app", None)
if not app_instance:
raise RuntimeError("Celery app instance not available in task context")
return WorkflowOrchestrationUtils.create_batch_task_signatures(
batch_files, task_name, base_kwargs, queue_name, app_instance
)
def calculate_metrics(self, total_files, batch_size, processing_time=None):
"""Calculate processing metrics using standardized calculation."""
return WorkflowOrchestrationUtils.calculate_batch_processing_metrics(
total_files, batch_size, processing_time
)
def determine_callback_queue(self, workflow_type, default_queue="celery"):
"""Determine callback queue using standardized logic."""
return WorkflowOrchestrationUtils.determine_callback_queue(
workflow_type, default_queue
)
def create_callback_data(self, execution_id, workflow_id, organization_id, **kwargs):
"""Create callback signature data using standardized format."""
return WorkflowOrchestrationUtils.create_callback_signature_data(
execution_id, workflow_id, organization_id, kwargs
)
def validate_parameters(self, execution_id, workflow_id, organization_id, files=None):
"""Validate orchestration parameters using standardized validation."""
WorkflowOrchestrationUtils.validate_orchestration_parameters(
execution_id, workflow_id, organization_id, files
)