Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

15.8 KB
Raw Blame
"""Workflow Orchestration Utilities for Worker Tasks

This module provides standardized workflow orchestration patterns,
chord execution, batch processing, and task coordination utilities.
"""

from __future__ import annotations

import os
from typing import TYPE_CHECKING, Any

from queue_backend import BarrierHandle, FairnessKey
from queue_backend.pg_barrier import PgBarrier

from unstract.core.data_models import ExecutionStatus

from ...enums import FileDestinationType, PipelineType
from ...enums.worker_enums import QueueName
from ...infrastructure.logging import WorkerLogger

if TYPE_CHECKING:
    from celery.canvas import Signature

logger = WorkerLogger.get_logger(__name__)


class WorkflowOrchestrationUtils:
    """Centralized workflow orchestration patterns and utilities."""

    @staticmethod
    def record_pg_orchestration_failure(
        *,
        api_client: Any,
        execution_id: str,
        total_files: int,
        error_message: str,
        logger: Any,
        workflow_logger: Any | None = None,
    ) -> None:
        """Record an orchestration failure: surface the error to the UI and
        reconcile the file counters, both best-effort.

        Two things happen, neither of which may disturb the caller's control flow
        (it still has the original orchestration exception to log / re-raise):

        * **(C)** if a ``workflow_logger`` is given, publish the error to the UI /
          WebSocket logs (guarded — a logging hiccup must not abort the rest).
        * **(B)** mark the attempted files failed via ``update_status`` so the run
          reads "N failed" not "N in progress" (UI derives in-progress as
          ``total - successful - failed``). ``total_files`` is sent alongside the
          aggregates because the backend serializer requires it ("total_files is
          required when file aggregates are provided") and enforces
          ``successful + failed <= total`` — both hold here (0 + N <= N). The call
          is wrapped: a status-update failure (e.g. a serializer/validation error)
          is logged but **not** re-raised, so it can never mask the real
          orchestration error or skip the caller's re-raise (greptile P1).
        """
        if workflow_logger:
            try:
                workflow_logger.log_error(
                    logger, f"❌ Workflow orchestration failed: {error_message}"
                )
            except Exception as log_error:
                logger.warning(f"Failed to publish error to UI logs: {log_error}")
        try:
            api_client.update_workflow_execution_status(
                execution_id=execution_id,
                status=ExecutionStatus.ERROR.value,
                error_message=error_message,
                total_files=total_files,
                successful_files=0,
                failed_files=total_files,
            )
        except Exception as status_error:
            logger.error(
                f"Failed to write ERROR status + file counts for execution "
                f"{execution_id}: {status_error}",
                exc_info=True,
            )

    @staticmethod
    def create_chord_execution(
        batch_tasks: list[Signature],
        callback_task_name: str,
        callback_kwargs: dict[str, Any],
        callback_queue: str,
        app_instance: Any,
        *,
        fairness: FairnessKey | None = None,
    ) -> BarrierHandle | None:
        """Standardized fan-out + callback pattern.

        Routes through :class:`PgBarrier`, which arms a ``pg_barrier_state`` row
        and dispatches the header tasks onto the PG queue. A fresh instance per
        call: the barrier holds no cross-execution state, and the substrate is no
        longer selectable, so there is nothing for a module-level singleton to
        memoise.

        Args:
            batch_tasks: List of batch task signatures
            callback_task_name: Name of callback task
            callback_kwargs: Keyword arguments for callback
            callback_queue: Queue name for callback task
            app_instance: Celery app instance
            fairness: Optional ``FairnessKey`` attached as a message
                header on every header task and the callback — closes
                the chord-fairness gap that Phase 5.1 (``dispatch()``)
                deliberately scoped out. See ``queue_backend.fairness``
                for the slot name constant.

        Returns:
            Barrier handle (``.id`` exposed for logging) or None if no
            batch tasks.

        Note:
            CRITICAL: Returns None for zero batch tasks, signaling to
            parent that direct pipeline status updates should be
            handled instead.
        """
        return PgBarrier().enqueue(
            batch_tasks,
            callback_task_name=callback_task_name,
            callback_kwargs=callback_kwargs,
            callback_queue=callback_queue,
            app_instance=app_instance,
            fairness=fairness,
        )

    @staticmethod
    def determine_manual_review_routing(
        files: dict[str, Any],
        manual_review_config: dict[str, Any] | None = None,
        default_destination: str = FileDestinationType.DESTINATION.value,
    ) -> dict[str, str]:
        """Determine manual review routing for files based on configuration.

        Args:
            files: Dictionary of files to route
            manual_review_config: Manual review configuration
            default_destination: Default destination if no manual review

        Returns:
            Dictionary mapping file keys to destinations

        Note:
            This consolidates the complex manual review decision logic found
            across multiple workers.
        """
        routing = {}
        manual_review_required = False

        # Check if manual review is globally enabled
        if manual_review_config:
            manual_review_required = manual_review_config.get("enabled", False)

        for file_key, file_data in files.items():
            if manual_review_required:
                # Additional per-file checks could go here
                routing[file_key] = FileDestinationType.MANUALREVIEW.value
                logger.debug(f"File {file_key} routed to manual review")
            else:
                routing[file_key] = default_destination
                logger.debug(f"File {file_key} routed to {default_destination}")

        if manual_review_required:
            logger.info(f"Manual review routing: {len(files)} files routed for review")

        return routing

    @staticmethod
    def create_batch_task_signatures(
        batch_files: list[Any],
        task_name: str,
        base_kwargs: dict[str, Any],
        queue_name: str,
        app_instance: Any,
    ) -> list[Any]:
        """Create standardized batch task signatures.

        Args:
            batch_files: List of file batches
            task_name: Name of task to execute
            base_kwargs: Base keyword arguments for all tasks
            queue_name: Queue name for task execution
            app_instance: Celery app instance

        Returns:
            List of task signatures

        Note:
            This standardizes batch task signature creation across workers.
        """
        signatures = []

        for batch_index, batch in enumerate(batch_files):
            batch_kwargs = base_kwargs.copy()
            batch_kwargs.update(
                {
                    "batch_files": batch,
                    "batch_index": batch_index,
                    "total_batches": len(batch_files),
                }
            )

            signature = app_instance.signature(
                task_name, kwargs=batch_kwargs, queue=queue_name
            )
            signatures.append(signature)

        logger.info(
            f"Created {len(signatures)} batch task signatures for {task_name} "
            f"on queue {queue_name}"
        )

        return signatures

    @staticmethod
    def calculate_batch_processing_metrics(
        total_files: int, batch_size: int, processing_time_seconds: float | None = None
    ) -> dict[str, int | float]:
        """Calculate batch processing metrics for monitoring and optimization.

        Args:
            total_files: Total number of files processed
            batch_size: Size of each batch
            processing_time_seconds: Total processing time

        Returns:
            Dictionary of metrics

        Note:
            This provides consistent metrics calculation across orchestration.
        """
        num_batches = (total_files + batch_size - 1) // batch_size  # Ceiling division
        avg_files_per_batch = total_files / num_batches if num_batches > 0 else 0

        metrics = {
            "total_files": total_files,
            "batch_size": batch_size,
            "num_batches": num_batches,
            "avg_files_per_batch": avg_files_per_batch,
        }

        if processing_time_seconds is not None:
            metrics.update(
                {
                    "processing_time_seconds": processing_time_seconds,
                    "avg_time_per_batch": processing_time_seconds / num_batches
                    if num_batches > 0
                    else 0,
                    "avg_time_per_file": processing_time_seconds / total_files
                    if total_files > 0
                    else 0,
                }
            )

        return metrics

    @staticmethod
    def determine_callback_queue(
        workflow_type: str, default_queue: str = "celery"
    ) -> str:
        """Determine appropriate callback queue based on workflow type.

        Args:
            workflow_type: Type of workflow being processed
            default_queue: Default queue if no specific mapping

        Returns:
            Queue name for callback processing

        Note:
            This centralizes queue determination logic found across workers.
        """
        # Map workflow types to specific queues using enums
        queue_mapping = {
            PipelineType.API.value: QueueName.API_DEPLOYMENTS.value,
            PipelineType.ETL.value: QueueName.GENERAL.value,
            PipelineType.TASK.value: QueueName.GENERAL.value,
            PipelineType.APP.value: QueueName.GENERAL.value,
        }

        # Check for environment-specific overrides
        env_queue = os.getenv(f"CALLBACK_QUEUE_{workflow_type}")
        if env_queue:
            logger.info(
                f"Using environment-specified queue for {workflow_type}: {env_queue}"
            )
            return env_queue

        queue = queue_mapping.get(workflow_type.upper(), default_queue)
        logger.debug(f"Determined callback queue for {workflow_type}: {queue}")

        return queue

    @staticmethod
    def create_callback_signature_data(
        execution_id: str,
        workflow_id: str,
        organization_id: str,
        additional_context: dict[str, Any] | None = None,
    ) -> dict[str, Any]:
        """Create standardized callback signature data.

        Args:
            execution_id: Workflow execution ID
            workflow_id: Workflow ID
            organization_id: Organization ID
            additional_context: Additional context data

        Returns:
            Dictionary of callback signature data

        Note:
            This standardizes callback signature creation across workers.
        """
        callback_data = {
            "execution_id": execution_id,
            "workflow_id": workflow_id,
            "organization_id": organization_id,
            "callback_metadata": {
                "created_at": None,  # Will be set by callback handler
                "orchestrator": "WorkflowOrchestrationUtils",
            },
        }

        if additional_context:
            callback_data.update(additional_context)

        return callback_data

    @staticmethod
    def validate_orchestration_parameters(
        execution_id: str,
        workflow_id: str,
        organization_id: str,
        files: dict[str, Any] | None = None,
    ) -> None:
        """Validate common orchestration parameters.

        Args:
            execution_id: Workflow execution ID
            workflow_id: Workflow ID
            organization_id: Organization ID
            files: Optional files dictionary

        Raises:
            ValueError: If validation fails

        Note:
            This provides consistent parameter validation across orchestration.
        """
        if not execution_id:
            raise ValueError("execution_id is required for workflow orchestration")

        if not workflow_id:
            raise ValueError("workflow_id is required for workflow orchestration")

        if not organization_id:
            raise ValueError("organization_id is required for workflow orchestration")

        if files is not None and not isinstance(files, dict):
            raise ValueError("files must be a dictionary when provided")

        logger.debug(
            f"Orchestration parameters validated - "
            f"exec_id={execution_id}, workflow_id={workflow_id}, org_id={organization_id}"
        )


class WorkflowOrchestrationMixin:
    """Mixin class to add orchestration utilities to worker tasks."""

    def create_chord(
        self,
        batch_tasks,
        callback_task_name,
        callback_kwargs,
        callback_queue,
        *,
        fairness: FairnessKey | None = None,
    ):
        """Create chord using standardized pattern.

        Forwards ``fairness`` to ``create_chord_execution`` so mixin
        callers can opt into the chord-fairness plumbing that bare
        ``dispatch()`` sites get via Phase 5.1. ``None`` keeps the
        pre-Barrier wire shape exactly.
        """
        # Get app instance from task context
        app_instance = getattr(self, "app", None)
        if not app_instance:
            raise RuntimeError("Celery app instance not available in task context")

        return WorkflowOrchestrationUtils.create_chord_execution(
            batch_tasks,
            callback_task_name,
            callback_kwargs,
            callback_queue,
            app_instance,
            fairness=fairness,
        )

    def determine_manual_review_routing(self, files, manual_review_config=None):
        """Determine manual review routing using standardized logic."""
        return WorkflowOrchestrationUtils.determine_manual_review_routing(
            files, manual_review_config
        )

    def create_batch_signatures(self, batch_files, task_name, base_kwargs, queue_name):
        """Create batch task signatures using standardized pattern."""
        app_instance = getattr(self, "app", None)
        if not app_instance:
            raise RuntimeError("Celery app instance not available in task context")

        return WorkflowOrchestrationUtils.create_batch_task_signatures(
            batch_files, task_name, base_kwargs, queue_name, app_instance
        )

    def calculate_metrics(self, total_files, batch_size, processing_time=None):
        """Calculate processing metrics using standardized calculation."""
        return WorkflowOrchestrationUtils.calculate_batch_processing_metrics(
            total_files, batch_size, processing_time
        )

    def determine_callback_queue(self, workflow_type, default_queue="celery"):
        """Determine callback queue using standardized logic."""
        return WorkflowOrchestrationUtils.determine_callback_queue(
            workflow_type, default_queue
        )

    def create_callback_data(self, execution_id, workflow_id, organization_id, **kwargs):
        """Create callback signature data using standardized format."""
        return WorkflowOrchestrationUtils.create_callback_signature_data(
            execution_id, workflow_id, organization_id, kwargs
        )

    def validate_parameters(self, execution_id, workflow_id, organization_id, files=None):
        """Validate orchestration parameters using standardized validation."""
        WorkflowOrchestrationUtils.validate_orchestration_parameters(
            execution_id, workflow_id, organization_id, files
        )