Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

6.1 KB
Raw Blame
import logging

from django.core.exceptions import ValidationError
from django.utils import timezone

from pipeline_v2.exceptions import InactivePipelineError
from pipeline_v2.models import Pipeline
from pipeline_v2.notification import PipelineNotification

logger = logging.getLogger(__name__)


class PipelineProcessor:
    @staticmethod
    def initialize_pipeline_sync(pipeline_id: str) -> Pipeline:
        """Fetches and initializes the sync for a pipeline.

        Args:
            pipeline_id (str): UUID of the pipeline to sync
        """
        pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
        pipeline.run_count = pipeline.run_count + 1
        return PipelineProcessor._update_pipeline_status(
            pipeline=pipeline,
            status=Pipeline.PipelineStatus.RESTARTING,
            is_end=False,
        )

    @staticmethod
    def fetch_pipeline(pipeline_id: str, check_active: bool = True) -> Pipeline:
        """Retrieves and checks for an active pipeline.

        Args:
            pipeline_id (str): UUID of the pipeline
            check_active (bool): Whether to check if the pipeline is active

        Raises:
            InactivePipelineError: If an active pipeline is not found
        """
        pipeline: Pipeline = Pipeline.objects.get(pk=pipeline_id)
        if check_active and not pipeline.is_active():
            logger.error(f"Inactive pipeline fetched: {pipeline_id}")
            raise InactivePipelineError(pipeline_name=pipeline.pipeline_name)
        return pipeline

    @classmethod
    def get_active_pipeline(cls, pipeline_id: str) -> Pipeline | None:
        """Retrieve a pipeline, requiring it to be active.

        Raises ``InactivePipelineError`` (422) when the row exists but is
        paused; use :meth:`get_pipeline_by_id` when merely identifying the row.

        A malformed identifier is "not found", not a fault -- see
        :meth:`get_pipeline_by_id` for why ``ValidationError`` is caught here.
        This path is reached *unauthenticated*: the public execution endpoint
        looks the pipeline up before validating the API key, so letting it
        escape turns any non-UUID path segment into a 500.
        """
        try:
            return cls.fetch_pipeline(pipeline_id, check_active=True)
        except Pipeline.DoesNotExist:
            return None
        except ValidationError:
            logger.debug("Malformed pipeline identifier: %s", pipeline_id)
            return None

    @classmethod
    def get_pipeline_by_id(cls, pipeline_id: str) -> Pipeline | None:
        """Retrieve a pipeline regardless of whether it is currently active.

        The active/inactive distinction matters to callers that are about to
        *run* something. Callers that only need to identify the row -- to check
        ownership, say -- must not be forced through ``get_active_pipeline``,
        which raises ``InactivePipelineError`` (422) and logs at ERROR for what
        is an ordinary request against a paused pipeline.

        A malformed identifier is "not found", not a fault: ``pk`` is a UUID
        column, so a non-UUID string raises ``ValidationError`` out of
        ``to_python`` rather than ``DoesNotExist``. Callers reach this with
        unvalidated caller input (path segments are ``<str:>``, and request
        bodies are read before the serializer runs), so letting that escape
        turns ordinary client garbage into a 500.
        """
        try:
            return cls.fetch_pipeline(pipeline_id, check_active=False)
        except Pipeline.DoesNotExist:
            return None
        except ValidationError:
            # Logged so a malformed identifier stays distinguishable from an
            # absent row: both answer 404, and without this the difference is
            # invisible when triaging.
            logger.debug("Malformed pipeline identifier: %s", pipeline_id)
            return None

    @staticmethod
    def _update_pipeline_status(
        pipeline: Pipeline,
        status: tuple[str, str],
        is_end: bool,
        is_active: bool | None = None,
    ) -> Pipeline:
        """Updates pipeline status during execution.

        Raises:
            PipelineSaveError: Exception while saving a pipeline

        Returns:
            Pipeline: Updated pipeline
        """
        if is_end:
            pipeline.last_run_time = timezone.now()
        if status:
            pipeline.last_run_status = status
        if is_active is not None:
            pipeline.active = is_active

        pipeline.save()
        return pipeline

    @staticmethod
    def _send_notification(
        pipeline: Pipeline,
        execution_id: str | None = None,
        error_message: str | None = None,
    ) -> None:
        """Sends a notification for the pipeline.

        Args:
            pipeline (Pipeline): Pipeline to send notification for

        Returns:
            None
        """
        pipeline_notification = PipelineNotification(
            pipeline=pipeline, execution_id=execution_id, error_message=error_message
        )
        pipeline_notification.send()

    @staticmethod
    def update_pipeline(
        pipeline_guid: str | None,
        status: tuple[str, str],
        is_active: bool | None = None,
        execution_id: str | None = None,
        error_message: str | None = None,
        is_end: bool = False,
    ) -> None:
        if not pipeline_guid:
            return
        # Skip check if we are enabling an inactive pipeline
        check_active = not is_active
        pipeline: Pipeline = PipelineProcessor.fetch_pipeline(
            pipeline_id=pipeline_guid, check_active=check_active
        )
        pipeline = PipelineProcessor._update_pipeline_status(
            pipeline=pipeline, is_end=is_end, status=status, is_active=is_active
        )

        # Only send notifications if execution_id is provided
        # This avoids duplicate notifications when called from workers (who handle notifications separately)
        if execution_id:
            PipelineProcessor._send_notification(
                pipeline=pipeline, execution_id=execution_id, error_message=error_message
            )

        logger.info(f"Updated pipeline {pipeline_guid} status: {status}")