Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

5.4 KB
Raw Blame
import logging
from typing import Any

from pipeline_v2.models import Pipeline
from rest_framework.serializers import ValidationError
from utils.user_context import UserContext
from workflow_manager.workflow_v2.constants import WorkflowKey
from workflow_manager.workflow_v2.serializers import ExecuteWorkflowSerializer

from scheduler.constants import SchedulerConstants as SC
from scheduler.exceptions import JobDeletionError, JobSchedulingError
from scheduler.ownership import reconcile_ownership_for
from scheduler.serializer import AddJobSerializer
from scheduler.tasks import (
    create_or_update_periodic_task,
    delete_periodic_task,
    disable_task,
    enable_task,
    mirror_periodic_schedule_upsert,
)

logger = logging.getLogger(__name__)


class SchedulerHelper:
    @staticmethod
    def _schedule_task_job(pipeline: Pipeline, job_data: Any) -> None:
        if "cron_string" not in job_data:
            raise ValidationError("cron_string is required in job_data")
        if "id" not in job_data:
            raise ValidationError("id is required in job_data")
        if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):
            raise ValidationError("job_kwargs is required to be a dict")

        cron_string = job_data.get("cron_string")
        name = job_data.get("name")
        job_kwargs = job_data.get("job_kwargs", {})
        task_data = job_kwargs.get("data", {})

        task_data[WorkflowKey.WF_ID] = pipeline.workflow.id
        serializer = ExecuteWorkflowSerializer(data=task_data)
        serializer.is_valid(raise_exception=True)
        workflow_id = serializer.get_workflow_id(serializer.validated_data)
        # TODO: Remove unused argument in execute_pipeline_task
        execution_action = serializer.get_execution_action(serializer.validated_data)
        organization_id = UserContext.get_organization_identifier()

        create_or_update_periodic_task(
            cron_string=cron_string,
            task_name=str(pipeline.pk),
            task_path="scheduler.tasks.execute_pipeline_task",
            task_args=[
                str(workflow_id),
                organization_id,
                execution_action or "",
                # TODO: execution_id parameter cannot be removed without a migration.
                "",
                str(pipeline.pk),
                # Added to remain backward compatible - remove after data migration
                # which removes unused args in execute_pipeline_task
                False,
                str(name),
            ],
            enabled=pipeline.active,
        )
        # Inert PG mirror (Phase 9, ②a): sourced from the Pipeline object here so
        # it stores the real pipeline_name + ids (the PeriodicTask args carry the
        # synthetic "Pipeline job-<id>" label at index 6, not the user name).
        # Best-effort — never raises, so Beat scheduling is unaffected.
        mirror_periodic_schedule_upsert(
            pipeline_id=str(pipeline.pk),
            organization_id=organization_id or "",
            workflow_id=str(workflow_id),
            pipeline_name=pipeline.pipeline_name,
            cron_string=cron_string,
            enabled=pipeline.active,
        )
        # Align this schedule's firer (Beat vs PG). NOT a no-op: since UN-4046
        # PG_SCHEDULER_ENABLED defaults to true, so the normal path here hands the
        # schedule to PG and disables the Beat PeriodicTask in the same transaction
        # (which is what stops the two both firing). Best-effort (never raises).
        # organization_id is the org identifier string (what the mirror stores).
        reconcile_ownership_for(str(pipeline.pk), organization_id, active=pipeline.active)

    @staticmethod
    def add_or_update_job(pipeline: Pipeline) -> None:
        logger.info(f"Scheduling job for {pipeline}")
        name = f"Pipeline job-{str(pipeline.id)}"
        job_serialize_data = {
            SC.ID: str(pipeline.id),
            SC.CRON_STRING: pipeline.cron_string,
            SC.NAME: name,
        }
        job_serializer = AddJobSerializer(data=job_serialize_data)
        job_serializer.is_valid(raise_exception=True)
        try:
            SchedulerHelper._schedule_task_job(pipeline, job_serializer.validated_data)
        except Exception as e:
            logger.error(f"Exception while adding job: {e}")
            raise JobSchedulingError from e

    @staticmethod
    def remove_job(pipeline_id: str) -> None:
        logger.info(f"Removing job for {pipeline_id}")
        try:
            logger.info("Celery scheduler - removing job")
            delete_periodic_task(pipeline_id)
        except Exception as e:
            logger.error(f"Exception while removing job: {e}")
            raise JobDeletionError from e

    @staticmethod
    def pause_job(pipeline_id: str) -> None:
        logger.info(f"Pausing job for {pipeline_id}")
        try:
            logger.info("Celery scheduler - pausing job")
            disable_task(pipeline_id)
        except Exception as e:
            logger.error(f"Exception while pausing job: {e}")
            raise JobSchedulingError from e

    @staticmethod
    def resume_job(pipeline_id: str) -> None:
        logger.info(f"Resuming job for {pipeline_id}")
        try:
            logger.info("Celery scheduler - resuming job")
            enable_task(pipeline_id)
        except Exception as e:
            logger.error(f"Exception while resuming job: {e}")
            raise JobSchedulingError from e