LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Scheduler Worker Tasks
This worker handles scheduled pipeline executions, migrated from @backend/scheduler/tasks.py
to support the new workers architecture while maintaining backward compatibility.
"""
import traceback
from typing import Any
from queue_backend import FairnessKey, QueueBackend, dispatch, worker_task
from queue_backend.fairness import WorkloadType
# Register the dashboard-metrics proxy tasks on this worker type (UN-3796). Imported
# purely for the side effect.
#
# ABSOLUTE package import, and it must stay that way — this module is reached by TWO
# different mechanisms and only this form survives both:
# 1. worker.py loads /app/scheduler/tasks.py BY PATH with the worker dir on sys.path.
# No parent package is set, so `from . import ...` cannot resolve.
# 2. general/tasks.py does `from scheduler.tasks import execute_pipeline_task_v2`,
# importing it as a package module. Then /app/scheduler is NOT on sys.path, so a
# bare `import dashboard_metrics_tasks` raises ModuleNotFoundError and crash-loops
# worker-general (this happened — the bare form shipped and broke the general
# worker at flag-off, where PG is not even involved).
# `/app` is on PYTHONPATH (run-worker-docker.sh), so `scheduler.` resolves under both.
#
# The same reasoning covers agent_kv_tasks, which registers the Agent-KV
# sweep/TTL-cleanup periodics (spec §5.4) on this worker type.
from scheduler import agent_kv_tasks, dashboard_metrics_tasks, inbox_tasks
from shared.enums.status_enums import PipelineStatus
from shared.enums.worker_enums import QueueName
from shared.infrastructure.config import WorkerConfig
from shared.infrastructure.logging import WorkerLogger
from shared.models.pipeline_models import PipelineApiResponse
from shared.models.scheduler_models import (
ExecutionMode,
ScheduledPipelineContext,
SchedulerExecutionResult,
WorkflowExecutionRequest,
)
from shared.patterns.notification.helper import trigger_notification
from shared.utils.api_client_singleton import get_singleton_api_client
from unstract.core.data_models import (
LEGACY_TRANSPORT_KEY,
LEGACY_TRANSPORT_VALUE,
NotificationPayload,
NotificationSource,
)
# Import the exact backend logic to ensure consistency
# Both modules above are imported ONLY for the side effect of registering their
# @worker_task entries on this worker; nothing here calls into them. Naming them
# in a module-level tuple is what keeps them imported.
#
# A bare `noqa` directive is NOT enough, and that is not a style preference — it
# reproduced cause of this exact line being deleted once already. Two hooks
# interact:
# 1. ruff's isort (I001) merges two `from scheduler import X` statements into
# one parenthesised statement, which relocates each trailing noqa directive onto
# a MEMBER line.
# 2. F401 is reported against the statement, whose first line now carries no
# directive, so pycln (`[tool.pycln] all = true` — it removes side-effect
# imports by design) deletes both members.
# That is what happened in a79e9d69: pre-commit.ci silently dropped BOTH this
# import and the pre-existing dashboard-metrics one, un-registering the
# dashboard-metrics periodics that had been live since UN-3796 as collateral.
# A genuine reference cannot be fixed away by either hook.
#
# Deleting this tuple re-arms that failure, and it fails SILENTLY: the worker
# starts fine and only the scheduled messages are rejected as unknown tasks.
_PERIODIC_TASK_MODULES = (dashboard_metrics_tasks, agent_kv_tasks, inbox_tasks)
logger = WorkerLogger.get_logger(__name__)
# Initialize worker configuration
config = WorkerConfig.from_env("SCHEDULER")
def _send_pipeline_status_notification(
api_client,
pipeline_id: str,
pipeline_name: str,
pipeline_type: str,
status: str,
organization_id: str,
execution_id: str | None = None,
) -> None:
"""Send notification for pipeline status change.
Non-critical operation - logs warning on failure but doesn't raise.
Args:
api_client: Internal API client instance
pipeline_id: Pipeline identifier
pipeline_name: Pipeline name
pipeline_type: Pipeline type (ETL, TASK, etc.)
status: Pipeline status (INPROGRESS, SUCCESS, FAILURE)
organization_id: Organization context
execution_id: Optional execution identifier
"""
# Validate execution status for notifications
try:
notification = NotificationPayload(
type=pipeline_type,
pipeline_id=pipeline_id,
pipeline_name=pipeline_name,
status=status,
execution_id=execution_id,
organization_id=organization_id,
_source=NotificationSource.SCHEDULED_EXECUTION,
)
except ValueError as e:
logger.warning(f"Cannot create notification payload: {e}")
return
logger.info(
f"Processing notification for {pipeline_type} {pipeline_id} with status {status}"
)
# Use proper notification configuration lookup based on workflow type
try:
# For ETL/TASK/other pipeline types
trigger_notification(
api_client=api_client,
pipeline_id=pipeline_id,
pipeline_name=pipeline_name,
notification_payload=notification,
execution_id=execution_id,
)
logger.info(f"Notification sent successfully for {pipeline_type} {pipeline_id}")
except Exception as notification_error:
logger.warning(
f"Failed to send notification for {pipeline_type} {pipeline_id}: {notification_error}"
)
def _execute_scheduled_workflow(
api_client,
context: ScheduledPipelineContext,
) -> SchedulerExecutionResult:
"""Execute scheduled workflow using worker-native logic with type safety.
This replaces the Django-heavy backend complete_execution method with
a worker-native implementation that uses internal APIs and dataclasses.
Args:
api_client: Internal API client instance
context: Scheduled pipeline execution context
Returns:
SchedulerExecutionResult with execution status and details
"""
try:
logger.info(
f"Creating workflow execution for scheduled pipeline: {context.pipeline_name}"
)
# Step 1: Create workflow execution via internal API using dataclass
# For scheduled executions, let backend handle execution_log_id (falls back to pipeline_id)
# This matches the backend logic: log_events_id if provided, else pipeline_id
execution_request = WorkflowExecutionRequest(
workflow_id=context.workflow_id,
pipeline_id=context.pipeline_id,
organization_id=context.organization_id,
single_step=False,
mode=ExecutionMode.QUEUE,
total_files=0, # Will be updated during execution
scheduled=True,
# log_events_id=None - let backend fall back to pipeline_id for scheduled executions
)
workflow_execution = api_client.create_workflow_execution(
execution_request.to_dict()
)
execution_id = workflow_execution.get("execution_id")
if not execution_id:
return SchedulerExecutionResult.error(
error="Failed to create workflow execution",
workflow_id=context.workflow_id,
pipeline_id=context.pipeline_id,
)
logger.info(
f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Created workflow execution for scheduled pipeline {context.pipeline_name}"
)
# Step 2: Trigger async workflow execution
logger.info(
f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Triggering async execution for workflow {context.workflow_id}"
)
logger.info(
f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Dispatching async_execute_bin task for scheduled execution"
)
try:
async_result = dispatch(
"async_execute_bin",
args=[
context.organization_id,
context.workflow_id,
execution_id,
{},
True, # scheduled
],
kwargs={
"use_file_history": context.use_file_history,
"pipeline_id": context.pipeline_id,
# Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.
LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,
},
queue=QueueName.GENERAL,
# Stated explicitly rather than left to select_backend(): the whole
# execution pipeline is pinned to one substrate at its entry point.
backend=QueueBackend.PG,
fairness=FairnessKey(
org_id=context.organization_id,
workload_type=WorkloadType.NON_API,
),
)
task_id = async_result.id
logger.info(
f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Successfully dispatched async_execute_bin task {task_id} for scheduled execution"
)
execution_response = SchedulerExecutionResult.success(
execution_id=execution_id,
workflow_id=context.workflow_id,
pipeline_id=context.pipeline_id,
task_id=task_id,
message="Async execution task dispatched successfully",
)
except Exception as e:
logger.error(f"Failed to dispatch async execution task: {e}")
execution_response = SchedulerExecutionResult.error(
error=f"Failed to dispatch async execution: {str(e)}",
execution_id=execution_id,
workflow_id=context.workflow_id,
pipeline_id=context.pipeline_id,
)
if execution_response.is_success:
logger.info(
f"Successfully started scheduled execution {execution_id} for pipeline '{context.pipeline_name}'"
)
return execution_response # Already a SchedulerExecutionResult
else:
logger.error(
f"Failed to start async execution for pipeline '{context.pipeline_name}': {execution_response.error}"
)
return execution_response # Already a SchedulerExecutionResult with error
except Exception as e:
logger.error(f"Exception in scheduled workflow execution: {e}")
return SchedulerExecutionResult.error(
error=f"Scheduler execution failed: {str(e)}",
workflow_id=context.workflow_id,
pipeline_id=context.pipeline_id,
)
@worker_task(name="scheduler.tasks.execute_pipeline_task", bind=True)
def execute_pipeline_task(
self,
workflow_id: Any,
org_schema: Any,
execution_action: Any,
execution_id: Any,
pipepline_id: Any, # Note: keeping original typo for compatibility
with_logs: Any,
name: Any,
) -> None:
"""Execute pipeline task - maintains exact signature from backend scheduler.
This is the main entry point for scheduled pipeline executions, delegating
to the v2 implementation for actual processing.
"""
return execute_pipeline_task_v2(
organization_id=org_schema,
pipeline_id=pipepline_id,
pipeline_name=name,
)
@worker_task(name="execute_pipeline_task_v2", bind=True)
def execute_pipeline_task_v2(
self,
organization_id: Any,
pipeline_id: Any,
pipeline_name: Any,
) -> None:
"""V2 of execute_pipeline method - worker implementation.
This method replicates the exact logic from backend/scheduler/tasks.py
but uses worker clients instead of direct Django ORM access.
Args:
organization_id: Organization identifier
pipeline_id: UID of pipeline entity
pipeline_name: Pipeline name for logging
"""
try:
# Initialize API client with organization context
api_client = get_singleton_api_client(config)
api_client.set_organization_context(organization_id)
logger.info(
f"Executing scheduled pipeline: {pipeline_id}, "
f"organization: {organization_id}, pipeline name: {pipeline_name}"
)
# Fetch pipeline data via API client with type safety
try:
pipeline_response = api_client.get_pipeline_data(
pipeline_id=pipeline_id, check_active=True
)
if not pipeline_response.success:
logger.error(
f"Failed to fetch pipeline {pipeline_id}: {pipeline_response.error}"
)
return
# Parse response using type-safe dataclass
pipeline_api_data = PipelineApiResponse.from_dict(pipeline_response.data)
pipeline_data = pipeline_api_data.pipeline
# Use dataclass properties for type-safe access
workflow_id = pipeline_data.workflow_id
pipeline_name_from_api = pipeline_data.pipeline_name
logger.info(
f"Found pipeline '{pipeline_name_from_api}' with workflow {workflow_id} "
f"for pipeline ID {pipeline_id}"
)
except Exception as e:
logger.error(
f"Error fetching or parsing pipeline data for {pipeline_id}: {e}"
)
return
# Check subscription if validation is enabled
# Note: In workers, we'll skip subscription validation for now as it requires
# backend plugins. This can be added later via internal API if needed.
logger.debug("Skipping subscription validation in worker context")
# Update pipeline status to INPROGRESS when scheduled execution starts
try:
logger.info(
f"Updating pipeline {pipeline_id} status to {PipelineStatus.INPROGRESS}"
)
api_client.update_pipeline_status(
pipeline_id=pipeline_id,
status=PipelineStatus.INPROGRESS.value,
organization_id=organization_id,
)
logger.info(
f"Successfully updated pipeline {pipeline_id} status to {PipelineStatus.INPROGRESS}"
)
# Send INPROGRESS notification when scheduled execution starts
_send_pipeline_status_notification(
api_client=api_client,
pipeline_id=pipeline_id,
pipeline_name=pipeline_name_from_api,
pipeline_type=pipeline_data.pipeline_type,
status=PipelineStatus.INPROGRESS.value,
organization_id=organization_id,
execution_id=None,
)
except Exception as e:
logger.warning(f"Failed to update pipeline status to INPROGRESS: {e}")
# Don't fail the entire execution for status update failures
# Implement scheduler logic directly in worker using type-safe dataclasses
# This replaces the Django-heavy backend complete_execution method
try:
# Create execution context using dataclass
context = ScheduledPipelineContext(
pipeline_id=pipeline_id,
pipeline_name=pipeline_name_from_api,
workflow_id=workflow_id,
organization_id=organization_id,
use_file_history=True, # Always true for scheduled executions
)
execution_result = _execute_scheduled_workflow(
api_client=api_client,
context=context,
)
if execution_result.is_success:
logger.info(
f"[exec:{execution_result.execution_id}] [pipeline:{pipeline_id}] Scheduled execution task dispatched successfully for pipeline '{pipeline_name_from_api}' "
f"in organization {organization_id}"
)
# Pipeline status will be updated to COMPLETED/FAILED by the actual workflow execution
else:
logger.error(
f"[exec:{execution_result.execution_id}] [pipeline:{pipeline_id}] Failed to dispatch scheduled execution for pipeline '{pipeline_name_from_api}': {execution_result.error}"
)
# Update pipeline status to FAILED since we couldn't even start the execution
try:
api_client.update_pipeline_status(
pipeline_id=pipeline_id,
status=PipelineStatus.FAILURE.value,
organization_id=organization_id,
)
logger.info(
f"Updated pipeline {pipeline_id} status to {PipelineStatus.FAILURE} due to dispatch failure"
)
except Exception as e:
logger.warning(f"Failed to update pipeline status to FAILED: {e}")
except Exception as e:
logger.error(
f"Error during scheduled workflow execution for pipeline '{pipeline_name_from_api}': {e}"
)
# Update pipeline status to FAILED due to scheduler error
try:
api_client.update_pipeline_status(
pipeline_id=pipeline_id,
status=PipelineStatus.FAILURE.value,
organization_id=organization_id,
)
logger.info(
f"Updated pipeline {pipeline_id} status to {PipelineStatus.FAILURE} due to scheduler exception"
)
except Exception as status_error:
logger.warning(
f"Failed to update pipeline status to FAILED: {status_error}"
)
raise
except Exception as e:
logger.error(
f"Failed to execute pipeline: {pipeline_name}. Error: {e}"
f"\n\n'''{traceback.format_exc()}```"
)
# Update pipeline status to FAILED for top-level scheduler errors
try:
api_client = get_singleton_api_client(config)
api_client.set_organization_context(
organization_id if "organization_id" in locals() else None
)
if "pipeline_id" in locals() and pipeline_id:
api_client.update_pipeline_status(
pipeline_id=pipeline_id,
status=PipelineStatus.FAILURE.value,
organization_id=organization_id
if "organization_id" in locals()
else None,
)
logger.info(
f"Updated pipeline {pipeline_id} status to {PipelineStatus.FAILURE} due to top-level scheduler error"
)
except Exception as status_error:
logger.warning(
f"Failed to update pipeline status to FAILED in outer exception: {status_error}"
)
# Health check task for monitoring
@worker_task(name="scheduler_health_check")
def health_check() -> dict[str, Any]:
"""Health check task for scheduler worker.
Returns:
Health status information
"""
try:
# Check API client connectivity
api_client = get_singleton_api_client(config)
api_status = "healthy" if api_client else "unhealthy"
except Exception as e:
api_status = f"unhealthy: {e}"
return {
"worker": "scheduler",
"status": "healthy" if "healthy" in api_status else "degraded",
"api": api_status,
"config": {
"queue": config.queue_name,
},
}