LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2850 [FIX] Add INPROGRESS notification for scheduled ETL executions and adjust status mapping (#1562)
ali26f423cea587
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex 6de7002a6..eb26ed749 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -473,6 +473,7 @@ class NotificationStatus(Enum):COMPLETED = "COMPLETED"ERROR = "ERROR"STOPPED = "STOPPED"+ INPROGRESS = "INPROGRESS"def __str__(self):"""Return enum value for Django CharField compatibility."""@@ -487,6 +488,7 @@ class NotificationSource(Enum):PIPELINE_COMPLETION = "pipeline-completion"API_EXECUTION = "api-execution"MANUAL_TRIGGER = "manual-trigger"+ SCHEDULED_EXECUTION = "scheduled-execution"def __str__(self):"""Return enum value for Django CharField compatibility."""@@ -586,7 +588,11 @@ class NotificationPayload:"""# Map execution status to notification statusif execution_status in [ExecutionStatus.COMPLETED]:- notification_status = NotificationStatus.COMPLETED+ if workflow_type in [WorkflowType.API]:+ notification_status = NotificationStatus.COMPLETED+ else:+ # For Backward compatibility+ notification_status = NotificationStatus.SUCCESSelif execution_status in [ExecutionStatus.ERROR]:notification_status = NotificationStatus.ERRORelif execution_status in [ExecutionStatus.STOPPED]:diff --git a/workers/notification/providers/slack_webhook.py b/workers/notification/providers/slack_webhook.pyindex cb80cfb68..89206646a 100644--- a/workers/notification/providers/slack_webhook.py+++ b/workers/notification/providers/slack_webhook.py@@ -101,13 +101,13 @@ class SlackWebhook(WebhookProvider):# Format value based on typeformatted_value = self._format_value(value)- # Create section block+ # Create section block with inline formatblocks.append({"type": "section","text": {"type": "mrkdwn",- "text": f"*{formatted_key}:*\n{formatted_value}",+ "text": f"*{formatted_key}:* {formatted_value}",},})@@ -179,8 +179,8 @@ class SlackWebhook(WebhookProvider):"Status": "Status","Error": "Error","Success": "Success",- "Execution Id": "Execution ID",- "Organization Id": "Organization ID",+ "Execution Id": "Execution Id",+ "Organization Id": "Organization Id",}return key_mapping.get(formatted, formatted)diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.pyindex a1a2aa78e..2e22946a5 100644--- a/workers/scheduler/tasks.py+++ b/workers/scheduler/tasks.py@@ -19,8 +19,11 @@ from shared.models.scheduler_models import (SchedulerExecutionResult,WorkflowExecutionRequest,)+from shared.patterns.notification.helper import trigger_notificationfrom shared.utils.api_client_singleton import get_singleton_api_client+from unstract.core.data_models import NotificationPayload, NotificationSource+# Import the exact backend logic to ensure consistencylogger = WorkerLogger.get_logger(__name__)@@ -29,6 +32,63 @@ logger = WorkerLogger.get_logger(__name__)config = WorkerConfig.from_env("SCHEDULER")+def _send_pipeline_status_notification(+ api_client,+ pipeline_id: str,+ pipeline_name: str,+ pipeline_type: str,+ status: str,+ organization_id: str,+ execution_id: str | None = None,+) -> None:+ """Send notification for pipeline status change.++ Non-critical operation - logs warning on failure but doesn't raise.++ Args:+ api_client: Internal API client instance+ pipeline_id: Pipeline identifier+ pipeline_name: Pipeline name+ pipeline_type: Pipeline type (ETL, TASK, etc.)+ status: Pipeline status (INPROGRESS, SUCCESS, FAILURE)+ organization_id: Organization context+ execution_id: Optional execution identifier+ """+ # Validate execution status for notifications+ try:+ notification = NotificationPayload(+ type=pipeline_type,+ pipeline_id=pipeline_id,+ pipeline_name=pipeline_name,+ status=status,+ execution_id=execution_id,+ organization_id=organization_id,+ _source=NotificationSource.SCHEDULED_EXECUTION,+ )+ except ValueError as e:+ logger.warning(f"Cannot create notification payload: {e}")+ return++ logger.info(+ f"Processing notification for {pipeline_type} {pipeline_id} with status {status}"+ )++ # Use proper notification configuration lookup based on workflow type+ try:+ # For ETL/TASK/other pipeline types+ trigger_notification(+ api_client=api_client,+ pipeline_id=pipeline_id,+ pipeline_name=pipeline_name,+ notification_payload=notification,+ )+ logger.info(f"Notification sent successfully for {pipeline_type} {pipeline_id}")+ except Exception as notification_error:+ logger.warning(+ f"Failed to send notification for {pipeline_type} {pipeline_id}: {notification_error}"+ )++def _execute_scheduled_workflow(api_client,context: ScheduledPipelineContext,@@ -250,6 +310,17 @@ def execute_pipeline_task_v2(logger.info(f"Successfully updated pipeline {pipeline_id} status to {PipelineStatus.INPROGRESS}")++ # Send INPROGRESS notification when scheduled execution starts+ _send_pipeline_status_notification(+ api_client=api_client,+ pipeline_id=pipeline_id,+ pipeline_name=pipeline_name_from_api,+ pipeline_type=pipeline_data.pipeline_type,+ status=PipelineStatus.INPROGRESS.value,+ organization_id=organization_id,+ execution_id=None,+ )except Exception as e:logger.warning(f"Failed to update pipeline status to INPROGRESS: {e}")# Don't fail the entire execution for status update failuresdiff --git a/workers/shared/patterns/notification/helper.py b/workers/shared/patterns/notification/helper.pyindex a56a6ade7..977f5a875 100644--- a/workers/shared/patterns/notification/helper.py+++ b/workers/shared/patterns/notification/helper.py@@ -99,6 +99,61 @@ def send_notification_to_worker(return False+def trigger_notification(+ api_client,+ pipeline_id: str,+ pipeline_name: str,+ notification_payload: NotificationPayload,+) -> None:+ """Trigger notifications for pipeline status updates.++ Called by callback worker after successful status update.+ Uses API client to fetch notification configuration.+ """+ try:+ # Fetch pipeline notifications via API+ response_data = api_client._make_request(+ method="GET",+ endpoint=f"v1/webhook/pipeline/{pipeline_id}/notifications/",+ timeout=10,+ )++ # _make_request already handles status codes and returns parsed data+ # If we get here, the request was successful (status 200)+ notifications_data = response_data.get("notifications", [])+ active_notifications = [+ n for n in notifications_data if n.get("is_active", False)+ ]++ if not active_notifications:+ logger.info(f"No active notifications found for pipeline {pipeline_id}")+ return++ logger.info(+ f"Sending {len(active_notifications)} notifications for pipeline {pipeline_name}"+ )++ # Send each notification+ for notification in active_notifications:+ if notification.get("notification_type") == "WEBHOOK":+ send_notification_to_worker(+ url=notification["url"],+ payload=notification_payload,+ auth_type=notification.get("authorization_type", "NONE"),+ auth_key=notification.get("authorization_key"),+ auth_header=notification.get("authorization_header"),+ max_retries=notification.get("max_retries", 0),+ platform=notification.get("platform"),+ )+ else:+ logger.debug(+ f"Skipping non-webhook notification type: {notification.get('notification_type')}"+ )++ except Exception as e:+ logger.error(f"Error triggering pipeline notifications for {pipeline_id}: {e}")++def trigger_pipeline_notifications(api_client,pipeline_id: str,