Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2850 [FIX] Add INPROGRESS notification for scheduled ETL executions and adjust status mapping (#1562)

ali26f423cea587
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index 6de7002a6..eb26ed749 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -473,6 +473,7 @@ class NotificationStatus(Enum):
COMPLETED = "COMPLETED"
ERROR = "ERROR"
STOPPED = "STOPPED"
+ INPROGRESS = "INPROGRESS"
def __str__(self):
"""Return enum value for Django CharField compatibility."""
@@ -487,6 +488,7 @@ class NotificationSource(Enum):
PIPELINE_COMPLETION = "pipeline-completion"
API_EXECUTION = "api-execution"
MANUAL_TRIGGER = "manual-trigger"
+ SCHEDULED_EXECUTION = "scheduled-execution"
def __str__(self):
"""Return enum value for Django CharField compatibility."""
@@ -586,7 +588,11 @@ class NotificationPayload:
"""
# Map execution status to notification status
if execution_status in [ExecutionStatus.COMPLETED]:
- notification_status = NotificationStatus.COMPLETED
+ if workflow_type in [WorkflowType.API]:
+ notification_status = NotificationStatus.COMPLETED
+ else:
+ # For Backward compatibility
+ notification_status = NotificationStatus.SUCCESS
elif execution_status in [ExecutionStatus.ERROR]:
notification_status = NotificationStatus.ERROR
elif execution_status in [ExecutionStatus.STOPPED]:
diff --git a/workers/notification/providers/slack_webhook.py b/workers/notification/providers/slack_webhook.py
index cb80cfb68..89206646a 100644
--- a/workers/notification/providers/slack_webhook.py
+++ b/workers/notification/providers/slack_webhook.py
@@ -101,13 +101,13 @@ class SlackWebhook(WebhookProvider):
# Format value based on type
formatted_value = self._format_value(value)
- # Create section block
+ # Create section block with inline format
blocks.append(
{
"type": "section",
"text": {
"type": "mrkdwn",
- "text": f"*{formatted_key}:*\n{formatted_value}",
+ "text": f"*{formatted_key}:* {formatted_value}",
},
}
)
@@ -179,8 +179,8 @@ class SlackWebhook(WebhookProvider):
"Status": "Status",
"Error": "Error",
"Success": "Success",
- "Execution Id": "Execution ID",
- "Organization Id": "Organization ID",
+ "Execution Id": "Execution Id",
+ "Organization Id": "Organization Id",
}
return key_mapping.get(formatted, formatted)
diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py
index a1a2aa78e..2e22946a5 100644
--- a/workers/scheduler/tasks.py
+++ b/workers/scheduler/tasks.py
@@ -19,8 +19,11 @@ from shared.models.scheduler_models import (
SchedulerExecutionResult,
WorkflowExecutionRequest,
)
+from shared.patterns.notification.helper import trigger_notification
from shared.utils.api_client_singleton import get_singleton_api_client
+from unstract.core.data_models import NotificationPayload, NotificationSource
+
# Import the exact backend logic to ensure consistency
logger = WorkerLogger.get_logger(__name__)
@@ -29,6 +32,63 @@ logger = WorkerLogger.get_logger(__name__)
config = WorkerConfig.from_env("SCHEDULER")
+def _send_pipeline_status_notification(
+ api_client,
+ pipeline_id: str,
+ pipeline_name: str,
+ pipeline_type: str,
+ status: str,
+ organization_id: str,
+ execution_id: str | None = None,
+) -> None:
+ """Send notification for pipeline status change.
+
+ Non-critical operation - logs warning on failure but doesn't raise.
+
+ Args:
+ api_client: Internal API client instance
+ pipeline_id: Pipeline identifier
+ pipeline_name: Pipeline name
+ pipeline_type: Pipeline type (ETL, TASK, etc.)
+ status: Pipeline status (INPROGRESS, SUCCESS, FAILURE)
+ organization_id: Organization context
+ execution_id: Optional execution identifier
+ """
+ # Validate execution status for notifications
+ try:
+ notification = NotificationPayload(
+ type=pipeline_type,
+ pipeline_id=pipeline_id,
+ pipeline_name=pipeline_name,
+ status=status,
+ execution_id=execution_id,
+ organization_id=organization_id,
+ _source=NotificationSource.SCHEDULED_EXECUTION,
+ )
+ except ValueError as e:
+ logger.warning(f"Cannot create notification payload: {e}")
+ return
+
+ logger.info(
+ f"Processing notification for {pipeline_type} {pipeline_id} with status {status}"
+ )
+
+ # Use proper notification configuration lookup based on workflow type
+ try:
+ # For ETL/TASK/other pipeline types
+ trigger_notification(
+ api_client=api_client,
+ pipeline_id=pipeline_id,
+ pipeline_name=pipeline_name,
+ notification_payload=notification,
+ )
+ logger.info(f"Notification sent successfully for {pipeline_type} {pipeline_id}")
+ except Exception as notification_error:
+ logger.warning(
+ f"Failed to send notification for {pipeline_type} {pipeline_id}: {notification_error}"
+ )
+
+
def _execute_scheduled_workflow(
api_client,
context: ScheduledPipelineContext,
@@ -250,6 +310,17 @@ def execute_pipeline_task_v2(
logger.info(
f"Successfully updated pipeline {pipeline_id} status to {PipelineStatus.INPROGRESS}"
)
+
+ # Send INPROGRESS notification when scheduled execution starts
+ _send_pipeline_status_notification(
+ api_client=api_client,
+ pipeline_id=pipeline_id,
+ pipeline_name=pipeline_name_from_api,
+ pipeline_type=pipeline_data.pipeline_type,
+ status=PipelineStatus.INPROGRESS.value,
+ organization_id=organization_id,
+ execution_id=None,
+ )
except Exception as e:
logger.warning(f"Failed to update pipeline status to INPROGRESS: {e}")
# Don't fail the entire execution for status update failures
diff --git a/workers/shared/patterns/notification/helper.py b/workers/shared/patterns/notification/helper.py
index a56a6ade7..977f5a875 100644
--- a/workers/shared/patterns/notification/helper.py
+++ b/workers/shared/patterns/notification/helper.py
@@ -99,6 +99,61 @@ def send_notification_to_worker(
return False
+def trigger_notification(
+ api_client,
+ pipeline_id: str,
+ pipeline_name: str,
+ notification_payload: NotificationPayload,
+) -> None:
+ """Trigger notifications for pipeline status updates.
+
+ Called by callback worker after successful status update.
+ Uses API client to fetch notification configuration.
+ """
+ try:
+ # Fetch pipeline notifications via API
+ response_data = api_client._make_request(
+ method="GET",
+ endpoint=f"v1/webhook/pipeline/{pipeline_id}/notifications/",
+ timeout=10,
+ )
+
+ # _make_request already handles status codes and returns parsed data
+ # If we get here, the request was successful (status 200)
+ notifications_data = response_data.get("notifications", [])
+ active_notifications = [
+ n for n in notifications_data if n.get("is_active", False)
+ ]
+
+ if not active_notifications:
+ logger.info(f"No active notifications found for pipeline {pipeline_id}")
+ return
+
+ logger.info(
+ f"Sending {len(active_notifications)} notifications for pipeline {pipeline_name}"
+ )
+
+ # Send each notification
+ for notification in active_notifications:
+ if notification.get("notification_type") == "WEBHOOK":
+ send_notification_to_worker(
+ url=notification["url"],
+ payload=notification_payload,
+ auth_type=notification.get("authorization_type", "NONE"),
+ auth_key=notification.get("authorization_key"),
+ auth_header=notification.get("authorization_header"),
+ max_retries=notification.get("max_retries", 0),
+ platform=notification.get("platform"),
+ )
+ else:
+ logger.debug(
+ f"Skipping non-webhook notification type: {notification.get('notification_type')}"
+ )
+
+ except Exception as e:
+ logger.error(f"Error triggering pipeline notifications for {pipeline_id}: {e}")
+
+
def trigger_pipeline_notifications(
api_client,
pipeline_id: str,