Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Async API and some minor fixes (#574)

alif35efedc3345
diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.py
index bc2b8aeff..3866af51d 100644
--- a/backend/api/deployment_helper.py
+++ b/backend/api/deployment_helper.py
@@ -13,8 +13,8 @@ from api.exceptions import (
)
from api.key_helper import KeyHelper
from api.models import APIDeployment, APIKey
-from api.notification import APINotification
from api.serializers import APIExecutionResponseSerializer
+from api.utils import APIDeploymentUtils
from django.core.files.uploadedfile import UploadedFile
from django.db import connection
from rest_framework.request import Request
@@ -75,11 +75,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
@staticmethod
def get_api_by_id(api_id: str) -> Optional[APIDeployment]:
- try:
- api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)
- return api_deployment
- except APIDeployment.DoesNotExist:
- return None
+ return APIDeploymentUtils.get_api_by_id(api_id=api_id)
@staticmethod
def construct_complete_endpoint(api_name: str) -> str:
@@ -183,7 +179,6 @@ class DeploymentHelper(BaseAPIKeyValidator):
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
)
- cls._send_notification(api=api, result=result)
except Exception as error:
DestinationConnector.delete_api_storage_dir(
workflow_id=workflow_id, execution_id=execution_id
@@ -194,7 +189,6 @@ class DeploymentHelper(BaseAPIKeyValidator):
execution_status=ExecutionStatus.ERROR.value,
error=str(error),
)
- cls._send_notification(api=api, result=result)
return APIExecutionResponseSerializer(result).data
@staticmethod
@@ -211,15 +205,3 @@ class DeploymentHelper(BaseAPIKeyValidator):
execution_id=execution_id
)
return execution_response
-
- @staticmethod
- def _send_notification(api: APIDeployment, result: ExecutionResponse) -> None:
- """Sends a notification for the pipeline.
- Args:
- api (APIDeployment): APIDeployment to send notification for
-
- Returns:
- None
- """
- api_notification = APINotification(api=api, result=result)
- api_notification.send()
diff --git a/backend/api/notification.py b/backend/api/notification.py
index 6702ab801..ebf3cc390 100644
--- a/backend/api/notification.py
+++ b/backend/api/notification.py
@@ -4,16 +4,18 @@ from api.models import APIDeployment
from notification.helper import NotificationHelper
from notification.models import Notification
from pipeline.dto import PipelineStatusPayload
-from workflow_manager.workflow.dto import ExecutionResponse
+from workflow_manager.workflow.models.execution import WorkflowExecution
logger = logging.getLogger(__name__)
class APINotification:
- def __init__(self, api: APIDeployment, result: ExecutionResponse) -> None:
+ def __init__(
+ self, api: APIDeployment, workflow_execution: WorkflowExecution
+ ) -> None:
self.notifications = Notification.objects.filter(api=api, is_active=True)
self.api = api
- self.result = result
+ self.workflow_execution = workflow_execution
def send(self):
if not self.notifications.count():
@@ -25,9 +27,9 @@ class APINotification:
type="API",
pipeline_id=self.api.id,
pipeline_name=self.api.api_name,
- status=self.result.execution_status,
- execution_id=self.result.execution_id,
- error_message=self.result.error,
+ status=self.workflow_execution.status,
+ execution_id=self.workflow_execution.id,
+ error_message=self.workflow_execution.error_message,
)
NotificationHelper.send_notification(
diff --git a/backend/api/utils.py b/backend/api/utils.py
new file mode 100644
index 000000000..727fe480d
--- /dev/null
+++ b/backend/api/utils.py
@@ -0,0 +1,45 @@
+from typing import Optional
+
+from api.models import APIDeployment
+from api.notification import APINotification
+from workflow_manager.workflow.models.execution import WorkflowExecution
+
+
+class APIDeploymentUtils:
+ @staticmethod
+ def get_api_by_id(api_id: str) -> Optional[APIDeployment]:
+ """Retrieves an APIDeployment instance by its unique ID.
+
+ Args:
+ api_id (str): The unique identifier of the APIDeployment to retrieve.
+
+ Returns:
+ Optional[APIDeployment]: The APIDeployment instance if found,
+ otherwise None.
+ """
+ try:
+ api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)
+ return api_deployment
+ except APIDeployment.DoesNotExist:
+ return None
+
+ @staticmethod
+ def send_notification(
+ api: APIDeployment, workflow_execution: WorkflowExecution
+ ) -> None:
+ """Sends a notification for the specified API deployment and workflow
+ execution.
+
+ Args:
+ api (APIDeployment): The APIDeployment instance for which the
+ notification is being sent.
+ workflow_execution (WorkflowExecution): The WorkflowExecution instance
+ related to the notification.
+
+ Returns:
+ None
+ """
+ api_notification = APINotification(
+ api=api, workflow_execution=workflow_execution
+ )
+ api_notification.send()
diff --git a/backend/workflow_manager/workflow/constants.py b/backend/workflow_manager/workflow/constants.py
index 95aab13e3..2b9d3cf8e 100644
--- a/backend/workflow_manager/workflow/constants.py
+++ b/backend/workflow_manager/workflow/constants.py
@@ -58,3 +58,4 @@ class WorkflowMessages:
)
FILE_MARKER_CLEAR_SUCCESS = "File marker cleared successfully."
FILE_MARKER_CLEAR_FAILED = "Failed to clear file marker."
+ WORKFLOW_EXECUTION_NOT_FOUND = "Workflow execution not found."
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index 94f4bcddb..c9efb4690 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -421,3 +421,10 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
workflow: Workflow,
) -> WorkflowDto:
return WorkflowDto(id=workflow.id)
+
+ @staticmethod
+ def get_execution_by_id(execution_id: str) -> Optional[WorkflowExecution]:
+ try:
+ return WorkflowExecution.objects.get(id=execution_id)
+ except WorkflowExecution.DoesNotExist:
+ return None
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index e8c1444ba..9f35d4fee 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -2,12 +2,12 @@ import json
import logging
import os
import traceback
-import uuid
from typing import Any, Optional
from account.constants import Common
from account.models import Organization
from api.models import APIDeployment
+from api.utils import APIDeploymentUtils
from celery import current_task
from celery import exceptions as celery_exceptions
from celery import shared_task
@@ -149,7 +149,7 @@ class WorkflowHelper:
except Exception as error:
error_message = str(error)
error_raised += 1
- if error_raised and error_raised >= processed_files:
+ if error_raised and error_raised >= total_files:
execution_service.update_execution(
ExecutionStatus.ERROR, error=error_message
)
@@ -283,7 +283,6 @@ class WorkflowHelper:
single_step=single_step,
hash_values_of_files=hash_values_of_files,
)
- # TODO: Update through signals
WorkflowHelper._update_pipeline_status(
pipeline_id=pipeline_id, workflow_execution=workflow_execution
)
@@ -334,7 +333,11 @@ class WorkflowHelper:
)
# Expected exception since API deployments are not tracked in Pipeline
except Pipeline.DoesNotExist:
- pass
+ api = APIDeploymentUtils.get_api_by_id(api_id=pipeline_id)
+ if api:
+ APIDeploymentUtils.send_notification(
+ api=api, workflow_execution=workflow_execution
+ )
except Exception as e:
logger.warning(
f"Error updating pipeline {pipeline_id} status: {e}, "
@@ -367,7 +370,7 @@ class WorkflowHelper:
return ExecutionResponse(
execution.workflow_id,
execution_id,
- task.status,
+ execution.status,
result=task.result,
)
@@ -413,10 +416,13 @@ class WorkflowHelper:
logger.info(f"Job {async_execution} enqueued.")
celery_result = task.to_dict()
task_result = celery_result.get("result")
+ workflow_execution = WorkflowExecutionServiceHelper.get_execution_by_id(
+ execution_id=execution_id
+ )
return ExecutionResponse(
workflow_id,
execution_id,
- task.status,
+ workflow_execution.status,
result=task_result,
)
except celery_exceptions.TimeoutError:
@@ -525,10 +531,21 @@ class WorkflowHelper:
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
+ if not execution_id:
+ workflow_execution = (
+ WorkflowExecutionServiceHelper.create_workflow_execution(
+ workflow_id=workflow.id,
+ single_step=False,
+ pipeline_id=pipeline_id,
+ mode=WorkflowExecution.Mode.QUEUE,
+ execution_id=execution_id,
+ )
+ )
+ execution_id = workflow_execution.id
response: ExecutionResponse = WorkflowHelper.execute_workflow_async(
workflow_id=workflow.id,
pipeline_id=pipeline_id,
- execution_id=str(uuid.uuid4()),
+ execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
)
return response