LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Async API and some minor fixes (#574)
alif35efedc3345
diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.pyindex bc2b8aeff..3866af51d 100644--- a/backend/api/deployment_helper.py+++ b/backend/api/deployment_helper.py@@ -13,8 +13,8 @@ from api.exceptions import ()from api.key_helper import KeyHelperfrom api.models import APIDeployment, APIKey-from api.notification import APINotificationfrom api.serializers import APIExecutionResponseSerializer+from api.utils import APIDeploymentUtilsfrom django.core.files.uploadedfile import UploadedFilefrom django.db import connectionfrom rest_framework.request import Request@@ -75,11 +75,7 @@ class DeploymentHelper(BaseAPIKeyValidator):@staticmethoddef get_api_by_id(api_id: str) -> Optional[APIDeployment]:- try:- api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)- return api_deployment- except APIDeployment.DoesNotExist:- return None+ return APIDeploymentUtils.get_api_by_id(api_id=api_id)@staticmethoddef construct_complete_endpoint(api_name: str) -> str:@@ -183,7 +179,6 @@ class DeploymentHelper(BaseAPIKeyValidator):result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_id)- cls._send_notification(api=api, result=result)except Exception as error:DestinationConnector.delete_api_storage_dir(workflow_id=workflow_id, execution_id=execution_id@@ -194,7 +189,6 @@ class DeploymentHelper(BaseAPIKeyValidator):execution_status=ExecutionStatus.ERROR.value,error=str(error),)- cls._send_notification(api=api, result=result)return APIExecutionResponseSerializer(result).data@staticmethod@@ -211,15 +205,3 @@ class DeploymentHelper(BaseAPIKeyValidator):execution_id=execution_id)return execution_response-- @staticmethod- def _send_notification(api: APIDeployment, result: ExecutionResponse) -> None:- """Sends a notification for the pipeline.- Args:- api (APIDeployment): APIDeployment to send notification for-- Returns:- None- """- api_notification = APINotification(api=api, result=result)- api_notification.send()diff --git a/backend/api/notification.py b/backend/api/notification.pyindex 6702ab801..ebf3cc390 100644--- a/backend/api/notification.py+++ b/backend/api/notification.py@@ -4,16 +4,18 @@ from api.models import APIDeploymentfrom notification.helper import NotificationHelperfrom notification.models import Notificationfrom pipeline.dto import PipelineStatusPayload-from workflow_manager.workflow.dto import ExecutionResponse+from workflow_manager.workflow.models.execution import WorkflowExecutionlogger = logging.getLogger(__name__)class APINotification:- def __init__(self, api: APIDeployment, result: ExecutionResponse) -> None:+ def __init__(+ self, api: APIDeployment, workflow_execution: WorkflowExecution+ ) -> None:self.notifications = Notification.objects.filter(api=api, is_active=True)self.api = api- self.result = result+ self.workflow_execution = workflow_executiondef send(self):if not self.notifications.count():@@ -25,9 +27,9 @@ class APINotification:type="API",pipeline_id=self.api.id,pipeline_name=self.api.api_name,- status=self.result.execution_status,- execution_id=self.result.execution_id,- error_message=self.result.error,+ status=self.workflow_execution.status,+ execution_id=self.workflow_execution.id,+ error_message=self.workflow_execution.error_message,)NotificationHelper.send_notification(diff --git a/backend/api/utils.py b/backend/api/utils.pynew file mode 100644index 000000000..727fe480d--- /dev/null+++ b/backend/api/utils.py@@ -0,0 +1,45 @@+from typing import Optional++from api.models import APIDeployment+from api.notification import APINotification+from workflow_manager.workflow.models.execution import WorkflowExecution+++class APIDeploymentUtils:+ @staticmethod+ def get_api_by_id(api_id: str) -> Optional[APIDeployment]:+ """Retrieves an APIDeployment instance by its unique ID.++ Args:+ api_id (str): The unique identifier of the APIDeployment to retrieve.++ Returns:+ Optional[APIDeployment]: The APIDeployment instance if found,+ otherwise None.+ """+ try:+ api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)+ return api_deployment+ except APIDeployment.DoesNotExist:+ return None++ @staticmethod+ def send_notification(+ api: APIDeployment, workflow_execution: WorkflowExecution+ ) -> None:+ """Sends a notification for the specified API deployment and workflow+ execution.++ Args:+ api (APIDeployment): The APIDeployment instance for which the+ notification is being sent.+ workflow_execution (WorkflowExecution): The WorkflowExecution instance+ related to the notification.++ Returns:+ None+ """+ api_notification = APINotification(+ api=api, workflow_execution=workflow_execution+ )+ api_notification.send()diff --git a/backend/workflow_manager/workflow/constants.py b/backend/workflow_manager/workflow/constants.pyindex 95aab13e3..2b9d3cf8e 100644--- a/backend/workflow_manager/workflow/constants.py+++ b/backend/workflow_manager/workflow/constants.py@@ -58,3 +58,4 @@ class WorkflowMessages:)FILE_MARKER_CLEAR_SUCCESS = "File marker cleared successfully."FILE_MARKER_CLEAR_FAILED = "Failed to clear file marker."+ WORKFLOW_EXECUTION_NOT_FOUND = "Workflow execution not found."diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex 94f4bcddb..c9efb4690 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -421,3 +421,10 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):workflow: Workflow,) -> WorkflowDto:return WorkflowDto(id=workflow.id)++ @staticmethod+ def get_execution_by_id(execution_id: str) -> Optional[WorkflowExecution]:+ try:+ return WorkflowExecution.objects.get(id=execution_id)+ except WorkflowExecution.DoesNotExist:+ return Nonediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex e8c1444ba..9f35d4fee 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -2,12 +2,12 @@ import jsonimport loggingimport osimport traceback-import uuidfrom typing import Any, Optionalfrom account.constants import Commonfrom account.models import Organizationfrom api.models import APIDeployment+from api.utils import APIDeploymentUtilsfrom celery import current_taskfrom celery import exceptions as celery_exceptionsfrom celery import shared_task@@ -149,7 +149,7 @@ class WorkflowHelper:except Exception as error:error_message = str(error)error_raised += 1- if error_raised and error_raised >= processed_files:+ if error_raised and error_raised >= total_files:execution_service.update_execution(ExecutionStatus.ERROR, error=error_message)@@ -283,7 +283,6 @@ class WorkflowHelper:single_step=single_step,hash_values_of_files=hash_values_of_files,)- # TODO: Update through signalsWorkflowHelper._update_pipeline_status(pipeline_id=pipeline_id, workflow_execution=workflow_execution)@@ -334,7 +333,11 @@ class WorkflowHelper:)# Expected exception since API deployments are not tracked in Pipelineexcept Pipeline.DoesNotExist:- pass+ api = APIDeploymentUtils.get_api_by_id(api_id=pipeline_id)+ if api:+ APIDeploymentUtils.send_notification(+ api=api, workflow_execution=workflow_execution+ )except Exception as e:logger.warning(f"Error updating pipeline {pipeline_id} status: {e}, "@@ -367,7 +370,7 @@ class WorkflowHelper:return ExecutionResponse(execution.workflow_id,execution_id,- task.status,+ execution.status,result=task.result,)@@ -413,10 +416,13 @@ class WorkflowHelper:logger.info(f"Job {async_execution} enqueued.")celery_result = task.to_dict()task_result = celery_result.get("result")+ workflow_execution = WorkflowExecutionServiceHelper.get_execution_by_id(+ execution_id=execution_id+ )return ExecutionResponse(workflow_id,execution_id,- task.status,+ workflow_execution.status,result=task_result,)except celery_exceptions.TimeoutError:@@ -525,10 +531,21 @@ class WorkflowHelper:) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")+ if not execution_id:+ workflow_execution = (+ WorkflowExecutionServiceHelper.create_workflow_execution(+ workflow_id=workflow.id,+ single_step=False,+ pipeline_id=pipeline_id,+ mode=WorkflowExecution.Mode.QUEUE,+ execution_id=execution_id,+ )+ )+ execution_id = workflow_execution.idresponse: ExecutionResponse = WorkflowHelper.execute_workflow_async(workflow_id=workflow.id,pipeline_id=pipeline_id,- execution_id=str(uuid.uuid4()),+ execution_id=execution_id,hash_values_of_files=hash_values_of_files,)return response