LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Pipline flow changed from asyncrounous execution function (#659)
alic2f654b01744
diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.pyindex 8e5262f3d..630ffe6b4 100644--- a/backend/scheduler/helper.py+++ b/backend/scheduler/helper.py@@ -1,5 +1,4 @@import logging-import uuidfrom typing import Anyfrom django.db import connection@@ -20,11 +19,11 @@ from unstract.flags.feature_flag import check_feature_flag_statusif check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):from pipeline_v2.models import Pipelinefrom utils.user_context import UserContext- from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey+ from workflow_manager.workflow_v2.constants import WorkflowKeyfrom workflow_manager.workflow_v2.serializers import ExecuteWorkflowSerializerelse:from pipeline.models import Pipeline- from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey+ from workflow_manager.workflow.constants import WorkflowKeyfrom workflow_manager.workflow.serializers import ExecuteWorkflowSerializerlogger = logging.getLogger(__name__)@@ -46,8 +45,6 @@ class SchedulerHelper:task_data = job_kwargs.get("data", {})task_data[WorkflowKey.WF_ID] = pipeline.workflow.id- execution_id = str(uuid.uuid4())- task_data[WorkflowExecutionKey.EXECUTION_ID] = execution_idserializer = ExecuteWorkflowSerializer(data=task_data)serializer.is_valid(raise_exception=True)workflow_id = serializer.get_workflow_id(serializer.validated_data)@@ -66,7 +63,8 @@ class SchedulerHelper:str(workflow_id),organization_id,execution_action or "",- execution_id,+ # TODO: execution_id parameter cannot be removed without a migration.+ "",str(pipeline.pk),# Added to remain backward compatible - remove after data migration# which removes unused args in execute_pipeline_taskdiff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex a7e905904..762be6609 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -80,7 +80,6 @@ def execute_pipeline_task(execute_pipeline_task_v2(workflow_id=workflow_id,organization_id=org_schema,- execution_id=execution_id,pipeline_id=pipepline_id,pipeline_name=name,)@@ -120,7 +119,7 @@ def execute_pipeline_task(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)execution_response = WorkflowHelper.complete_execution(- workflow, execution_id, pipepline_id+ workflow=workflow, pipeline_id=pipepline_id)execution_response.remove_result_metadata_keys()logger.info(f"Execution response: {execution_response}")@@ -132,7 +131,6 @@ def execute_pipeline_task(def execute_pipeline_task_v2(workflow_id: Any,organization_id: Any,- execution_id: Any,pipeline_id: Any,pipeline_name: Any,) -> None:@@ -141,7 +139,6 @@ def execute_pipeline_task_v2(Args:workflow_id (Any): UID of workflow entityorg_schema (Any): Organization Identifier- execution_id (Any): UID of execution entitypipeline_id (Any): UID of pipeline entityname (Any): pipeline name"""@@ -170,7 +167,7 @@ def execute_pipeline_task_v2(pipeline_id, Pipeline.PipelineStatus.INPROGRESS)execution_response = WorkflowHelper.complete_execution(- workflow, execution_id, pipeline_id+ workflow=workflow, pipeline_id=pipeline_id)execution_response.remove_result_metadata_keys()logger.info(diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex b2a7676a9..c1514e9a1 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -103,8 +103,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.compilation_result = self.compile_workflow(execution_id=self.execution_id)- @staticmethod+ @classmethoddef create_workflow_execution(+ cls,workflow_id: str,pipeline_id: Optional[str] = None,single_step: bool = False,@@ -113,6 +114,11 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):execution_id: Optional[str] = None,mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,) -> WorkflowExecution:+ # Validating with existing execution+ existing_execution = cls.get_execution_instance_by_id(execution_id)+ if existing_execution:+ return existing_execution+execution_method: tuple[str, str] = (WorkflowExecution.Method.SCHEDULEDif scheduled@@ -168,6 +174,26 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):)return execution+ @classmethod+ def get_execution_instance_by_id(+ cls, execution_id: str+ ) -> Optional[WorkflowExecution]:+ """Get execution by execution ID.++ Args:+ execution_id (str): UID of execution entity++ Returns:+ Optional[WorkflowExecution]: WorkflowExecution Entity+ """+ try:+ execution: WorkflowExecution = WorkflowExecution.objects.get(+ pk=execution_id+ )+ return execution+ except WorkflowExecution.DoesNotExist:+ return None+def build(self) -> None:if self.compilation_result["success"] is True:self.build_workflow()diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex a572025f3..08c1c6f3f 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -30,6 +30,7 @@ from workflow_manager.workflow.exceptions import (WorkflowRegenerationError,)from workflow_manager.workflow.generator import WorkflowGenerator+from workflow_manager.workflow.models.execution import WorkflowExecutionfrom workflow_manager.workflow.models.workflow import Workflowfrom workflow_manager.workflow.serializers import (ExecuteWorkflowResponseSerializer,@@ -233,12 +234,14 @@ class WorkflowViewSet(viewsets.ModelViewSet):workflow=workflow,execution_id=execution_id,pipeline_id=pipeline_guid,+ execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,)else:execution_response = WorkflowHelper.complete_execution(workflow=workflow,execution_id=execution_id,+ execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,)return execution_responsediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 035c58913..0a8561744 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -545,28 +545,54 @@ class WorkflowHelper:workflow: Workflow,execution_id: Optional[str] = None,pipeline_id: Optional[str] = None,+ execution_mode: Optional[WorkflowExecution] = WorkflowExecution.Mode.QUEUE,hash_values_of_files: dict[str, FileHash] = {},) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")- if not execution_id:- workflow_execution = (- WorkflowExecutionServiceHelper.create_workflow_execution(- workflow_id=workflow.id,- single_step=False,- pipeline_id=pipeline_id,- mode=WorkflowExecution.Mode.QUEUE,- execution_id=execution_id,- )+ # Create a new WorkflowExecution entity for each pipeline execution.+ # This ensures every pipeline run is tracked as a distinct execution.+ workflow_execution = (+ WorkflowExecutionServiceHelper.create_workflow_execution(+ workflow_id=workflow.id,+ single_step=False,+ pipeline_id=pipeline_id,+ mode=execution_mode,)- execution_id = workflow_execution.id- response: ExecutionResponse = WorkflowHelper.execute_workflow_async(- workflow_id=workflow.id,- pipeline_id=pipeline_id,- execution_id=execution_id,- hash_values_of_files=hash_values_of_files,)- return response+ execution_id = workflow_execution.id+ log_events_id = StateStore.get(Common.LOG_EVENTS_ID)+ org_schema = connection.tenant.schema_name+ if execution_mode == WorkflowExecution.Mode.INSTANT:+ # Instant request from UX (Sync now in ETL and Workflow page)+ response: ExecutionResponse = WorkflowHelper.execute_workflow_async(+ workflow_id=workflow.id,+ pipeline_id=pipeline_id,+ execution_id=execution_id,+ hash_values_of_files=hash_values_of_files,+ )+ return response+ else:+ execution_result = WorkflowHelper.execute_bin(+ schema_name=org_schema,+ workflow_id=workflow.id,+ execution_id=workflow_execution.id,+ hash_values_of_files=hash_values_of_files,+ scheduled=True,+ execution_mode=execution_mode,+ pipeline_id=pipeline_id,+ log_events_id=log_events_id,+ )++ updated_execution = WorkflowExecution.objects.get(id=execution_id)+ execution_response = ExecutionResponse(+ workflow.id,+ execution_id,+ updated_execution.status,+ result=execution_result,+ )+ return execution_response+if execution_id is None:# Creating execution entity and returnreturn WorkflowHelper.create_and_make_execution_response(