LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Error handling improvements for pipelines and scheduler (#340)
Chandrasekharan M62f42bbbcf6e
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex ab8d0dfce..d043d1664 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -352,13 +352,6 @@ CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task# Feature FlagFEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}-SCHEDULER_KWARGS = {- "coalesce": True,- "misfire_grace_time": 300,- "max_instances": 1,- "replace_existing": True,-}-# Password validation# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validatorsdiff --git a/backend/pipeline/exceptions.py b/backend/pipeline/exceptions.pyindex da8c9dff7..0b9078ed2 100644--- a/backend/pipeline/exceptions.py+++ b/backend/pipeline/exceptions.py@@ -1,3 +1,5 @@+from typing import Optional+from rest_framework.exceptions import APIException@@ -20,6 +22,19 @@ class InactivePipelineError(APIException):status_code = 422default_detail = "Pipeline is inactive, please activate the pipeline"+ def __init__(+ self,+ pipeline_name: Optional[str] = None,+ detail: Optional[str] = None,+ code: Optional[str] = None,+ ):+ if pipeline_name:+ self.default_detail = (+ f"Pipeline '{pipeline_name}' is inactive, "+ "please activate the pipeline"+ )+ super().__init__(detail, code)+class MandatoryPipelineType(APIException):status_code = 400diff --git a/backend/pipeline/manager.py b/backend/pipeline/manager.pyindex bee1b0c93..f6a339210 100644--- a/backend/pipeline/manager.py+++ b/backend/pipeline/manager.py@@ -4,7 +4,6 @@ from typing import Any, Optionalfrom django.conf import settingsfrom django.urls import reversefrom pipeline.constants import PipelineKey, PipelineURL-from pipeline.exceptions import PipelineExecuteErrorfrom pipeline.models import Pipelinefrom pipeline.pipeline_processor import PipelineProcessorfrom rest_framework.request import Request@@ -26,7 +25,6 @@ class PipelineManager:request: Request,pipeline_id: str,execution_id: Optional[str] = None,- with_log: Optional[bool] = None,) -> Response:"""Used to execute a pipeline.@@ -34,29 +32,15 @@ class PipelineManager:pipeline_id (str): UUID of the pipeline to executeexecution_id (Optional[str], optional):Uniquely identifies an execution. Defaults to None.- with_log (Optional[bool], optional): Flag to pass logs to FE.- Defaults to None."""- logger.info(- f"Executing pipeline {pipeline_id}, execution: {execution_id}, "- f"with_log = {with_log}"- )- try:- pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)- # TODO: Use DRF's request and as_view() instead- request.data[WorkflowKey.WF_ID] = pipeline.workflow.id- if execution_id is not None:- request.data[WorkflowExecutionKey.EXECUTION_ID] = execution_id- wf_viewset = WorkflowViewSet()- return WorkflowViewSet.execute(- wf_viewset,- request=request,- pipeline_guid=pipeline.pk,- with_log=with_log,- )- except Exception as e:- logger.error(f"Error executing pipeline: {e}")- raise PipelineExecuteError()+ logger.info(f"Executing pipeline {pipeline_id}, execution: {execution_id}")+ pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)+ # TODO: Use DRF's request and as_view() instead+ request.data[WorkflowKey.WF_ID] = pipeline.workflow.id+ if execution_id is not None:+ request.data[WorkflowExecutionKey.EXECUTION_ID] = execution_id+ wf_viewset = WorkflowViewSet()+ return wf_viewset.execute(request=request, pipeline_guid=str(pipeline.pk))@staticmethoddef get_pipeline_execution_data_for_scheduled_run(@@ -66,12 +50,10 @@ class PipelineManager:changes to pipeline execution needs to be propagated here."""callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}- job_params = {WorkflowExecutionKey.WITH_LOG: "False"}job_kwargs = {RequestConstants.VERB: "POST",RequestConstants.URL: callback_url,RequestConstants.HEADERS: job_headers,- RequestConstants.PARAMS: job_params,RequestConstants.DATA: {PipelineKey.PIPELINE_ID: pipeline_id},}return job_kwargsdiff --git a/backend/pipeline/pipeline_processor.py b/backend/pipeline/pipeline_processor.pyindex d3068641d..e84280da6 100644--- a/backend/pipeline/pipeline_processor.py+++ b/backend/pipeline/pipeline_processor.py@@ -34,7 +34,7 @@ class PipelineProcessor:pipeline: Pipeline = Pipeline.objects.get(pk=pipeline_id)if check_active and not pipeline.is_active():logger.error(f"Inactive pipeline fetched: {pipeline_id}")- raise InactivePipelineError+ raise InactivePipelineError(pipeline_name=pipeline.pipeline_name)return pipeline@staticmethoddiff --git a/backend/pipeline/views.py b/backend/pipeline/views.pyindex 96cadd74f..371032d50 100644--- a/backend/pipeline/views.py+++ b/backend/pipeline/views.py@@ -4,8 +4,6 @@ from typing import Any, Optionalfrom account.custom_exceptions import DuplicateDatafrom django.db import IntegrityErrorfrom django.db.models import QuerySet-from drf_yasg import openapi-from drf_yasg.utils import swagger_auto_schemafrom permissions.permission import IsOwnerfrom pipeline.constants import PipelineConstants, PipelineErrors, PipelineExecutionKeyfrom pipeline.constants import PipelineKey as PK@@ -20,7 +18,6 @@ from rest_framework.request import Requestfrom rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom scheduler.helper import SchedulerHelper-from workflow_manager.workflow.constants import WorkflowExecutionKey as WFExecKeylogger = logging.getLogger(__name__)@@ -31,15 +28,6 @@ class PipelineViewSet(viewsets.ModelViewSet):permission_classes = [IsOwner]serializer_class = PipelineSerializer- # For doc strings- with_log = openapi.Parameter(- "with_log",- openapi.IN_QUERY,- description="To pass workflow execution logs to FE",- type=openapi.TYPE_BOOLEAN,- enum=[True, False],- )-def get_queryset(self) -> Optional[QuerySet]:type = self.request.query_params.get(PipelineConstants.TYPE)if type is not None:@@ -61,10 +49,7 @@ class PipelineViewSet(viewsets.ModelViewSet):# For eg, passing pipeline ID and with_log=False -> executes pipeline# For FE however we call the same API twice# (first call generates execution ID)- @swagger_auto_schema(manual_parameters=[with_log])def execute(self, request: Request) -> Response:- # TODO: Use a serializer instead to validate args- with_log = request.query_params.get(WFExecKey.WITH_LOG)serializer: ExecuteSerializer = self.get_serializer(data=request.data)serializer.is_valid(raise_exception=True)execution_id = serializer.validated_data.get("execution_id", None)@@ -74,7 +59,6 @@ class PipelineViewSet(viewsets.ModelViewSet):request=request,pipeline_id=pipeline_id,execution_id=execution_id,- with_log=with_log,)pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)serializer = PipelineSerializer(pipeline)diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.pyindex 20a2d7d29..141df78f1 100644--- a/backend/scheduler/helper.py+++ b/backend/scheduler/helper.py@@ -1,16 +1,70 @@import logging+import uuid+from typing import Any-from django.core.exceptions import ValidationError-from scheduler import views+from django.db import connection+from pipeline.models import Pipeline+from pipeline.pipeline_processor import PipelineProcessor+from rest_framework.serializers import ValidationErrorfrom scheduler.constants import SchedulerConstants as SCfrom scheduler.exceptions import JobDeletionError, JobSchedulingErrorfrom scheduler.serializer import AddJobSerializer-from scheduler.tasks import delete_periodic_task, disable_task, enable_task+from scheduler.tasks import (+ create_periodic_task,+ delete_periodic_task,+ disable_task,+ enable_task,+)+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey+from workflow_manager.workflow.serializers import ExecuteWorkflowSerializerlogger = logging.getLogger(__name__)class SchedulerHelper:++ @staticmethod+ def _schedule_task_job(pipeline_id: str, job_data: Any) -> None:+ if "cron_string" not in job_data:+ raise ValidationError("cron_string is required in job_data")+ if "id" not in job_data:+ raise ValidationError("id is required in job_data")+ if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):+ raise ValidationError("job_kwargs is required to be a dict")++ cron_string = job_data.get("cron_string")+ name = job_data.get("name")+ job_kwargs = job_data.get("job_kwargs", {})+ task_data = job_kwargs.get("data", {})++ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)+ task_data[WorkflowKey.WF_ID] = pipeline.workflow.id+ execution_id = str(uuid.uuid4())+ task_data[WorkflowExecutionKey.EXECUTION_ID] = execution_id+ serializer = ExecuteWorkflowSerializer(data=task_data)+ serializer.is_valid(raise_exception=True)+ workflow_id = serializer.get_workflow_id(serializer.validated_data)+ # TODO: Remove unused argument in execute_pipeline_task+ execution_action = serializer.get_execution_action(serializer.validated_data)+ org_schema = connection.tenant.schema_name++ create_periodic_task(+ cron_string=cron_string,+ task_name=pipeline.pk,+ task_path="scheduler.tasks.execute_pipeline_task",+ task_args=[+ str(workflow_id),+ org_schema,+ execution_action or "",+ execution_id,+ str(pipeline.pk),+ # Added to remain backward compatible - remove after data migration+ # which removes unused args in execute_pipeline_task+ bool(False),+ str(name),+ ],+ )+@staticmethoddef add_job(pipeline_id: str, cron_string: str = SC.DEFAULT_CRON_STRING) -> None:logger.info(f"Scheduling job for {pipeline_id} with {cron_string}")@@ -20,22 +74,12 @@ class SchedulerHelper:SC.CRON_STRING: cron_string,SC.NAME: name,}+ job_serializer = AddJobSerializer(data=job_serialize_data)+ job_serializer.is_valid(raise_exception=True)try:- job_serializer = AddJobSerializer(data=job_serialize_data)- job_serializer.is_valid(raise_exception=True)- except ValidationError as e:- logger.error(f"Validation error while scheduling job: {e}")- raise JobSchedulingError- except Exception as e:- logger.error(f"Unhandled exception while scheduling job: {e}")- raise JobSchedulingError-- try:- # Celery based scheduler- scheduler_response = views.schedule_task_job(+ SchedulerHelper._schedule_task_job(pipeline_id, job_serializer.validated_data)- logger.info(f"Scheduler response: {scheduler_response}")except Exception as e:logger.error(f"Exception while adding job: {e}")raise JobSchedulingErrordiff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.pyindex 2ea3c3b53..bc0e32f42 100644--- a/backend/scheduler/serializer.py+++ b/backend/scheduler/serializer.py@@ -1,7 +1,6 @@import loggingfrom typing import Any-from django.conf import settingsfrom pipeline.manager import PipelineManagerfrom rest_framework import serializersfrom scheduler.constants import SchedulerConstants as SC@@ -23,13 +22,6 @@ class JobKwargsSerializer(serializers.Serializer):data = serializers.JSONField()-class SchedulerKwargsSerializer(serializers.Serializer):- coalesce = serializers.BooleanField()- misfire_grace_time = serializers.IntegerField(allow_null=True, required=False)- max_instances = serializers.IntegerField()- replace_existing = serializers.BooleanField()--class AddJobSerializer(serializers.Serializer):id = serializers.CharField(max_length=FieldLength.UUID_LENGTH)cron_string = serializers.CharField(max_length=FieldLength.CRON_LENGTH)@@ -37,7 +29,6 @@ class AddJobSerializer(serializers.Serializer):max_length=JOB_NAME_LENGTH, required=False, allow_blank=True)job_kwargs = JobKwargsSerializer(write_only=True)- scheduler_kwargs = SchedulerKwargsSerializer(write_only=True)def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:if SC.NAME not in data:@@ -47,5 +38,4 @@ class AddJobSerializer(serializers.Serializer):pipeline_id=data[SC.ID]))- data[SC.SCHEDULER_KWARGS] = settings.SCHEDULER_KWARGSreturn super().to_internal_value(data) # type: ignorediff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex b42244576..71673bc6b 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -8,7 +8,6 @@ from django_celery_beat.models import CrontabSchedule, PeriodicTaskfrom django_tenants.utils import get_tenant_model, tenant_contextfrom pipeline.models import Pipelinefrom pipeline.pipeline_processor import PipelineProcessor-from pymysql import IntegrityErrorfrom workflow_manager.workflow.models.workflow import Workflowfrom workflow_manager.workflow.workflow_helper import WorkflowHelper@@ -21,12 +20,8 @@ def create_periodic_task(task_path: str,task_args: list[Any],) -> None:- try:- # Convert task_args to JSON- task_args_json = json.dumps(task_args)- except TypeError as te:- logger.error(f"Failed : task_args is not JSON serializable: {te}")- return+ # Convert task_args to JSON+ task_args_json = json.dumps(task_args)# Parse the cron stringminute, hour, day_of_month, month_of_year, day_of_week = cron_string.split()@@ -39,21 +34,15 @@ def create_periodic_task(day_of_month=day_of_month,month_of_year=month_of_year,)-- logger.info(f"Creating periodic task with cron string: {cron_string}")+ logger.info(f"Creating periodic task {task_name} with cron string: {cron_string}")# Create periodic task- try:- logger.info(f"Running periodic task addition to beat table: {schedule}")- PeriodicTask.objects.create(- crontab=schedule,- name=task_name,- task=task_path,- args=task_args_json,- )- logger.info(f"Completed running task addition to beat table: {task_name}")- except Exception as e:- logger.error(f"Failed to create periodic task: {e}")+ PeriodicTask.objects.create(+ crontab=schedule,+ name=task_name,+ task=task_path,+ args=task_args_json,+ )def update_pipeline(@@ -61,19 +50,21 @@ def update_pipeline(status: tuple[str, str],is_active: Optional[bool] = None,) -> Any:- if pipeline_guid:- check_active = True- if is_active is True:- check_active = False- pipeline: Pipeline = PipelineProcessor.fetch_pipeline(- pipeline_id=pipeline_guid, check_active=check_active- )- PipelineProcessor.update_pipeline_status(- pipeline=pipeline, is_end=True, status=status, is_active=is_active- )- logger.info(f"Updated pipeline status: {status}")+ if not pipeline_guid:+ return+ check_active = True+ if is_active is True:+ check_active = False+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(+ pipeline_id=pipeline_guid, check_active=check_active+ )+ PipelineProcessor.update_pipeline_status(+ pipeline=pipeline, is_end=True, status=status, is_active=is_active+ )+ logger.info(f"Updated pipeline status: {status}")+# TODO: Remove unused args with a migration@shared_taskdef execute_pipeline_task(workflow_id: Any,@@ -93,20 +84,17 @@ def execute_pipeline_task()with tenant_context(tenant):workflow = Workflow.objects.get(id=workflow_id)- try:- logger.info(f"Executing workflow: {workflow}")- update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)- execution_response = WorkflowHelper.complete_execution(- workflow, execution_id, pipepline_id, with_logs- )- logger.info(f"Execution response: {execution_response}")- update_pipeline(pipepline_id, Pipeline.PipelineStatus.SUCCESS)- except IntegrityError:- logger.error(f"Failed to complete execution for workflow: {workflow}")- update_pipeline(pipepline_id, Pipeline.PipelineStatus.FAILURE)+ logger.info(f"Executing workflow: {workflow}")+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)+ execution_response = WorkflowHelper.complete_execution(+ workflow, execution_id, pipepline_id+ )+ logger.info(f"Execution response: {execution_response}")+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.SUCCESS)logger.info(f"Execution completed for pipeline: {name}")except Exception as e:logger.error(f"Failed to execute pipeline: {name}. Error: {e}")+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.FAILURE)def delete_periodic_task(task_name: str) -> None:diff --git a/backend/scheduler/views.py b/backend/scheduler/views.pydeleted file mode 100644index 1e02a93d1..000000000--- a/backend/scheduler/views.py+++ /dev/null@@ -1,75 +0,0 @@-import logging-import uuid-from typing import Any--from django.db import connection-from pipeline.models import Pipeline-from pipeline.pipeline_processor import PipelineProcessor-from rest_framework.response import Response-from scheduler.tasks import create_periodic_task-from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey-from workflow_manager.workflow.exceptions import WorkflowExecutionBadRequestException-from workflow_manager.workflow.serializers import ExecuteWorkflowSerializer-from workflow_manager.workflow.views import WorkflowViewSet--logger = logging.getLogger(__name__)---def schedule_task_job(pipeline_id: str, job_data: Any) -> Response:- try:- # Validating request- if "cron_string" not in job_data:- return Response({"error": "cron_string is required"}, status=400)- if "id" not in job_data:- return Response({"error": "id is required"}, status=400)- if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):- return Response(- {"error": "job_kwargs is required to be a dict"}, status=400- )-- cron_string = job_data.get("cron_string")- name = job_data.get("name")- job_kwargs = job_data.get("job_kwargs", {})- task_data = job_kwargs.get("data", {})- params = job_kwargs.get("params", {})- with_logs = params.get("with_logs", False)-- pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)-- task_data[WorkflowKey.WF_ID] = pipeline.workflow.id- execution_id = str(uuid.uuid4())- task_data[WorkflowExecutionKey.EXECUTION_ID] = execution_id- workflow_viewset = WorkflowViewSet()- workflow_viewset.serializer_class = ExecuteWorkflowSerializer- serializer = ExecuteWorkflowSerializer(data=task_data)-- if not serializer.is_valid():- raise WorkflowExecutionBadRequestException(- workflow_viewset.get_error_from_serializer(serializer.errors)- )-- workflow_id = serializer.get_workflow_id(serializer.validated_data)-- execution_action = serializer.get_execution_action(serializer.validated_data)- org_schema = connection.tenant.schema_name-- create_periodic_task(- cron_string=cron_string,- task_name=pipeline.pk,- task_path="scheduler.tasks.execute_pipeline_task",- task_args=[- str(workflow_id),- org_schema,- execution_action or "",- execution_id,- str(pipeline.pk),- bool(with_logs),- str(name),- ],- )-- return Response({"message": "Job scheduled successfully"}, status=200)-- except Exception as e:- logger.error(f"Exception while scheduling job: {e}")- return Response({"error": str(e)}, status=500)diff --git a/backend/workflow_manager/workflow/exceptions.py b/backend/workflow_manager/workflow/exceptions.pyindex 56f111600..839a36429 100644--- a/backend/workflow_manager/workflow/exceptions.py+++ b/backend/workflow_manager/workflow/exceptions.py@@ -1,5 +1,3 @@-from typing import Optional-from rest_framework.exceptions import APIException@@ -58,18 +56,6 @@ class WorkflowExecutionNotExist(APIException):default_detail = "Workflow execution does not exist"-class WorkflowExecutionBadRequestException(APIException):- status_code = 400- default_detail = "Bad request"-- def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):- if detail is not None:- self.detail = detail- if code is not None:- self.code = code- super().__init__(detail, code)--class ToolValidationError(APIException):status_code = 400default_detail = "Tool validation error"diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex b332c9ecb..a108c196a 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -275,13 +275,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):)def publish_initial_tool_execution_logs(- self, current_step: int, total_step: int, file_name: str+ self, current_file_idx: int, total_files: int, file_name: str) -> None:"""Publishes the initial logs for tool execution.Args:- current_step (int): The current step number.- total_step (int): The total number of steps.+ current_file_idx (int): 1-based index for the current file being processed+ total_files (int): The total number of files to processfile_name (str): The name of the file being processed.Returns:@@ -290,7 +290,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.publish_update_log(component=LogComponent.STATUS_BAR,state=LogState.MESSAGE,- message=f"Processing file {current_step}/{total_step}",+ message=f"Processing file {file_name} {current_file_idx}/{total_files}",)self.publish_log(f"Processing file {file_name}")@@ -348,8 +348,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):def initiate_tool_execution(self,- current_step: int,- total_step: int,+ current_file_idx: int,+ total_files: int,file_name: str,single_step: bool,) -> None:@@ -357,11 +357,11 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):workflow.Args:- current_step (int): The current step number in the workflow.- total_step (int): The total number of steps in the workflow.- file_name (str): The name of the file being processed.+ current_file_idx (int): 1-based index for the current file being processed+ total_step (int): The total number of files to process in the workflow+ file_name (str): The name of the file being processedsingle_step (bool): Flag indicating whether the execution is in- single-step mode.+ single-step modeReturns:None@@ -372,11 +372,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):execution_type = ExecutionType.COMPLETEif single_step:execution_type = ExecutionType.STEP- self.publish_initial_tool_execution_logs(current_step, total_step, file_name)+ self.publish_initial_tool_execution_logs(+ current_file_idx, total_files, file_name+ )self._handle_execution_type(execution_type)source_status_message = (- f"({current_step}/{total_step})Processing file {file_name}"+ f"({current_file_idx}/{total_files})Processing file {file_name}")self.publish_update_log(state=LogState.RUNNING,diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex 7ce7b2cdc..978153432 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -58,13 +58,6 @@ def make_execution_response(response: ExecutionResponse) -> Any:return ExecuteWorkflowResponseSerializer(response).data-def handle_false(string: str) -> bool:- if string.lower() == "false":- return False- else:- return True--class WorkflowViewSet(viewsets.ModelViewSet):versioning_class = URLPathVersioningpermission_classes = [IsOwner]@@ -153,13 +146,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution = WorkflowHelper.get_current_execution(pk)return Response(make_execution_response(execution), status=status.HTTP_200_OK)- def get_error_from_serializer(self, error_details: dict[str, Any]) -> Optional[str]:- """Validation error."""- error_key = next(iter(error_details))- # Get the first error message- error_message: str = f"{error_details[error_key][0]} : {error_key}"- return error_message-def get_workflow_by_id_or_project_id(self,workflow_id: Optional[str] = None,@@ -189,12 +175,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):self,request: Request,pipeline_guid: Optional[str] = None,- with_log: Optional[bool] = None,) -> Response:- if with_log is not None:- # Handle string field- with_log = handle_false(str(with_log))-self.serializer_class = ExecuteWorkflowSerializerserializer = ExecuteWorkflowSerializer(data=request.data)serializer.is_valid(raise_exception=True)@@ -220,7 +201,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_action=execution_action,execution_id=execution_id,pipeline_guid=pipeline_guid,- with_log=with_log,hash_values_of_files=hashes_of_files,)return Response(@@ -232,8 +212,8 @@ class WorkflowViewSet(viewsets.ModelViewSet):update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)raise exceptionexcept MissingEnvException as exception:- update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)logger.error(f"Error while executing workflow: {exception}")+ update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)return Response({"error": "Please check the logs for more details: " + str(exception)},status=status.HTTP_400_BAD_REQUEST,@@ -253,7 +233,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_action: Optional[str] = None,execution_id: Optional[str] = None,pipeline_guid: Optional[str] = None,- with_log: Optional[bool] = None,hash_values_of_files: dict[str, str] = {},) -> ExecutionResponse:if execution_action is not None:@@ -271,7 +250,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):workflow=workflow,execution_id=execution_id,pipeline_id=pipeline_guid,- log_required=with_log,hash_values_of_files=hash_values_of_files,)update_pipeline(pipeline_guid, Pipeline.PipelineStatus.SUCCESS)@@ -279,7 +257,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_response = WorkflowHelper.complete_execution(workflow=workflow,execution_id=execution_id,- log_required=with_log,hash_values_of_files=hash_values_of_files,)return execution_responsediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex c74f2ccb2..4ac6364d5 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -122,8 +122,8 @@ class WorkflowHelper:file_number = index + 1try:is_executed, error = WorkflowHelper.process_file(- current_step=file_number,- total_step=total_files,+ current_file_idx=file_number,+ total_files=total_files,input_file=input_file,workflow=workflow,source=source,@@ -153,8 +153,8 @@ class WorkflowHelper:@staticmethoddef process_file(- current_step: int,- total_step: int,+ current_file_idx: int,+ total_files: int,input_file: str,workflow: Workflow,source: SourceConnector,@@ -172,7 +172,7 @@ class WorkflowHelper:)try:execution_service.initiate_tool_execution(- current_step, total_step, file_name, single_step+ current_file_idx, total_files, file_name, single_step)file_history = FileHistoryHelper.get_file_history(workflow=workflow, cache_key=file_hash@@ -192,7 +192,7 @@ class WorkflowHelper:error = str(e)execution_service.publish_update_log(LogState.RUNNING,- f"{file_name} is processing",+ f"Processing output for {file_name}",LogComponent.DESTINATION,)destination.handle_output(@@ -204,7 +204,7 @@ class WorkflowHelper:)execution_service.publish_update_log(LogState.SUCCESS,- f"{file_name} is Processed successfully",+ f"{file_name}'s output is processed successfully",LogComponent.DESTINATION,)return is_executed, error@@ -456,32 +456,17 @@ class WorkflowHelper:workflow: Workflow,execution_id: Optional[str] = None,pipeline_id: Optional[str] = None,- log_required: Optional[bool] = True,hash_values_of_files: dict[str, str] = {},) -> ExecutionResponse:- # For scheduler workflow execution- logger.info(f"Workflow pipeline ID: {pipeline_id}")if pipeline_id:- return WorkflowHelper.run_workflow(- workflow=workflow,- hash_values_of_files=hash_values_of_files,+ logger.info(f"Executing pipeline: {pipeline_id}")+ response: ExecutionResponse = WorkflowHelper.execute_workflow_async(+ workflow_id=workflow.id,pipeline_id=pipeline_id,+ execution_id=str(uuid.uuid4()),+ hash_values_of_files=hash_values_of_files,)-- # TODO: Review required here.- if log_required is not None and not log_required:- # Without log and log Id- if pipeline_id:- # pipeline scheduled execution- execution_id = str(uuid.uuid4())- response: ExecutionResponse = WorkflowHelper.execute_workflow_async(- workflow_id=workflow.id,- pipeline_id=pipeline_id,- execution_id=execution_id,- hash_values_of_files=hash_values_of_files,- )- return response- return WorkflowHelper.run_workflow(workflow=workflow)+ return responseif execution_id is None:# Creating execution entity and returnreturn WorkflowHelper.create_and_make_execution_response(diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 7e3d73273..cad6cd16a 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -218,7 +218,7 @@ class WorkflowExecutionService:self.publish_log(log_message, step=actual_step)except Exception as error:- logger.info(f"Error while execution tool {tool_uid}: {str(error)}")+ logger.info(f"Error while running tool {tool_uid}: {str(error)}")self.publish_log(str(error), LogLevel.ERROR, step=actual_step)self.publish_update_log(state=LogState.ERROR,