Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Error handling improvements for pipelines and scheduler (#340)

Chandrasekharan M62f42bbbcf6e
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index ab8d0dfce..d043d1664 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -352,13 +352,6 @@ CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task
# Feature Flag
FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}
-SCHEDULER_KWARGS = {
- "coalesce": True,
- "misfire_grace_time": 300,
- "max_instances": 1,
- "replace_existing": True,
-}
-
# Password validation
# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
diff --git a/backend/pipeline/exceptions.py b/backend/pipeline/exceptions.py
index da8c9dff7..0b9078ed2 100644
--- a/backend/pipeline/exceptions.py
+++ b/backend/pipeline/exceptions.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
from rest_framework.exceptions import APIException
@@ -20,6 +22,19 @@ class InactivePipelineError(APIException):
status_code = 422
default_detail = "Pipeline is inactive, please activate the pipeline"
+ def __init__(
+ self,
+ pipeline_name: Optional[str] = None,
+ detail: Optional[str] = None,
+ code: Optional[str] = None,
+ ):
+ if pipeline_name:
+ self.default_detail = (
+ f"Pipeline '{pipeline_name}' is inactive, "
+ "please activate the pipeline"
+ )
+ super().__init__(detail, code)
+
class MandatoryPipelineType(APIException):
status_code = 400
diff --git a/backend/pipeline/manager.py b/backend/pipeline/manager.py
index bee1b0c93..f6a339210 100644
--- a/backend/pipeline/manager.py
+++ b/backend/pipeline/manager.py
@@ -4,7 +4,6 @@ from typing import Any, Optional
from django.conf import settings
from django.urls import reverse
from pipeline.constants import PipelineKey, PipelineURL
-from pipeline.exceptions import PipelineExecuteError
from pipeline.models import Pipeline
from pipeline.pipeline_processor import PipelineProcessor
from rest_framework.request import Request
@@ -26,7 +25,6 @@ class PipelineManager:
request: Request,
pipeline_id: str,
execution_id: Optional[str] = None,
- with_log: Optional[bool] = None,
) -> Response:
"""Used to execute a pipeline.
@@ -34,29 +32,15 @@ class PipelineManager:
pipeline_id (str): UUID of the pipeline to execute
execution_id (Optional[str], optional):
Uniquely identifies an execution. Defaults to None.
- with_log (Optional[bool], optional): Flag to pass logs to FE.
- Defaults to None.
"""
- logger.info(
- f"Executing pipeline {pipeline_id}, execution: {execution_id}, "
- f"with_log = {with_log}"
- )
- try:
- pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)
- # TODO: Use DRF's request and as_view() instead
- request.data[WorkflowKey.WF_ID] = pipeline.workflow.id
- if execution_id is not None:
- request.data[WorkflowExecutionKey.EXECUTION_ID] = execution_id
- wf_viewset = WorkflowViewSet()
- return WorkflowViewSet.execute(
- wf_viewset,
- request=request,
- pipeline_guid=pipeline.pk,
- with_log=with_log,
- )
- except Exception as e:
- logger.error(f"Error executing pipeline: {e}")
- raise PipelineExecuteError()
+ logger.info(f"Executing pipeline {pipeline_id}, execution: {execution_id}")
+ pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)
+ # TODO: Use DRF's request and as_view() instead
+ request.data[WorkflowKey.WF_ID] = pipeline.workflow.id
+ if execution_id is not None:
+ request.data[WorkflowExecutionKey.EXECUTION_ID] = execution_id
+ wf_viewset = WorkflowViewSet()
+ return wf_viewset.execute(request=request, pipeline_guid=str(pipeline.pk))
@staticmethod
def get_pipeline_execution_data_for_scheduled_run(
@@ -66,12 +50,10 @@ class PipelineManager:
changes to pipeline execution needs to be propagated here."""
callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)
job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}
- job_params = {WorkflowExecutionKey.WITH_LOG: "False"}
job_kwargs = {
RequestConstants.VERB: "POST",
RequestConstants.URL: callback_url,
RequestConstants.HEADERS: job_headers,
- RequestConstants.PARAMS: job_params,
RequestConstants.DATA: {PipelineKey.PIPELINE_ID: pipeline_id},
}
return job_kwargs
diff --git a/backend/pipeline/pipeline_processor.py b/backend/pipeline/pipeline_processor.py
index d3068641d..e84280da6 100644
--- a/backend/pipeline/pipeline_processor.py
+++ b/backend/pipeline/pipeline_processor.py
@@ -34,7 +34,7 @@ class PipelineProcessor:
pipeline: Pipeline = Pipeline.objects.get(pk=pipeline_id)
if check_active and not pipeline.is_active():
logger.error(f"Inactive pipeline fetched: {pipeline_id}")
- raise InactivePipelineError
+ raise InactivePipelineError(pipeline_name=pipeline.pipeline_name)
return pipeline
@staticmethod
diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py
index 96cadd74f..371032d50 100644
--- a/backend/pipeline/views.py
+++ b/backend/pipeline/views.py
@@ -4,8 +4,6 @@ from typing import Any, Optional
from account.custom_exceptions import DuplicateData
from django.db import IntegrityError
from django.db.models import QuerySet
-from drf_yasg import openapi
-from drf_yasg.utils import swagger_auto_schema
from permissions.permission import IsOwner
from pipeline.constants import PipelineConstants, PipelineErrors, PipelineExecutionKey
from pipeline.constants import PipelineKey as PK
@@ -20,7 +18,6 @@ from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from scheduler.helper import SchedulerHelper
-from workflow_manager.workflow.constants import WorkflowExecutionKey as WFExecKey
logger = logging.getLogger(__name__)
@@ -31,15 +28,6 @@ class PipelineViewSet(viewsets.ModelViewSet):
permission_classes = [IsOwner]
serializer_class = PipelineSerializer
- # For doc strings
- with_log = openapi.Parameter(
- "with_log",
- openapi.IN_QUERY,
- description="To pass workflow execution logs to FE",
- type=openapi.TYPE_BOOLEAN,
- enum=[True, False],
- )
-
def get_queryset(self) -> Optional[QuerySet]:
type = self.request.query_params.get(PipelineConstants.TYPE)
if type is not None:
@@ -61,10 +49,7 @@ class PipelineViewSet(viewsets.ModelViewSet):
# For eg, passing pipeline ID and with_log=False -> executes pipeline
# For FE however we call the same API twice
# (first call generates execution ID)
- @swagger_auto_schema(manual_parameters=[with_log])
def execute(self, request: Request) -> Response:
- # TODO: Use a serializer instead to validate args
- with_log = request.query_params.get(WFExecKey.WITH_LOG)
serializer: ExecuteSerializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
execution_id = serializer.validated_data.get("execution_id", None)
@@ -74,7 +59,6 @@ class PipelineViewSet(viewsets.ModelViewSet):
request=request,
pipeline_id=pipeline_id,
execution_id=execution_id,
- with_log=with_log,
)
pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
serializer = PipelineSerializer(pipeline)
diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.py
index 20a2d7d29..141df78f1 100644
--- a/backend/scheduler/helper.py
+++ b/backend/scheduler/helper.py
@@ -1,16 +1,70 @@
import logging
+import uuid
+from typing import Any
-from django.core.exceptions import ValidationError
-from scheduler import views
+from django.db import connection
+from pipeline.models import Pipeline
+from pipeline.pipeline_processor import PipelineProcessor
+from rest_framework.serializers import ValidationError
from scheduler.constants import SchedulerConstants as SC
from scheduler.exceptions import JobDeletionError, JobSchedulingError
from scheduler.serializer import AddJobSerializer
-from scheduler.tasks import delete_periodic_task, disable_task, enable_task
+from scheduler.tasks import (
+ create_periodic_task,
+ delete_periodic_task,
+ disable_task,
+ enable_task,
+)
+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey
+from workflow_manager.workflow.serializers import ExecuteWorkflowSerializer
logger = logging.getLogger(__name__)
class SchedulerHelper:
+
+ @staticmethod
+ def _schedule_task_job(pipeline_id: str, job_data: Any) -> None:
+ if "cron_string" not in job_data:
+ raise ValidationError("cron_string is required in job_data")
+ if "id" not in job_data:
+ raise ValidationError("id is required in job_data")
+ if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):
+ raise ValidationError("job_kwargs is required to be a dict")
+
+ cron_string = job_data.get("cron_string")
+ name = job_data.get("name")
+ job_kwargs = job_data.get("job_kwargs", {})
+ task_data = job_kwargs.get("data", {})
+
+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
+ task_data[WorkflowKey.WF_ID] = pipeline.workflow.id
+ execution_id = str(uuid.uuid4())
+ task_data[WorkflowExecutionKey.EXECUTION_ID] = execution_id
+ serializer = ExecuteWorkflowSerializer(data=task_data)
+ serializer.is_valid(raise_exception=True)
+ workflow_id = serializer.get_workflow_id(serializer.validated_data)
+ # TODO: Remove unused argument in execute_pipeline_task
+ execution_action = serializer.get_execution_action(serializer.validated_data)
+ org_schema = connection.tenant.schema_name
+
+ create_periodic_task(
+ cron_string=cron_string,
+ task_name=pipeline.pk,
+ task_path="scheduler.tasks.execute_pipeline_task",
+ task_args=[
+ str(workflow_id),
+ org_schema,
+ execution_action or "",
+ execution_id,
+ str(pipeline.pk),
+ # Added to remain backward compatible - remove after data migration
+ # which removes unused args in execute_pipeline_task
+ bool(False),
+ str(name),
+ ],
+ )
+
@staticmethod
def add_job(pipeline_id: str, cron_string: str = SC.DEFAULT_CRON_STRING) -> None:
logger.info(f"Scheduling job for {pipeline_id} with {cron_string}")
@@ -20,22 +74,12 @@ class SchedulerHelper:
SC.CRON_STRING: cron_string,
SC.NAME: name,
}
+ job_serializer = AddJobSerializer(data=job_serialize_data)
+ job_serializer.is_valid(raise_exception=True)
try:
- job_serializer = AddJobSerializer(data=job_serialize_data)
- job_serializer.is_valid(raise_exception=True)
- except ValidationError as e:
- logger.error(f"Validation error while scheduling job: {e}")
- raise JobSchedulingError
- except Exception as e:
- logger.error(f"Unhandled exception while scheduling job: {e}")
- raise JobSchedulingError
-
- try:
- # Celery based scheduler
- scheduler_response = views.schedule_task_job(
+ SchedulerHelper._schedule_task_job(
pipeline_id, job_serializer.validated_data
)
- logger.info(f"Scheduler response: {scheduler_response}")
except Exception as e:
logger.error(f"Exception while adding job: {e}")
raise JobSchedulingError
diff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.py
index 2ea3c3b53..bc0e32f42 100644
--- a/backend/scheduler/serializer.py
+++ b/backend/scheduler/serializer.py
@@ -1,7 +1,6 @@
import logging
from typing import Any
-from django.conf import settings
from pipeline.manager import PipelineManager
from rest_framework import serializers
from scheduler.constants import SchedulerConstants as SC
@@ -23,13 +22,6 @@ class JobKwargsSerializer(serializers.Serializer):
data = serializers.JSONField()
-class SchedulerKwargsSerializer(serializers.Serializer):
- coalesce = serializers.BooleanField()
- misfire_grace_time = serializers.IntegerField(allow_null=True, required=False)
- max_instances = serializers.IntegerField()
- replace_existing = serializers.BooleanField()
-
-
class AddJobSerializer(serializers.Serializer):
id = serializers.CharField(max_length=FieldLength.UUID_LENGTH)
cron_string = serializers.CharField(max_length=FieldLength.CRON_LENGTH)
@@ -37,7 +29,6 @@ class AddJobSerializer(serializers.Serializer):
max_length=JOB_NAME_LENGTH, required=False, allow_blank=True
)
job_kwargs = JobKwargsSerializer(write_only=True)
- scheduler_kwargs = SchedulerKwargsSerializer(write_only=True)
def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:
if SC.NAME not in data:
@@ -47,5 +38,4 @@ class AddJobSerializer(serializers.Serializer):
pipeline_id=data[SC.ID]
)
)
- data[SC.SCHEDULER_KWARGS] = settings.SCHEDULER_KWARGS
return super().to_internal_value(data) # type: ignore
diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py
index b42244576..71673bc6b 100644
--- a/backend/scheduler/tasks.py
+++ b/backend/scheduler/tasks.py
@@ -8,7 +8,6 @@ from django_celery_beat.models import CrontabSchedule, PeriodicTask
from django_tenants.utils import get_tenant_model, tenant_context
from pipeline.models import Pipeline
from pipeline.pipeline_processor import PipelineProcessor
-from pymysql import IntegrityError
from workflow_manager.workflow.models.workflow import Workflow
from workflow_manager.workflow.workflow_helper import WorkflowHelper
@@ -21,12 +20,8 @@ def create_periodic_task(
task_path: str,
task_args: list[Any],
) -> None:
- try:
- # Convert task_args to JSON
- task_args_json = json.dumps(task_args)
- except TypeError as te:
- logger.error(f"Failed : task_args is not JSON serializable: {te}")
- return
+ # Convert task_args to JSON
+ task_args_json = json.dumps(task_args)
# Parse the cron string
minute, hour, day_of_month, month_of_year, day_of_week = cron_string.split()
@@ -39,21 +34,15 @@ def create_periodic_task(
day_of_month=day_of_month,
month_of_year=month_of_year,
)
-
- logger.info(f"Creating periodic task with cron string: {cron_string}")
+ logger.info(f"Creating periodic task {task_name} with cron string: {cron_string}")
# Create periodic task
- try:
- logger.info(f"Running periodic task addition to beat table: {schedule}")
- PeriodicTask.objects.create(
- crontab=schedule,
- name=task_name,
- task=task_path,
- args=task_args_json,
- )
- logger.info(f"Completed running task addition to beat table: {task_name}")
- except Exception as e:
- logger.error(f"Failed to create periodic task: {e}")
+ PeriodicTask.objects.create(
+ crontab=schedule,
+ name=task_name,
+ task=task_path,
+ args=task_args_json,
+ )
def update_pipeline(
@@ -61,19 +50,21 @@ def update_pipeline(
status: tuple[str, str],
is_active: Optional[bool] = None,
) -> Any:
- if pipeline_guid:
- check_active = True
- if is_active is True:
- check_active = False
- pipeline: Pipeline = PipelineProcessor.fetch_pipeline(
- pipeline_id=pipeline_guid, check_active=check_active
- )
- PipelineProcessor.update_pipeline_status(
- pipeline=pipeline, is_end=True, status=status, is_active=is_active
- )
- logger.info(f"Updated pipeline status: {status}")
+ if not pipeline_guid:
+ return
+ check_active = True
+ if is_active is True:
+ check_active = False
+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(
+ pipeline_id=pipeline_guid, check_active=check_active
+ )
+ PipelineProcessor.update_pipeline_status(
+ pipeline=pipeline, is_end=True, status=status, is_active=is_active
+ )
+ logger.info(f"Updated pipeline status: {status}")
+# TODO: Remove unused args with a migration
@shared_task
def execute_pipeline_task(
workflow_id: Any,
@@ -93,20 +84,17 @@ def execute_pipeline_task(
)
with tenant_context(tenant):
workflow = Workflow.objects.get(id=workflow_id)
- try:
- logger.info(f"Executing workflow: {workflow}")
- update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)
- execution_response = WorkflowHelper.complete_execution(
- workflow, execution_id, pipepline_id, with_logs
- )
- logger.info(f"Execution response: {execution_response}")
- update_pipeline(pipepline_id, Pipeline.PipelineStatus.SUCCESS)
- except IntegrityError:
- logger.error(f"Failed to complete execution for workflow: {workflow}")
- update_pipeline(pipepline_id, Pipeline.PipelineStatus.FAILURE)
+ logger.info(f"Executing workflow: {workflow}")
+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)
+ execution_response = WorkflowHelper.complete_execution(
+ workflow, execution_id, pipepline_id
+ )
+ logger.info(f"Execution response: {execution_response}")
+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.SUCCESS)
logger.info(f"Execution completed for pipeline: {name}")
except Exception as e:
logger.error(f"Failed to execute pipeline: {name}. Error: {e}")
+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.FAILURE)
def delete_periodic_task(task_name: str) -> None:
diff --git a/backend/scheduler/views.py b/backend/scheduler/views.py
deleted file mode 100644
index 1e02a93d1..000000000
--- a/backend/scheduler/views.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import logging
-import uuid
-from typing import Any
-
-from django.db import connection
-from pipeline.models import Pipeline
-from pipeline.pipeline_processor import PipelineProcessor
-from rest_framework.response import Response
-from scheduler.tasks import create_periodic_task
-from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey
-from workflow_manager.workflow.exceptions import WorkflowExecutionBadRequestException
-from workflow_manager.workflow.serializers import ExecuteWorkflowSerializer
-from workflow_manager.workflow.views import WorkflowViewSet
-
-logger = logging.getLogger(__name__)
-
-
-def schedule_task_job(pipeline_id: str, job_data: Any) -> Response:
- try:
- # Validating request
- if "cron_string" not in job_data:
- return Response({"error": "cron_string is required"}, status=400)
- if "id" not in job_data:
- return Response({"error": "id is required"}, status=400)
- if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):
- return Response(
- {"error": "job_kwargs is required to be a dict"}, status=400
- )
-
- cron_string = job_data.get("cron_string")
- name = job_data.get("name")
- job_kwargs = job_data.get("job_kwargs", {})
- task_data = job_kwargs.get("data", {})
- params = job_kwargs.get("params", {})
- with_logs = params.get("with_logs", False)
-
- pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
-
- task_data[WorkflowKey.WF_ID] = pipeline.workflow.id
- execution_id = str(uuid.uuid4())
- task_data[WorkflowExecutionKey.EXECUTION_ID] = execution_id
- workflow_viewset = WorkflowViewSet()
- workflow_viewset.serializer_class = ExecuteWorkflowSerializer
- serializer = ExecuteWorkflowSerializer(data=task_data)
-
- if not serializer.is_valid():
- raise WorkflowExecutionBadRequestException(
- workflow_viewset.get_error_from_serializer(serializer.errors)
- )
-
- workflow_id = serializer.get_workflow_id(serializer.validated_data)
-
- execution_action = serializer.get_execution_action(serializer.validated_data)
- org_schema = connection.tenant.schema_name
-
- create_periodic_task(
- cron_string=cron_string,
- task_name=pipeline.pk,
- task_path="scheduler.tasks.execute_pipeline_task",
- task_args=[
- str(workflow_id),
- org_schema,
- execution_action or "",
- execution_id,
- str(pipeline.pk),
- bool(with_logs),
- str(name),
- ],
- )
-
- return Response({"message": "Job scheduled successfully"}, status=200)
-
- except Exception as e:
- logger.error(f"Exception while scheduling job: {e}")
- return Response({"error": str(e)}, status=500)
diff --git a/backend/workflow_manager/workflow/exceptions.py b/backend/workflow_manager/workflow/exceptions.py
index 56f111600..839a36429 100644
--- a/backend/workflow_manager/workflow/exceptions.py
+++ b/backend/workflow_manager/workflow/exceptions.py
@@ -1,5 +1,3 @@
-from typing import Optional
-
from rest_framework.exceptions import APIException
@@ -58,18 +56,6 @@ class WorkflowExecutionNotExist(APIException):
default_detail = "Workflow execution does not exist"
-class WorkflowExecutionBadRequestException(APIException):
- status_code = 400
- default_detail = "Bad request"
-
- def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
- if detail is not None:
- self.detail = detail
- if code is not None:
- self.code = code
- super().__init__(detail, code)
-
-
class ToolValidationError(APIException):
status_code = 400
default_detail = "Tool validation error"
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index b332c9ecb..a108c196a 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -275,13 +275,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
)
def publish_initial_tool_execution_logs(
- self, current_step: int, total_step: int, file_name: str
+ self, current_file_idx: int, total_files: int, file_name: str
) -> None:
"""Publishes the initial logs for tool execution.
Args:
- current_step (int): The current step number.
- total_step (int): The total number of steps.
+ current_file_idx (int): 1-based index for the current file being processed
+ total_files (int): The total number of files to process
file_name (str): The name of the file being processed.
Returns:
@@ -290,7 +290,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.publish_update_log(
component=LogComponent.STATUS_BAR,
state=LogState.MESSAGE,
- message=f"Processing file {current_step}/{total_step}",
+ message=f"Processing file {file_name} {current_file_idx}/{total_files}",
)
self.publish_log(f"Processing file {file_name}")
@@ -348,8 +348,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
def initiate_tool_execution(
self,
- current_step: int,
- total_step: int,
+ current_file_idx: int,
+ total_files: int,
file_name: str,
single_step: bool,
) -> None:
@@ -357,11 +357,11 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
workflow.
Args:
- current_step (int): The current step number in the workflow.
- total_step (int): The total number of steps in the workflow.
- file_name (str): The name of the file being processed.
+ current_file_idx (int): 1-based index for the current file being processed
+ total_step (int): The total number of files to process in the workflow
+ file_name (str): The name of the file being processed
single_step (bool): Flag indicating whether the execution is in
- single-step mode.
+ single-step mode
Returns:
None
@@ -372,11 +372,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
execution_type = ExecutionType.COMPLETE
if single_step:
execution_type = ExecutionType.STEP
- self.publish_initial_tool_execution_logs(current_step, total_step, file_name)
+ self.publish_initial_tool_execution_logs(
+ current_file_idx, total_files, file_name
+ )
self._handle_execution_type(execution_type)
source_status_message = (
- f"({current_step}/{total_step})Processing file {file_name}"
+ f"({current_file_idx}/{total_files})Processing file {file_name}"
)
self.publish_update_log(
state=LogState.RUNNING,
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index 7ce7b2cdc..978153432 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -58,13 +58,6 @@ def make_execution_response(response: ExecutionResponse) -> Any:
return ExecuteWorkflowResponseSerializer(response).data
-def handle_false(string: str) -> bool:
- if string.lower() == "false":
- return False
- else:
- return True
-
-
class WorkflowViewSet(viewsets.ModelViewSet):
versioning_class = URLPathVersioning
permission_classes = [IsOwner]
@@ -153,13 +146,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution = WorkflowHelper.get_current_execution(pk)
return Response(make_execution_response(execution), status=status.HTTP_200_OK)
- def get_error_from_serializer(self, error_details: dict[str, Any]) -> Optional[str]:
- """Validation error."""
- error_key = next(iter(error_details))
- # Get the first error message
- error_message: str = f"{error_details[error_key][0]} : {error_key}"
- return error_message
-
def get_workflow_by_id_or_project_id(
self,
workflow_id: Optional[str] = None,
@@ -189,12 +175,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
self,
request: Request,
pipeline_guid: Optional[str] = None,
- with_log: Optional[bool] = None,
) -> Response:
- if with_log is not None:
- # Handle string field
- with_log = handle_false(str(with_log))
-
self.serializer_class = ExecuteWorkflowSerializer
serializer = ExecuteWorkflowSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
@@ -220,7 +201,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_action=execution_action,
execution_id=execution_id,
pipeline_guid=pipeline_guid,
- with_log=with_log,
hash_values_of_files=hashes_of_files,
)
return Response(
@@ -232,8 +212,8 @@ class WorkflowViewSet(viewsets.ModelViewSet):
update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)
raise exception
except MissingEnvException as exception:
- update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)
logger.error(f"Error while executing workflow: {exception}")
+ update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)
return Response(
{"error": "Please check the logs for more details: " + str(exception)},
status=status.HTTP_400_BAD_REQUEST,
@@ -253,7 +233,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_action: Optional[str] = None,
execution_id: Optional[str] = None,
pipeline_guid: Optional[str] = None,
- with_log: Optional[bool] = None,
hash_values_of_files: dict[str, str] = {},
) -> ExecutionResponse:
if execution_action is not None:
@@ -271,7 +250,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
workflow=workflow,
execution_id=execution_id,
pipeline_id=pipeline_guid,
- log_required=with_log,
hash_values_of_files=hash_values_of_files,
)
update_pipeline(pipeline_guid, Pipeline.PipelineStatus.SUCCESS)
@@ -279,7 +257,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_response = WorkflowHelper.complete_execution(
workflow=workflow,
execution_id=execution_id,
- log_required=with_log,
hash_values_of_files=hash_values_of_files,
)
return execution_response
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index c74f2ccb2..4ac6364d5 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -122,8 +122,8 @@ class WorkflowHelper:
file_number = index + 1
try:
is_executed, error = WorkflowHelper.process_file(
- current_step=file_number,
- total_step=total_files,
+ current_file_idx=file_number,
+ total_files=total_files,
input_file=input_file,
workflow=workflow,
source=source,
@@ -153,8 +153,8 @@ class WorkflowHelper:
@staticmethod
def process_file(
- current_step: int,
- total_step: int,
+ current_file_idx: int,
+ total_files: int,
input_file: str,
workflow: Workflow,
source: SourceConnector,
@@ -172,7 +172,7 @@ class WorkflowHelper:
)
try:
execution_service.initiate_tool_execution(
- current_step, total_step, file_name, single_step
+ current_file_idx, total_files, file_name, single_step
)
file_history = FileHistoryHelper.get_file_history(
workflow=workflow, cache_key=file_hash
@@ -192,7 +192,7 @@ class WorkflowHelper:
error = str(e)
execution_service.publish_update_log(
LogState.RUNNING,
- f"{file_name} is processing",
+ f"Processing output for {file_name}",
LogComponent.DESTINATION,
)
destination.handle_output(
@@ -204,7 +204,7 @@ class WorkflowHelper:
)
execution_service.publish_update_log(
LogState.SUCCESS,
- f"{file_name} is Processed successfully",
+ f"{file_name}'s output is processed successfully",
LogComponent.DESTINATION,
)
return is_executed, error
@@ -456,32 +456,17 @@ class WorkflowHelper:
workflow: Workflow,
execution_id: Optional[str] = None,
pipeline_id: Optional[str] = None,
- log_required: Optional[bool] = True,
hash_values_of_files: dict[str, str] = {},
) -> ExecutionResponse:
- # For scheduler workflow execution
- logger.info(f"Workflow pipeline ID: {pipeline_id}")
if pipeline_id:
- return WorkflowHelper.run_workflow(
- workflow=workflow,
- hash_values_of_files=hash_values_of_files,
+ logger.info(f"Executing pipeline: {pipeline_id}")
+ response: ExecutionResponse = WorkflowHelper.execute_workflow_async(
+ workflow_id=workflow.id,
pipeline_id=pipeline_id,
+ execution_id=str(uuid.uuid4()),
+ hash_values_of_files=hash_values_of_files,
)
-
- # TODO: Review required here.
- if log_required is not None and not log_required:
- # Without log and log Id
- if pipeline_id:
- # pipeline scheduled execution
- execution_id = str(uuid.uuid4())
- response: ExecutionResponse = WorkflowHelper.execute_workflow_async(
- workflow_id=workflow.id,
- pipeline_id=pipeline_id,
- execution_id=execution_id,
- hash_values_of_files=hash_values_of_files,
- )
- return response
- return WorkflowHelper.run_workflow(workflow=workflow)
+ return response
if execution_id is None:
# Creating execution entity and return
return WorkflowHelper.create_and_make_execution_response(
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 7e3d73273..cad6cd16a 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -218,7 +218,7 @@ class WorkflowExecutionService:
self.publish_log(log_message, step=actual_step)
except Exception as error:
- logger.info(f"Error while execution tool {tool_uid}: {str(error)}")
+ logger.info(f"Error while running tool {tool_uid}: {str(error)}")
self.publish_log(str(error), LogLevel.ERROR, step=actual_step)
self.publish_update_log(
state=LogState.ERROR,