Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Minor pipeline log related enhancements (#594)

Chandrasekharan M3be69d922aa0
diff --git a/.gitignore b/.gitignore
index 82fca7fce..5ab655842 100644
--- a/.gitignore
+++ b/.gitignore
@@ -612,6 +612,12 @@ backend/plugins/authentication/*
# Processor Plugins
backend/plugins/processor/*
+# Subscription Plugins
+backend/plugins/subscription/*
+
+# BE pluggable-apps
+backend/pluggable_apps/*
+
# FE Plugins
frontend/src/plugins/*
@@ -645,8 +651,5 @@ backend/backend/settings/*
# Local Dependencies for docker testing
unstract/unstract-sdk/
-# BE pluggable-apps
-backend/pluggable_apps/*
-
# Backend requirements.txt
backend/requirements.txt
diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py
index b922f4e64..a7e905904 100644
--- a/backend/scheduler/tasks.py
+++ b/backend/scheduler/tasks.py
@@ -95,21 +95,26 @@ def execute_pipeline_task(
pipeline_id=pipepline_id, check_active=True
)
workflow = pipeline.workflow
- logger.info(f"Executing workflow: {workflow} by pipeline {pipeline}")
+ logger.info(
+ f"Executing pipeline: {pipeline}, "
+ f"workflow: {workflow}, pipeline name: {name}"
+ )
if (
subscription_loader
and subscription_loader[0]
and not validate_etl_run(org_schema)
):
try:
- logger.info(f"Disabling ETL task: {pipepline_id}")
+ logger.info(
+ f"Subscription expired for '{org_schema}', "
+ f"disabling pipeline: {pipepline_id}"
+ )
disable_task(pipepline_id)
except Exception as e:
logger.warning(
f"Failed to disable task: {pipepline_id}. Error: {e}"
)
return
- logger.info(f"Executing workflow: {workflow}")
PipelineProcessor.initialize_pipeline_sync(pipeline_id=pipepline_id)
PipelineProcessor.update_pipeline(
pipepline_id, Pipeline.PipelineStatus.INPROGRESS
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index c94d7eb70..90db646e4 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -365,10 +365,11 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.publish_log("Trying to fetch results from cache")
@staticmethod
- def update_execution_status(execution_id: str, status: ExecutionStatus) -> None:
+ def update_execution_err(execution_id: str, err_msg: str = "") -> None:
try:
execution = WorkflowExecution.objects.get(pk=execution_id)
- execution.status = status.value
+ execution.status = ExecutionStatus.ERROR.value
+ execution.error_message = err_msg
execution.save()
except WorkflowExecution.DoesNotExist:
logger.error(f"execution doesn't exist {execution_id}")
@@ -404,10 +405,3 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
workflow: Workflow,
) -> WorkflowDto:
return WorkflowDto(id=workflow.id)
-
- @staticmethod
- def get_execution_by_id(execution_id: str) -> Optional[WorkflowExecution]:
- try:
- return WorkflowExecution.objects.get(id=execution_id)
- except WorkflowExecution.DoesNotExist:
- return None
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index 199c5005e..ccc257331 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -421,9 +421,7 @@ class WorkflowHelper:
logger.info(f"Job {async_execution} enqueued.")
celery_result = task.to_dict()
task_result = celery_result.get("result")
- workflow_execution = WorkflowExecutionServiceHelper.get_execution_by_id(
- execution_id=execution_id
- )
+ workflow_execution = WorkflowExecution.objects.get(id=execution_id)
execution_response = ExecutionResponse(
workflow_id,
execution_id,
@@ -439,8 +437,8 @@ class WorkflowHelper:
message=WorkflowMessages.CELERY_TIMEOUT_MESSAGE,
)
except Exception as error:
- WorkflowExecutionServiceHelper.update_execution_status(
- execution_id, ExecutionStatus.ERROR
+ WorkflowExecutionServiceHelper.update_execution_err(
+ execution_id, str(error)
)
logger.error(f"Errors while job enqueueing {str(error)}")
logger.error(f"Error {traceback.format_exc()}")