Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

missed workflow final logs (#1315)

alidf45478456b3
diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.py
index 1769f527e..299fc75d2 100644
--- a/backend/workflow_manager/utils/workflow_log.py
+++ b/backend/workflow_manager/utils/workflow_log.py
@@ -89,3 +89,60 @@ class WorkflowLog:
self.publish_update_log(
LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW
)
+
+ def publish_final_workflow_logs(
+ self,
+ total_files: int,
+ successful_files: int,
+ failed_files: int,
+ ) -> None:
+ """Publishes the final logs for the workflow.
+
+ Args:
+ total_files (int): The total number of matched files.
+ successful_files (int): The number of successfully executed files.
+ failed_files (int): The number of failed files.
+
+ Returns:
+ None
+ """
+ self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)
+ self.publish_update_log(
+ LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
+ )
+ self.publish_log(
+ f"Total files: {total_files}, "
+ f"{successful_files} successfully executed and {failed_files} error(s)"
+ )
+
+ def publish_average_cost_log(
+ self,
+ logger: logging.Logger,
+ total_files: int,
+ execution_id: str,
+ total_cost: float,
+ ):
+ """Publishes the average cost log for the workflow.
+
+ Args:
+ logger (logging.Logger): The logger to use for logging.
+ total_files (int): The total number of files.
+ execution_id (str): The ID of the execution.
+ total_cost (float): The total cost of the execution.
+
+ Returns:
+ None
+ """
+ try:
+ if total_cost is not None:
+ average_cost = round(total_cost / total_files, 5)
+ self.publish_log(
+ message=(
+ f"The average cost per file for execution "
+ f"'{execution_id}' is '${average_cost}'"
+ )
+ )
+ except Exception as e:
+ logger.warning(
+ f"Unable to get aggregated cost for '{execution_id}': {str(e)}"
+ )
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index 720108612..34dd2931e 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -274,64 +274,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
)
raise WorkflowExecutionError(message) from exception
- def publish_initial_workflow_logs(self, total_files: int) -> None:
- """Publishes the initial logs for the workflow.
-
- Args:
- total_files (int): The total number of matched files.
-
- Returns:
- None
- """
- self.publish_log(f"Total matched files: {total_files}")
- self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)
- self.publish_update_log(
- LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW
- )
-
- def publish_final_workflow_logs(
- self,
- total_files: int,
- successful_files: int,
- failed_files: int,
- ) -> None:
- """Publishes the final logs for the workflow.
-
- Returns:
- None
- """
- self.publish_average_cost_log(total_files=total_files)
-
- # To not associate final logs with a file execution
- self.file_execution_id = None
-
- self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)
- self.publish_update_log(
- LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
- )
- self.publish_log(
- f"Total files: {total_files}, "
- f"{successful_files} successfully executed and {failed_files} error(s)"
- )
-
- def publish_average_cost_log(self, total_files: int):
- try:
- execution = WorkflowExecution.objects.get(pk=self.execution_id)
- total_cost = execution.aggregated_usage_cost
-
- if total_cost is not None:
- average_cost = round(total_cost / total_files, 5)
- self.publish_log(
- message=(
- f"The average cost per file for execution "
- f"'{self.execution_id}' is '${average_cost}'"
- )
- )
- except Exception as e:
- logger.warning(
- f"Unable to get aggregated cost for '{self.execution_id}': {str(e)}"
- )
-
def log_total_cost_per_file(self, run_id: str, file_name: str):
"""Log cost details to user
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 69a2f6677..3d04ab850 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -217,6 +217,12 @@ class FileExecutionTasks:
organization_id = organization.organization_id
pipeline_id = str(workflow_execution.pipeline_id)
log_events_id = workflow_execution.execution_log_id
+ workflow_log = WorkflowLog(
+ execution_id=execution_id,
+ log_stage=LogStage.FINALIZE,
+ organization_id=organization_id,
+ pipeline_id=pipeline_id,
+ )
StateStore.set(Common.LOG_EVENTS_ID, log_events_id)
# Set organization ID in StateStore
@@ -248,6 +254,17 @@ class FileExecutionTasks:
DestinationConnector.delete_execution_and_api_storage_dir(
workflow_id=workflow.id, execution_id=execution_id
)
+ workflow_log.publish_average_cost_log(
+ logger=logger,
+ total_files=total_files,
+ execution_id=execution_id,
+ total_cost=workflow_execution.aggregated_usage_cost,
+ )
+ workflow_log.publish_final_workflow_logs(
+ total_files=total_files,
+ successful_files=total_successful,
+ failed_files=total_failed,
+ )
return {
"execution_id": execution_id,
"total_files": total_files,
@@ -311,6 +328,11 @@ class FileExecutionTasks:
workflow_log,
workflow_file_execution,
):
+ cls._complete_execution(
+ workflow_file_execution=workflow_file_execution,
+ workflow_log=workflow_log,
+ error=early_result.error,
+ )
return early_result
# Core Execution Phase
@@ -617,7 +639,7 @@ class FileExecutionTasks:
workflow_log: WorkflowLog,
error: str | None,
) -> None:
- """Final cleanup and status updates for the execution."""
+ """Final status updates for the file execution."""
try:
# Update execution status
final_status = ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED
@@ -634,7 +656,7 @@ class FileExecutionTasks:
)
except Exception as e:
- logger.error(f"Completion cleanup failed: {str(e)}", exc_info=True)
+ logger.error(f"Completion status update failed: {str(e)}", exc_info=True)
@classmethod
def _build_final_result(
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py
index dc141bb61..dae4550de 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py
@@ -36,6 +36,7 @@ class LogStage(Enum):
SOURCE = "SOURCE"
DESTINATION = "DESTINATION"
INITIALIZE = "INITIALIZE"
+ FINALIZE = "FINALIZE"
PROCESSING = "PROCESSING"