LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
missed workflow final logs (#1315)
alidf45478456b3
diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.pyindex 1769f527e..299fc75d2 100644--- a/backend/workflow_manager/utils/workflow_log.py+++ b/backend/workflow_manager/utils/workflow_log.py@@ -89,3 +89,60 @@ class WorkflowLog:self.publish_update_log(LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW)++ def publish_final_workflow_logs(+ self,+ total_files: int,+ successful_files: int,+ failed_files: int,+ ) -> None:+ """Publishes the final logs for the workflow.++ Args:+ total_files (int): The total number of matched files.+ successful_files (int): The number of successfully executed files.+ failed_files (int): The number of failed files.++ Returns:+ None+ """+ self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)+ self.publish_update_log(+ LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW+ )+ self.publish_log(+ f"Total files: {total_files}, "+ f"{successful_files} successfully executed and {failed_files} error(s)"+ )++ def publish_average_cost_log(+ self,+ logger: logging.Logger,+ total_files: int,+ execution_id: str,+ total_cost: float,+ ):+ """Publishes the average cost log for the workflow.++ Args:+ logger (logging.Logger): The logger to use for logging.+ total_files (int): The total number of files.+ execution_id (str): The ID of the execution.+ total_cost (float): The total cost of the execution.++ Returns:+ None+ """+ try:+ if total_cost is not None:+ average_cost = round(total_cost / total_files, 5)+ self.publish_log(+ message=(+ f"The average cost per file for execution "+ f"'{execution_id}' is '${average_cost}'"+ )+ )+ except Exception as e:+ logger.warning(+ f"Unable to get aggregated cost for '{execution_id}': {str(e)}"+ )diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 720108612..34dd2931e 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -274,64 +274,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):)raise WorkflowExecutionError(message) from exception- def publish_initial_workflow_logs(self, total_files: int) -> None:- """Publishes the initial logs for the workflow.-- Args:- total_files (int): The total number of matched files.-- Returns:- None- """- self.publish_log(f"Total matched files: {total_files}")- self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)- self.publish_update_log(- LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW- )-- def publish_final_workflow_logs(- self,- total_files: int,- successful_files: int,- failed_files: int,- ) -> None:- """Publishes the final logs for the workflow.-- Returns:- None- """- self.publish_average_cost_log(total_files=total_files)-- # To not associate final logs with a file execution- self.file_execution_id = None-- self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)- self.publish_update_log(- LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW- )- self.publish_log(- f"Total files: {total_files}, "- f"{successful_files} successfully executed and {failed_files} error(s)"- )-- def publish_average_cost_log(self, total_files: int):- try:- execution = WorkflowExecution.objects.get(pk=self.execution_id)- total_cost = execution.aggregated_usage_cost-- if total_cost is not None:- average_cost = round(total_cost / total_files, 5)- self.publish_log(- message=(- f"The average cost per file for execution "- f"'{self.execution_id}' is '${average_cost}'"- )- )- except Exception as e:- logger.warning(- f"Unable to get aggregated cost for '{self.execution_id}': {str(e)}"- )-def log_total_cost_per_file(self, run_id: str, file_name: str):"""Log cost details to userdiff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 69a2f6677..3d04ab850 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -217,6 +217,12 @@ class FileExecutionTasks:organization_id = organization.organization_idpipeline_id = str(workflow_execution.pipeline_id)log_events_id = workflow_execution.execution_log_id+ workflow_log = WorkflowLog(+ execution_id=execution_id,+ log_stage=LogStage.FINALIZE,+ organization_id=organization_id,+ pipeline_id=pipeline_id,+ )StateStore.set(Common.LOG_EVENTS_ID, log_events_id)# Set organization ID in StateStore@@ -248,6 +254,17 @@ class FileExecutionTasks:DestinationConnector.delete_execution_and_api_storage_dir(workflow_id=workflow.id, execution_id=execution_id)+ workflow_log.publish_average_cost_log(+ logger=logger,+ total_files=total_files,+ execution_id=execution_id,+ total_cost=workflow_execution.aggregated_usage_cost,+ )+ workflow_log.publish_final_workflow_logs(+ total_files=total_files,+ successful_files=total_successful,+ failed_files=total_failed,+ )return {"execution_id": execution_id,"total_files": total_files,@@ -311,6 +328,11 @@ class FileExecutionTasks:workflow_log,workflow_file_execution,):+ cls._complete_execution(+ workflow_file_execution=workflow_file_execution,+ workflow_log=workflow_log,+ error=early_result.error,+ )return early_result# Core Execution Phase@@ -617,7 +639,7 @@ class FileExecutionTasks:workflow_log: WorkflowLog,error: str | None,) -> None:- """Final cleanup and status updates for the execution."""+ """Final status updates for the file execution."""try:# Update execution statusfinal_status = ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED@@ -634,7 +656,7 @@ class FileExecutionTasks:)except Exception as e:- logger.error(f"Completion cleanup failed: {str(e)}", exc_info=True)+ logger.error(f"Completion status update failed: {str(e)}", exc_info=True)@classmethoddef _build_final_result(diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py b/unstract/workflow-execution/src/unstract/workflow_execution/enums.pyindex dc141bb61..dae4550de 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py@@ -36,6 +36,7 @@ class LogStage(Enum):SOURCE = "SOURCE"DESTINATION = "DESTINATION"INITIALIZE = "INITIALIZE"+ FINALIZE = "FINALIZE"PROCESSING = "PROCESSING"