LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Minor pipeline log related enhancements (#594)
Chandrasekharan M3be69d922aa0
diff --git a/.gitignore b/.gitignoreindex 82fca7fce..5ab655842 100644--- a/.gitignore+++ b/.gitignore@@ -612,6 +612,12 @@ backend/plugins/authentication/*# Processor Pluginsbackend/plugins/processor/*+# Subscription Plugins+backend/plugins/subscription/*++# BE pluggable-apps+backend/pluggable_apps/*+# FE Pluginsfrontend/src/plugins/*@@ -645,8 +651,5 @@ backend/backend/settings/*# Local Dependencies for docker testingunstract/unstract-sdk/-# BE pluggable-apps-backend/pluggable_apps/*-# Backend requirements.txtbackend/requirements.txtdiff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex b922f4e64..a7e905904 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -95,21 +95,26 @@ def execute_pipeline_task(pipeline_id=pipepline_id, check_active=True)workflow = pipeline.workflow- logger.info(f"Executing workflow: {workflow} by pipeline {pipeline}")+ logger.info(+ f"Executing pipeline: {pipeline}, "+ f"workflow: {workflow}, pipeline name: {name}"+ )if (subscription_loaderand subscription_loader[0]and not validate_etl_run(org_schema)):try:- logger.info(f"Disabling ETL task: {pipepline_id}")+ logger.info(+ f"Subscription expired for '{org_schema}', "+ f"disabling pipeline: {pipepline_id}"+ )disable_task(pipepline_id)except Exception as e:logger.warning(f"Failed to disable task: {pipepline_id}. Error: {e}")return- logger.info(f"Executing workflow: {workflow}")PipelineProcessor.initialize_pipeline_sync(pipeline_id=pipepline_id)PipelineProcessor.update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESSdiff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex c94d7eb70..90db646e4 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -365,10 +365,11 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.publish_log("Trying to fetch results from cache")@staticmethod- def update_execution_status(execution_id: str, status: ExecutionStatus) -> None:+ def update_execution_err(execution_id: str, err_msg: str = "") -> None:try:execution = WorkflowExecution.objects.get(pk=execution_id)- execution.status = status.value+ execution.status = ExecutionStatus.ERROR.value+ execution.error_message = err_msgexecution.save()except WorkflowExecution.DoesNotExist:logger.error(f"execution doesn't exist {execution_id}")@@ -404,10 +405,3 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):workflow: Workflow,) -> WorkflowDto:return WorkflowDto(id=workflow.id)-- @staticmethod- def get_execution_by_id(execution_id: str) -> Optional[WorkflowExecution]:- try:- return WorkflowExecution.objects.get(id=execution_id)- except WorkflowExecution.DoesNotExist:- return Nonediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 199c5005e..ccc257331 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -421,9 +421,7 @@ class WorkflowHelper:logger.info(f"Job {async_execution} enqueued.")celery_result = task.to_dict()task_result = celery_result.get("result")- workflow_execution = WorkflowExecutionServiceHelper.get_execution_by_id(- execution_id=execution_id- )+ workflow_execution = WorkflowExecution.objects.get(id=execution_id)execution_response = ExecutionResponse(workflow_id,execution_id,@@ -439,8 +437,8 @@ class WorkflowHelper:message=WorkflowMessages.CELERY_TIMEOUT_MESSAGE,)except Exception as error:- WorkflowExecutionServiceHelper.update_execution_status(- execution_id, ExecutionStatus.ERROR+ WorkflowExecutionServiceHelper.update_execution_err(+ execution_id, str(error))logger.error(f"Errors while job enqueueing {str(error)}")logger.error(f"Error {traceback.format_exc()}")