LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Avoid pipeline status update for API deployments (#398)
Chandrasekharan Meae2e6b4dd76
diff --git a/backend/pdm.lock b/backend/pdm.lockindex c4cb33590..7ba96df7f 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy", "test"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:65995bec9035d980c5dbfa609f531964e573eb0bb13fa53c207a4ae006c253b0"+content_hash = "sha256:5924a0dcf9f3a5a94264fa1ed7a3d8bd5a2c7db3b565a2ba887c93e06dbb0e09"[[package]]name = "adlfs"diff --git a/backend/workflow_manager/workflow/models/execution.py b/backend/workflow_manager/workflow/models/execution.pyindex 8506d7e43..474c19244 100644--- a/backend/workflow_manager/workflow/models/execution.py+++ b/backend/workflow_manager/workflow/models/execution.py@@ -64,3 +64,13 @@ class WorkflowExecution(BaseModel):execution_time = models.FloatField(default=0, db_comment="execution time in seconds")++ def __str__(self) -> str:+ return (+ f"Workflow execution: {self.id} ("+ f"pipeline ID: {self.pipeline_id}, "+ f"workflow iD: {self.workflow_id}, "+ f"execution method: {self.execution_method}, "+ f"status: {self.status}, "+ f"error message: {self.error_message})"+ )diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex e7eea7590..0849b07ef 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -267,7 +267,7 @@ class WorkflowHelper:destination.validate()# Execution Processtry:- updated_execution = WorkflowHelper.process_input_files(+ workflow_execution = WorkflowHelper.process_input_files(workflow,source,destination,@@ -275,9 +275,30 @@ class WorkflowHelper:single_step=single_step,hash_values_of_files=hash_values_of_files,)+ # TODO: Update through signals+ WorkflowHelper._update_pipeline_status(+ pipeline_id=pipeline_id, workflow_execution=workflow_execution+ )+ return ExecutionResponse(+ str(workflow.id),+ str(workflow_execution.id),+ workflow_execution.status,+ log_id=str(execution_service.execution_log_id),+ error=workflow_execution.error_message,+ mode=workflow_execution.execution_mode,+ result=destination.api_results,+ )+ finally:+ destination.delete_execution_directory()++ @staticmethod+ def _update_pipeline_status(+ pipeline_id: Optional[str], workflow_execution: WorkflowExecution+ ) -> None:+ try:if pipeline_id:# Update pipeline status- if updated_execution.status != ExecutionStatus.ERROR.value:+ if workflow_execution.status != ExecutionStatus.ERROR.value:PipelineProcessor.update_pipeline(pipeline_id, Pipeline.PipelineStatus.SUCCESS)@@ -285,17 +306,14 @@ class WorkflowHelper:PipelineProcessor.update_pipeline(pipeline_id, Pipeline.PipelineStatus.FAILURE)- return ExecutionResponse(- str(workflow.id),- str(updated_execution.id),- updated_execution.status,- log_id=str(execution_service.execution_log_id),- error=updated_execution.error_message,- mode=updated_execution.execution_mode,- result=destination.api_results,+ # Expected exception since API deployments are not tracked in Pipeline+ except Pipeline.DoesNotExist:+ pass+ except Exception as e:+ logger.warning(+ f"Error updating pipeline {pipeline_id} status: {e}, "+ f"with workflow execution: {workflow_execution}")- finally:- destination.delete_execution_directory()@staticmethoddef get_status_of_async_task(diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex fb010b136..decce974d 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -837,7 +837,7 @@ def log_exceptions(e: HTTPException):"""code = 500if hasattr(e, "code"):- code = e.code+ code = e.code or codeif code >= 500:message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(