Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Avoid pipeline status update for API deployments (#398)

Chandrasekharan Meae2e6b4dd76
diff --git a/backend/pdm.lock b/backend/pdm.lock
index c4cb33590..7ba96df7f 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy", "test"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:65995bec9035d980c5dbfa609f531964e573eb0bb13fa53c207a4ae006c253b0"
+content_hash = "sha256:5924a0dcf9f3a5a94264fa1ed7a3d8bd5a2c7db3b565a2ba887c93e06dbb0e09"
[[package]]
name = "adlfs"
diff --git a/backend/workflow_manager/workflow/models/execution.py b/backend/workflow_manager/workflow/models/execution.py
index 8506d7e43..474c19244 100644
--- a/backend/workflow_manager/workflow/models/execution.py
+++ b/backend/workflow_manager/workflow/models/execution.py
@@ -64,3 +64,13 @@ class WorkflowExecution(BaseModel):
execution_time = models.FloatField(
default=0, db_comment="execution time in seconds"
)
+
+ def __str__(self) -> str:
+ return (
+ f"Workflow execution: {self.id} ("
+ f"pipeline ID: {self.pipeline_id}, "
+ f"workflow iD: {self.workflow_id}, "
+ f"execution method: {self.execution_method}, "
+ f"status: {self.status}, "
+ f"error message: {self.error_message})"
+ )
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index e7eea7590..0849b07ef 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -267,7 +267,7 @@ class WorkflowHelper:
destination.validate()
# Execution Process
try:
- updated_execution = WorkflowHelper.process_input_files(
+ workflow_execution = WorkflowHelper.process_input_files(
workflow,
source,
destination,
@@ -275,9 +275,30 @@ class WorkflowHelper:
single_step=single_step,
hash_values_of_files=hash_values_of_files,
)
+ # TODO: Update through signals
+ WorkflowHelper._update_pipeline_status(
+ pipeline_id=pipeline_id, workflow_execution=workflow_execution
+ )
+ return ExecutionResponse(
+ str(workflow.id),
+ str(workflow_execution.id),
+ workflow_execution.status,
+ log_id=str(execution_service.execution_log_id),
+ error=workflow_execution.error_message,
+ mode=workflow_execution.execution_mode,
+ result=destination.api_results,
+ )
+ finally:
+ destination.delete_execution_directory()
+
+ @staticmethod
+ def _update_pipeline_status(
+ pipeline_id: Optional[str], workflow_execution: WorkflowExecution
+ ) -> None:
+ try:
if pipeline_id:
# Update pipeline status
- if updated_execution.status != ExecutionStatus.ERROR.value:
+ if workflow_execution.status != ExecutionStatus.ERROR.value:
PipelineProcessor.update_pipeline(
pipeline_id, Pipeline.PipelineStatus.SUCCESS
)
@@ -285,17 +306,14 @@ class WorkflowHelper:
PipelineProcessor.update_pipeline(
pipeline_id, Pipeline.PipelineStatus.FAILURE
)
- return ExecutionResponse(
- str(workflow.id),
- str(updated_execution.id),
- updated_execution.status,
- log_id=str(execution_service.execution_log_id),
- error=updated_execution.error_message,
- mode=updated_execution.execution_mode,
- result=destination.api_results,
+ # Expected exception since API deployments are not tracked in Pipeline
+ except Pipeline.DoesNotExist:
+ pass
+ except Exception as e:
+ logger.warning(
+ f"Error updating pipeline {pipeline_id} status: {e}, "
+ f"with workflow execution: {workflow_execution}"
)
- finally:
- destination.delete_execution_directory()
@staticmethod
def get_status_of_async_task(
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index fb010b136..decce974d 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -837,7 +837,7 @@ def log_exceptions(e: HTTPException):
"""
code = 500
if hasattr(e, "code"):
- code = e.code
+ code = e.code or code
if code >= 500:
message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(