Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix for null metadata in api-execution (#1304)

aliea331a5e73e7
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index a1795945f..4ce3ab773 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -27,8 +27,6 @@ from workflow_manager.endpoint_v2.exceptions import (
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtils
from workflow_manager.utils.workflow_log import WorkflowLog
-from workflow_manager.workflow_v2.enums import ExecutionStatus
-from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -199,6 +197,7 @@ class DestinationConnector(BaseConnector):
self,
file_name: str,
file_hash: FileHash,
+ file_history: FileHistory | None,
workflow: Workflow,
input_file_path: str,
file_execution_id: str = None,
@@ -206,13 +205,7 @@ class DestinationConnector(BaseConnector):
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
tool_execution_result: str | None = None
- metadata: str | None = None
- file_history = None
- if self.use_file_history:
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash.file_hash
- )
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
@@ -236,16 +229,6 @@ class DestinationConnector(BaseConnector):
self.workflow_log.publish_log(
message=f"File '{file_name}' processed successfully"
)
-
- if self.use_file_history and not file_history:
- FileHistoryHelper.create_file_history(
- file_hash=file_hash,
- workflow=workflow,
- status=ExecutionStatus.COMPLETED,
- result=tool_execution_result,
- metadata=metadata,
- file_name=file_name,
- )
return tool_execution_result
def copy_output_to_output_directory(self) -> None:
@@ -521,6 +504,16 @@ class DestinationConnector(BaseConnector):
return result
+ def has_valid_metadata(self, metadata: Any) -> bool:
+ # Check if metadata is not None and metadata is a non-empty string
+ if not metadata:
+ return False
+ if not isinstance(metadata, str):
+ return False
+ if metadata.strip().lower() == "none":
+ return False
+ return True
+
def get_metadata(
self, file_history: FileHistory | None = None
) -> dict[str, Any] | None:
@@ -529,8 +522,11 @@ class DestinationConnector(BaseConnector):
Returns:
Union[dict[str, Any], str]: Metadata.
"""
- if file_history and file_history.metadata:
- return self.parse_string(file_history.metadata)
+ if file_history:
+ if self.has_valid_metadata(file_history.metadata):
+ return self.parse_string(file_history.metadata)
+ else:
+ return None
metadata: dict[str, Any] = self.get_workflow_metadata()
return metadata
diff --git a/backend/workflow_manager/utils/pipeline_utils.py b/backend/workflow_manager/utils/pipeline_utils.py
new file mode 100644
index 000000000..a18f1e889
--- /dev/null
+++ b/backend/workflow_manager/utils/pipeline_utils.py
@@ -0,0 +1,65 @@
+import logging
+
+from api_v2.utils import APIDeploymentUtils
+from pipeline_v2.models import Pipeline
+from pipeline_v2.pipeline_processor import PipelineProcessor
+
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineUtils:
+ """Utility class for pipeline-workflow related operations."""
+
+ @staticmethod
+ def update_pipeline_status(
+ pipeline_id: str | None, workflow_execution: WorkflowExecution
+ ) -> None:
+ """Updates the pipeline status based on the workflow execution status.
+
+ Args:
+ pipeline_id (str | None): The ID of the pipeline to update.
+ workflow_execution (WorkflowExecution): The workflow execution object.
+ """
+ try:
+ if pipeline_id:
+ # Update pipeline status
+ if workflow_execution.status == ExecutionStatus.COMPLETED.value:
+ PipelineProcessor.update_pipeline(
+ pipeline_id,
+ Pipeline.PipelineStatus.SUCCESS,
+ execution_id=workflow_execution.id,
+ is_end=True,
+ )
+ elif workflow_execution.status in [
+ ExecutionStatus.ERROR.value,
+ ExecutionStatus.STOPPED.value,
+ ]:
+ PipelineProcessor.update_pipeline(
+ pipeline_id,
+ Pipeline.PipelineStatus.FAILURE,
+ execution_id=workflow_execution.id,
+ error_message=workflow_execution.error_message,
+ is_end=True,
+ )
+ else:
+ # Update pipeline status for other statuses.Currently this method is
+ # called only for COMPLETED and ERROR statuses.
+ logger.warning(
+ f"Workflow execution {workflow_execution.id} with status {workflow_execution.status}"
+ f"is not handled in update_pipeline_status method."
+ )
+ # Expected exception since API deployments are not tracked in Pipeline
+ except Pipeline.DoesNotExist:
+ api = APIDeploymentUtils.get_api_by_id(api_id=pipeline_id)
+ if api:
+ APIDeploymentUtils.send_notification(
+ api=api, workflow_execution=workflow_execution
+ )
+ except Exception as e:
+ logger.warning(
+ f"Error updating pipeline {pipeline_id} status: {e}, "
+ f"with workflow execution: {workflow_execution}"
+ )
diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.py
index fba0f3460..fc52d36e7 100644
--- a/backend/workflow_manager/workflow_v2/dto.py
+++ b/backend/workflow_manager/workflow_v2/dto.py
@@ -203,6 +203,13 @@ class FinalOutputResult:
metadata: dict[str, Any] | None
error: str | None
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "output": self.output,
+ "metadata": self.metadata,
+ "error": self.error,
+ }
+
@dataclass
class ExecutionContext:
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 49fc52d9b..5098c72d8 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -25,6 +25,7 @@ from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
from workflow_manager.endpoint_v2.source import SourceConnector
from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
from workflow_manager.file_execution.models import WorkflowFileExecution
+from workflow_manager.utils.pipeline_utils import PipelineUtils
from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.dto import (
ExecutionContext,
@@ -210,6 +211,7 @@ class FileExecutionTasks:
workflow = workflow_execution.workflow
organization = workflow.organization
organization_id = organization.organization_id
+ pipeline_id = str(workflow_execution.pipeline_id)
# Set organization ID in StateStore
StateStore.set(Account.ORGANIZATION_ID, organization_id)
@@ -233,6 +235,9 @@ class FileExecutionTasks:
status=final_status,
error=error_message,
)
+ PipelineUtils.update_pipeline_status(
+ pipeline_id=pipeline_id, workflow_execution=workflow_execution
+ )
# clean up execution and api storage directories
DestinationConnector.delete_execution_and_api_storage_dir(
workflow_id=workflow.id, execution_id=execution_id
@@ -558,23 +563,36 @@ class FileExecutionTasks:
execution_metadata = None
try:
+ if destination.use_file_history:
+ # Collect metadata from file history if available
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash.file_hash
+ )
+ else:
+ file_history = None
+
if not processing_error:
# Process final output through destination
output_result = destination.handle_output(
file_name=file_hash.file_name,
file_hash=file_hash,
+ file_history=file_history,
workflow=workflow,
input_file_path=file_hash.file_path,
file_execution_id=file_execution_id,
)
- # Collect metadata from file history if available
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash.file_hash
- )
- if file_history and destination.is_api:
+ if destination.is_api:
execution_metadata = destination.get_metadata(file_history)
-
+ if destination.use_file_history and not file_history:
+ FileHistoryHelper.create_file_history(
+ file_hash=file_hash,
+ workflow=workflow,
+ status=ExecutionStatus.COMPLETED,
+ result=output_result,
+ metadata=execution_metadata,
+ file_name=file_hash.file_name,
+ )
except Exception as e:
error_msg = f"Final output processing failed: {str(e)}"
logger.error(error_msg, exc_info=True)
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index e7395447e..00ffa030b 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -46,7 +46,7 @@ class FileHistoryHelper:
workflow: Workflow,
status: ExecutionStatus,
result: Any,
- metadata: Any,
+ metadata: str | None,
error: str | None = None,
file_name: str | None = None,
) -> None:
@@ -65,7 +65,7 @@ class FileHistoryHelper:
provider_file_uuid=file_hash.provider_file_uuid,
status=status,
result=str(result),
- metadata=str(metadata),
+ metadata=str(metadata) if metadata else "",
error=str(error) if error else "",
)
except IntegrityError as e:
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 97a49682f..30a2f1cdf 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -8,14 +8,12 @@ from typing import Any
from account_v2.constants import Common
from api_v2.models import APIDeployment
-from api_v2.utils import APIDeploymentUtils
from celery import chord, current_task
from celery import exceptions as celery_exceptions
from celery.result import AsyncResult
from django.conf import settings
from django.db import IntegrityError
from pipeline_v2.models import Pipeline
-from pipeline_v2.pipeline_processor import PipelineProcessor
from rest_framework import serializers
from tool_instance_v2.constants import ToolInstanceKey
from tool_instance_v2.models import ToolInstance
@@ -35,6 +33,7 @@ from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
from workflow_manager.endpoint_v2.source import SourceConnector
from workflow_manager.execution.dto import ExecutionCache
from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
+from workflow_manager.utils.pipeline_utils import PipelineUtils
from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.constants import (
WorkflowErrors,
@@ -282,9 +281,6 @@ class WorkflowHelper:
use_file_history=use_file_history,
execution_mode=execution_mode,
)
- WorkflowHelper._update_pipeline_status(
- pipeline_id=pipeline_id, workflow_execution=workflow_execution
- )
api_results = []
return ExecutionResponse(
str(workflow.id),
@@ -302,46 +298,11 @@ class WorkflowHelper:
status=ExecutionStatus.ERROR,
error=str(e),
)
- WorkflowHelper._update_pipeline_status(
+ PipelineUtils.update_pipeline_status(
pipeline_id=pipeline_id, workflow_execution=workflow_execution
)
raise
- @staticmethod
- def _update_pipeline_status(
- pipeline_id: str | None, workflow_execution: WorkflowExecution
- ) -> None:
- try:
- if pipeline_id:
- # Update pipeline status
- if workflow_execution.status != ExecutionStatus.ERROR.value:
- PipelineProcessor.update_pipeline(
- pipeline_id,
- Pipeline.PipelineStatus.SUCCESS,
- execution_id=workflow_execution.id,
- is_end=True,
- )
- else:
- PipelineProcessor.update_pipeline(
- pipeline_id,
- Pipeline.PipelineStatus.FAILURE,
- execution_id=workflow_execution.id,
- error_message=workflow_execution.error_message,
- is_end=True,
- )
- # Expected exception since API deployments are not tracked in Pipeline
- except Pipeline.DoesNotExist:
- api = APIDeploymentUtils.get_api_by_id(api_id=pipeline_id)
- if api:
- APIDeploymentUtils.send_notification(
- api=api, workflow_execution=workflow_execution
- )
- except Exception as e:
- logger.warning(
- f"Error updating pipeline {pipeline_id} status: {e}, "
- f"with workflow execution: {workflow_execution}"
- )
-
@classmethod
def get_status_of_async_task(
cls,