LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix for null metadata in api-execution (#1304)
aliea331a5e73e7
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex a1795945f..4ce3ab773 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -27,8 +27,6 @@ from workflow_manager.endpoint_v2.exceptions import (from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtilsfrom workflow_manager.utils.workflow_log import WorkflowLog-from workflow_manager.workflow_v2.enums import ExecutionStatus-from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -199,6 +197,7 @@ class DestinationConnector(BaseConnector):self,file_name: str,file_hash: FileHash,+ file_history: FileHistory | None,workflow: Workflow,input_file_path: str,file_execution_id: str = None,@@ -206,13 +205,7 @@ class DestinationConnector(BaseConnector):"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_typetool_execution_result: str | None = None- metadata: str | None = None- file_history = None- if self.use_file_history:- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash.file_hash- )if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:@@ -236,16 +229,6 @@ class DestinationConnector(BaseConnector):self.workflow_log.publish_log(message=f"File '{file_name}' processed successfully")-- if self.use_file_history and not file_history:- FileHistoryHelper.create_file_history(- file_hash=file_hash,- workflow=workflow,- status=ExecutionStatus.COMPLETED,- result=tool_execution_result,- metadata=metadata,- file_name=file_name,- )return tool_execution_resultdef copy_output_to_output_directory(self) -> None:@@ -521,6 +504,16 @@ class DestinationConnector(BaseConnector):return result+ def has_valid_metadata(self, metadata: Any) -> bool:+ # Check if metadata is not None and metadata is a non-empty string+ if not metadata:+ return False+ if not isinstance(metadata, str):+ return False+ if metadata.strip().lower() == "none":+ return False+ return True+def get_metadata(self, file_history: FileHistory | None = None) -> dict[str, Any] | None:@@ -529,8 +522,11 @@ class DestinationConnector(BaseConnector):Returns:Union[dict[str, Any], str]: Metadata."""- if file_history and file_history.metadata:- return self.parse_string(file_history.metadata)+ if file_history:+ if self.has_valid_metadata(file_history.metadata):+ return self.parse_string(file_history.metadata)+ else:+ return Nonemetadata: dict[str, Any] = self.get_workflow_metadata()return metadatadiff --git a/backend/workflow_manager/utils/pipeline_utils.py b/backend/workflow_manager/utils/pipeline_utils.pynew file mode 100644index 000000000..a18f1e889--- /dev/null+++ b/backend/workflow_manager/utils/pipeline_utils.py@@ -0,0 +1,65 @@+import logging++from api_v2.utils import APIDeploymentUtils+from pipeline_v2.models import Pipeline+from pipeline_v2.pipeline_processor import PipelineProcessor++from workflow_manager.workflow_v2.enums import ExecutionStatus+from workflow_manager.workflow_v2.models.execution import WorkflowExecution++logger = logging.getLogger(__name__)+++class PipelineUtils:+ """Utility class for pipeline-workflow related operations."""++ @staticmethod+ def update_pipeline_status(+ pipeline_id: str | None, workflow_execution: WorkflowExecution+ ) -> None:+ """Updates the pipeline status based on the workflow execution status.++ Args:+ pipeline_id (str | None): The ID of the pipeline to update.+ workflow_execution (WorkflowExecution): The workflow execution object.+ """+ try:+ if pipeline_id:+ # Update pipeline status+ if workflow_execution.status == ExecutionStatus.COMPLETED.value:+ PipelineProcessor.update_pipeline(+ pipeline_id,+ Pipeline.PipelineStatus.SUCCESS,+ execution_id=workflow_execution.id,+ is_end=True,+ )+ elif workflow_execution.status in [+ ExecutionStatus.ERROR.value,+ ExecutionStatus.STOPPED.value,+ ]:+ PipelineProcessor.update_pipeline(+ pipeline_id,+ Pipeline.PipelineStatus.FAILURE,+ execution_id=workflow_execution.id,+ error_message=workflow_execution.error_message,+ is_end=True,+ )+ else:+ # Update pipeline status for other statuses.Currently this method is+ # called only for COMPLETED and ERROR statuses.+ logger.warning(+ f"Workflow execution {workflow_execution.id} with status {workflow_execution.status}"+ f"is not handled in update_pipeline_status method."+ )+ # Expected exception since API deployments are not tracked in Pipeline+ except Pipeline.DoesNotExist:+ api = APIDeploymentUtils.get_api_by_id(api_id=pipeline_id)+ if api:+ APIDeploymentUtils.send_notification(+ api=api, workflow_execution=workflow_execution+ )+ except Exception as e:+ logger.warning(+ f"Error updating pipeline {pipeline_id} status: {e}, "+ f"with workflow execution: {workflow_execution}"+ )diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.pyindex fba0f3460..fc52d36e7 100644--- a/backend/workflow_manager/workflow_v2/dto.py+++ b/backend/workflow_manager/workflow_v2/dto.py@@ -203,6 +203,13 @@ class FinalOutputResult:metadata: dict[str, Any] | Noneerror: str | None+ def to_dict(self) -> dict[str, Any]:+ return {+ "output": self.output,+ "metadata": self.metadata,+ "error": self.error,+ }+@dataclassclass ExecutionContext:diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 49fc52d9b..5098c72d8 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -25,6 +25,7 @@ from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtilsfrom workflow_manager.endpoint_v2.source import SourceConnectorfrom workflow_manager.execution.execution_cache_utils import ExecutionCacheUtilsfrom workflow_manager.file_execution.models import WorkflowFileExecution+from workflow_manager.utils.pipeline_utils import PipelineUtilsfrom workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.dto import (ExecutionContext,@@ -210,6 +211,7 @@ class FileExecutionTasks:workflow = workflow_execution.workfloworganization = workflow.organizationorganization_id = organization.organization_id+ pipeline_id = str(workflow_execution.pipeline_id)# Set organization ID in StateStoreStateStore.set(Account.ORGANIZATION_ID, organization_id)@@ -233,6 +235,9 @@ class FileExecutionTasks:status=final_status,error=error_message,)+ PipelineUtils.update_pipeline_status(+ pipeline_id=pipeline_id, workflow_execution=workflow_execution+ )# clean up execution and api storage directoriesDestinationConnector.delete_execution_and_api_storage_dir(workflow_id=workflow.id, execution_id=execution_id@@ -558,23 +563,36 @@ class FileExecutionTasks:execution_metadata = Nonetry:+ if destination.use_file_history:+ # Collect metadata from file history if available+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash.file_hash+ )+ else:+ file_history = None+if not processing_error:# Process final output through destinationoutput_result = destination.handle_output(file_name=file_hash.file_name,file_hash=file_hash,+ file_history=file_history,workflow=workflow,input_file_path=file_hash.file_path,file_execution_id=file_execution_id,)- # Collect metadata from file history if available- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash.file_hash- )- if file_history and destination.is_api:+ if destination.is_api:execution_metadata = destination.get_metadata(file_history)-+ if destination.use_file_history and not file_history:+ FileHistoryHelper.create_file_history(+ file_hash=file_hash,+ workflow=workflow,+ status=ExecutionStatus.COMPLETED,+ result=output_result,+ metadata=execution_metadata,+ file_name=file_hash.file_name,+ )except Exception as e:error_msg = f"Final output processing failed: {str(e)}"logger.error(error_msg, exc_info=True)diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex e7395447e..00ffa030b 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -46,7 +46,7 @@ class FileHistoryHelper:workflow: Workflow,status: ExecutionStatus,result: Any,- metadata: Any,+ metadata: str | None,error: str | None = None,file_name: str | None = None,) -> None:@@ -65,7 +65,7 @@ class FileHistoryHelper:provider_file_uuid=file_hash.provider_file_uuid,status=status,result=str(result),- metadata=str(metadata),+ metadata=str(metadata) if metadata else "",error=str(error) if error else "",)except IntegrityError as e:diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 97a49682f..30a2f1cdf 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -8,14 +8,12 @@ from typing import Anyfrom account_v2.constants import Commonfrom api_v2.models import APIDeployment-from api_v2.utils import APIDeploymentUtilsfrom celery import chord, current_taskfrom celery import exceptions as celery_exceptionsfrom celery.result import AsyncResultfrom django.conf import settingsfrom django.db import IntegrityErrorfrom pipeline_v2.models import Pipeline-from pipeline_v2.pipeline_processor import PipelineProcessorfrom rest_framework import serializersfrom tool_instance_v2.constants import ToolInstanceKeyfrom tool_instance_v2.models import ToolInstance@@ -35,6 +33,7 @@ from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtilsfrom workflow_manager.endpoint_v2.source import SourceConnectorfrom workflow_manager.execution.dto import ExecutionCachefrom workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils+from workflow_manager.utils.pipeline_utils import PipelineUtilsfrom workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.constants import (WorkflowErrors,@@ -282,9 +281,6 @@ class WorkflowHelper:use_file_history=use_file_history,execution_mode=execution_mode,)- WorkflowHelper._update_pipeline_status(- pipeline_id=pipeline_id, workflow_execution=workflow_execution- )api_results = []return ExecutionResponse(str(workflow.id),@@ -302,46 +298,11 @@ class WorkflowHelper:status=ExecutionStatus.ERROR,error=str(e),)- WorkflowHelper._update_pipeline_status(+ PipelineUtils.update_pipeline_status(pipeline_id=pipeline_id, workflow_execution=workflow_execution)raise- @staticmethod- def _update_pipeline_status(- pipeline_id: str | None, workflow_execution: WorkflowExecution- ) -> None:- try:- if pipeline_id:- # Update pipeline status- if workflow_execution.status != ExecutionStatus.ERROR.value:- PipelineProcessor.update_pipeline(- pipeline_id,- Pipeline.PipelineStatus.SUCCESS,- execution_id=workflow_execution.id,- is_end=True,- )- else:- PipelineProcessor.update_pipeline(- pipeline_id,- Pipeline.PipelineStatus.FAILURE,- execution_id=workflow_execution.id,- error_message=workflow_execution.error_message,- is_end=True,- )- # Expected exception since API deployments are not tracked in Pipeline- except Pipeline.DoesNotExist:- api = APIDeploymentUtils.get_api_by_id(api_id=pipeline_id)- if api:- APIDeploymentUtils.send_notification(- api=api, workflow_execution=workflow_execution- )- except Exception as e:- logger.warning(- f"Error updating pipeline {pipeline_id} status: {e}, "- f"with workflow execution: {workflow_execution}"- )-@classmethoddef get_status_of_async_task(cls,