LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix for workflow execution long polling from workflow page (#1313)
alif64ce6f8dfd4
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 41edec301..dbe0b0945 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -133,6 +133,9 @@ EXECUTION_RESULT_TTL_SECONDS = int(EXECUTION_CACHE_TTL_SECONDS = int(os.environ.get("EXECUTION_CACHE_TTL_SECONDS", 10800)) # 3 hours+INSTANT_WF_POLLING_TIMEOUT = int(+ os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")+) # 5 minutesMAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))diff --git a/backend/sample.env b/backend/sample.envindex e7463eb2f..a6f658a9e 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -174,6 +174,8 @@ ENABLE_HIGHLIGHT_API_DEPLOYMENT=FalseEXECUTION_RESULT_TTL_SECONDS=86400 # 24 hours# For execution metadata cached per workflow executionEXECUTION_CACHE_TTL_SECONDS=86400 # 24 hours+# Instant workflow polling timeout in seconds+INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes# Maximum number of batches (i.e., parallel tasks) created for a single workflow executionMAX_PARALLEL_FILE_BATCHES=1 # 1 file at a timediff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 1261424c4..6098af0b3 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -1,6 +1,7 @@import loggingfrom typing import Any+from django.conf import settingsfrom django.db.models.query import QuerySetfrom permissions.permission import IsOwnerfrom pipeline_v2.models import Pipeline@@ -227,6 +228,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,use_file_history=use_file_history,+ timeout=settings.INSTANT_WF_POLLING_TIMEOUT,)return execution_responsediff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex c79544cf8..c6371d51a 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -652,6 +652,7 @@ class WorkflowHelper:execution_mode: WorkflowExecution | None = WorkflowExecution.Mode.QUEUE,hash_values_of_files: dict[str, FileHash] = {},use_file_history: bool = False,+ timeout: int | None = None,) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")@@ -711,7 +712,7 @@ class WorkflowHelper:workflow_id=workflow.id, pipeline_id=pipeline_id)try:- # Normal execution+ # Normal Workflow page executionworkflow_execution = WorkflowExecution.objects.get(pk=execution_id)if (workflow_execution.status != ExecutionStatus.PENDING@@ -719,18 +720,53 @@ class WorkflowHelper:):raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)organization_identifier = UserContext.get_organization_identifier()- return WorkflowHelper.run_workflow(+ result: ExecutionResponse = WorkflowHelper.run_workflow(workflow=workflow,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,use_file_history=use_file_history,organization_id=organization_identifier,)+ result = WorkflowHelper.wait_for_execution(result, timeout=timeout)+ return result+except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(workflow_id=workflow.id, pipeline_id=pipeline_id)+ @classmethod+ def wait_for_execution(+ cls, result: ExecutionResponse, timeout: int | None = None+ ) -> ExecutionResponse:+ """Wait for the execution to complete.++ Args:+ result (ExecutionResponse): The execution response.+ timeout (int | None, optional): The timeout in seconds. Defaults to None.++ Returns:+ ExecutionResponse: The execution response.+ """+ if (+ result.execution_status in [ExecutionStatus.COMPLETED, ExecutionStatus.ERROR]+ or not timeout+ ):+ return result+ execution_status = result.execution_status+ workflow_id = result.workflow_id+ execution_id = result.execution_id+ if timeout > 0:+ while not ExecutionStatus.is_completed(execution_status) and timeout > 0:+ time.sleep(2)+ timeout -= 2++ execution_status = cls._get_execution_status(+ workflow_id=workflow_id, execution_id=execution_id+ )+ result.execution_status = execution_status+ return result+@staticmethoddef get_current_execution(execution_id: str) -> ExecutionResponse:try: