Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix for workflow execution long polling from workflow page (#1313)

alif64ce6f8dfd4
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 41edec301..dbe0b0945 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -133,6 +133,9 @@ EXECUTION_RESULT_TTL_SECONDS = int(
EXECUTION_CACHE_TTL_SECONDS = int(
os.environ.get("EXECUTION_CACHE_TTL_SECONDS", 10800)
) # 3 hours
+INSTANT_WF_POLLING_TIMEOUT = int(
+ os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")
+) # 5 minutes
MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))
INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
diff --git a/backend/sample.env b/backend/sample.env
index e7463eb2f..a6f658a9e 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -174,6 +174,8 @@ ENABLE_HIGHLIGHT_API_DEPLOYMENT=False
EXECUTION_RESULT_TTL_SECONDS=86400 # 24 hours
# For execution metadata cached per workflow execution
EXECUTION_CACHE_TTL_SECONDS=86400 # 24 hours
+# Instant workflow polling timeout in seconds
+INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes
# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution
MAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 1261424c4..6098af0b3 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -1,6 +1,7 @@
import logging
from typing import Any
+from django.conf import settings
from django.db.models.query import QuerySet
from permissions.permission import IsOwner
from pipeline_v2.models import Pipeline
@@ -227,6 +228,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_mode=WorkflowExecution.Mode.INSTANT,
hash_values_of_files=hash_values_of_files,
use_file_history=use_file_history,
+ timeout=settings.INSTANT_WF_POLLING_TIMEOUT,
)
return execution_response
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index c79544cf8..c6371d51a 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -652,6 +652,7 @@ class WorkflowHelper:
execution_mode: WorkflowExecution | None = WorkflowExecution.Mode.QUEUE,
hash_values_of_files: dict[str, FileHash] = {},
use_file_history: bool = False,
+ timeout: int | None = None,
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
@@ -711,7 +712,7 @@ class WorkflowHelper:
workflow_id=workflow.id, pipeline_id=pipeline_id
)
try:
- # Normal execution
+ # Normal Workflow page execution
workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
if (
workflow_execution.status != ExecutionStatus.PENDING
@@ -719,18 +720,53 @@ class WorkflowHelper:
):
raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
organization_identifier = UserContext.get_organization_identifier()
- return WorkflowHelper.run_workflow(
+ result: ExecutionResponse = WorkflowHelper.run_workflow(
workflow=workflow,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
use_file_history=use_file_history,
organization_id=organization_identifier,
)
+ result = WorkflowHelper.wait_for_execution(result, timeout=timeout)
+ return result
+
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
workflow_id=workflow.id, pipeline_id=pipeline_id
)
+ @classmethod
+ def wait_for_execution(
+ cls, result: ExecutionResponse, timeout: int | None = None
+ ) -> ExecutionResponse:
+ """Wait for the execution to complete.
+
+ Args:
+ result (ExecutionResponse): The execution response.
+ timeout (int | None, optional): The timeout in seconds. Defaults to None.
+
+ Returns:
+ ExecutionResponse: The execution response.
+ """
+ if (
+ result.execution_status in [ExecutionStatus.COMPLETED, ExecutionStatus.ERROR]
+ or not timeout
+ ):
+ return result
+ execution_status = result.execution_status
+ workflow_id = result.workflow_id
+ execution_id = result.execution_id
+ if timeout > 0:
+ while not ExecutionStatus.is_completed(execution_status) and timeout > 0:
+ time.sleep(2)
+ timeout -= 2
+
+ execution_status = cls._get_execution_status(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ result.execution_status = execution_status
+ return result
+
@staticmethod
def get_current_execution(execution_id: str) -> ExecutionResponse:
try: