Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2948 [FIX] Support API workflow testing from workflow page with proper queue routing (#1640)

ali390c7fadfee1
diff --git a/backend/workflow_manager/endpoint_v2/endpoint_utils.py b/backend/workflow_manager/endpoint_v2/endpoint_utils.py
index 4eeeee7ca..33b06f7b6 100644
--- a/backend/workflow_manager/endpoint_v2/endpoint_utils.py
+++ b/backend/workflow_manager/endpoint_v2/endpoint_utils.py
@@ -52,3 +52,33 @@ class WorkflowEndpointUtils:
endpoint.connector_instance.metadata
)
return endpoint
+
+ @staticmethod
+ def is_api_workflow(workflow: Workflow) -> bool:
+ """Check if workflow uses API connectors for both source and destination.
+
+ Args:
+ workflow (Workflow): The workflow to check
+
+ Returns:
+ bool: True if both source and destination are API type, False otherwise
+ """
+ try:
+ source_endpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+ destination_endpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
+ )
+ source_is_api = (
+ source_endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
+ )
+ destination_is_api = (
+ destination_endpoint.connection_type
+ == WorkflowEndpoint.ConnectionType.API
+ )
+ return source_is_api and destination_is_api
+ except WorkflowEndpoint.DoesNotExist:
+ return False
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 7bf448f04..aa4f869a9 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -279,6 +279,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):
hash_values_of_files: dict[str, FileHash] = {},
use_file_history: bool = False,
) -> ExecutionResponse:
+ # Detect if this is an API execution by checking connector types
+ is_api_execution = WorkflowEndpointUtils.is_api_workflow(workflow)
+
if execution_action is not None:
# Step execution
execution_response = WorkflowHelper.step_execution(
@@ -299,6 +302,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_mode=WorkflowExecution.Mode.INSTANT,
hash_values_of_files=hash_values_of_files,
use_file_history=use_file_history,
+ is_api_execution=is_api_execution,
)
else:
execution_response = WorkflowHelper.complete_execution(
@@ -308,6 +312,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
hash_values_of_files=hash_values_of_files,
use_file_history=use_file_history,
timeout=settings.INSTANT_WF_POLLING_TIMEOUT,
+ is_api_execution=is_api_execution,
)
return execution_response
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 99a4f55fe..73aef3e9d 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -20,7 +20,7 @@ from tool_instance_v2.constants import ToolInstanceKey
from tool_instance_v2.models import ToolInstance
from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
from utils.cache_service import CacheService
-from utils.constants import Account
+from utils.constants import Account, CeleryQueue
from utils.local_context import StateStore
from utils.user_context import UserContext
@@ -739,6 +739,7 @@ class WorkflowHelper:
hash_values_of_files: dict[str, FileHash] = {},
use_file_history: bool = False,
timeout: int | None = None,
+ is_api_execution: bool = False,
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
@@ -756,12 +757,15 @@ class WorkflowHelper:
org_schema = UserContext.get_organization_identifier()
if execution_mode == WorkflowExecution.Mode.INSTANT:
# Instant request from UX (Sync now in ETL and Workflow page)
+ # Route to API deployment queue if this is an API execution
+ queue = CeleryQueue.CELERY_API_DEPLOYMENTS if is_api_execution else None
response: ExecutionResponse = WorkflowHelper.execute_workflow_async(
workflow_id=workflow.id,
pipeline_id=pipeline_id,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
use_file_history=use_file_history,
+ queue=queue,
)
return response
else:
@@ -805,12 +809,15 @@ class WorkflowHelper:
or workflow_execution.execution_type != WorkflowExecution.Type.COMPLETE
):
raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
+ # Route to API deployment queue if this is an API execution
+ queue = CeleryQueue.CELERY_API_DEPLOYMENTS if is_api_execution else None
result: ExecutionResponse = WorkflowHelper.execute_workflow_async(
workflow_id=str(workflow.id) if workflow else None,
pipeline_id=str(pipeline_id) if pipeline_id else None,
execution_id=str(execution_id) if execution_id else None,
hash_values_of_files=hash_values_of_files,
use_file_history=use_file_history,
+ queue=queue,
)
result = WorkflowHelper.wait_for_execution(result, timeout=timeout)
return result
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py
index cccfb78c3..1e83559bb 100644
--- a/workers/callback/tasks.py
+++ b/workers/callback/tasks.py
@@ -1615,23 +1615,27 @@ def process_batch_callback_api(
# Get pipeline name and type (simplified approach)
if not pipeline_id:
- error_msg = f"No pipeline_id provided for API callback. execution_id={execution_id}, workflow_id={workflow_id}"
- logger.error(error_msg)
- raise ValueError(error_msg)
-
- # Use simplified pipeline data fetching
- pipeline_name, pipeline_type = _fetch_pipeline_data_simplified(
- pipeline_id, schema_name, api_client, is_api_deployment=True
- )
-
- if pipeline_name:
logger.info(
- f"✅ Found pipeline: name='{pipeline_name}', type='{pipeline_type}'"
+ f"No pipeline_id provided for API callback (testing mode). "
+ f"execution_id={execution_id}, workflow_id={workflow_id}. "
+ f"APIDeployment-specific updates and notifications will be skipped."
)
+ pipeline_name = None
+ pipeline_type = None
else:
- logger.warning(f"Could not fetch pipeline data for {pipeline_id}")
- pipeline_name = "Unknown API"
- pipeline_type = PipelineType.API.value
+ # Use simplified pipeline data fetching
+ pipeline_name, pipeline_type = _fetch_pipeline_data_simplified(
+ pipeline_id, schema_name, api_client, is_api_deployment=True
+ )
+
+ if pipeline_name:
+ logger.info(
+ f"✅ Found pipeline: name='{pipeline_name}', type='{pipeline_type}'"
+ )
+ else:
+ logger.warning(f"Could not fetch pipeline data for {pipeline_id}")
+ pipeline_name = "Unknown API"
+ pipeline_type = PipelineType.API.value
# Use unified status determination with timeout detection
aggregated_results, execution_status, expected_files = (
@@ -1676,10 +1680,20 @@ def process_batch_callback_api(
# Handle resource cleanup (matching ETL workflow behavior)
cleanup_result = _cleanup_execution_resources(context)
- # Handle pipeline updates (skip for API deployments)
- pipeline_result = _handle_pipeline_updates_unified(
- context=context, final_status=execution_status, is_api_deployment=True
- )
+ # Handle pipeline updates (only if pipeline_id exists)
+ if context.pipeline_id:
+ pipeline_result = _handle_pipeline_updates_unified(
+ context=context, final_status=execution_status, is_api_deployment=True
+ )
+ else:
+ logger.info(
+ f"Skipping pipeline status update for execution {execution_id} "
+ f"since no APIDeployment record"
+ )
+ pipeline_result = {
+ "skipped": True,
+ "reason": "execute without APIDeployment record",
+ }
# Track subscription usage if plugin is present
subscription_tracking_result = _track_subscription_usage_if_available(