LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2948 [FIX] Support API workflow testing from workflow page with proper queue routing (#1640)
ali390c7fadfee1
diff --git a/backend/workflow_manager/endpoint_v2/endpoint_utils.py b/backend/workflow_manager/endpoint_v2/endpoint_utils.pyindex 4eeeee7ca..33b06f7b6 100644--- a/backend/workflow_manager/endpoint_v2/endpoint_utils.py+++ b/backend/workflow_manager/endpoint_v2/endpoint_utils.py@@ -52,3 +52,33 @@ class WorkflowEndpointUtils:endpoint.connector_instance.metadata)return endpoint++ @staticmethod+ def is_api_workflow(workflow: Workflow) -> bool:+ """Check if workflow uses API connectors for both source and destination.++ Args:+ workflow (Workflow): The workflow to check++ Returns:+ bool: True if both source and destination are API type, False otherwise+ """+ try:+ source_endpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ )+ destination_endpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,+ )+ source_is_api = (+ source_endpoint.connection_type == WorkflowEndpoint.ConnectionType.API+ )+ destination_is_api = (+ destination_endpoint.connection_type+ == WorkflowEndpoint.ConnectionType.API+ )+ return source_is_api and destination_is_api+ except WorkflowEndpoint.DoesNotExist:+ return Falsediff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 7bf448f04..aa4f869a9 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -279,6 +279,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):hash_values_of_files: dict[str, FileHash] = {},use_file_history: bool = False,) -> ExecutionResponse:+ # Detect if this is an API execution by checking connector types+ is_api_execution = WorkflowEndpointUtils.is_api_workflow(workflow)+if execution_action is not None:# Step executionexecution_response = WorkflowHelper.step_execution(@@ -299,6 +302,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,use_file_history=use_file_history,+ is_api_execution=is_api_execution,)else:execution_response = WorkflowHelper.complete_execution(@@ -308,6 +312,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):hash_values_of_files=hash_values_of_files,use_file_history=use_file_history,timeout=settings.INSTANT_WF_POLLING_TIMEOUT,+ is_api_execution=is_api_execution,)return execution_responsediff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 99a4f55fe..73aef3e9d 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -20,7 +20,7 @@ from tool_instance_v2.constants import ToolInstanceKeyfrom tool_instance_v2.models import ToolInstancefrom tool_instance_v2.tool_instance_helper import ToolInstanceHelperfrom utils.cache_service import CacheService-from utils.constants import Account+from utils.constants import Account, CeleryQueuefrom utils.local_context import StateStorefrom utils.user_context import UserContext@@ -739,6 +739,7 @@ class WorkflowHelper:hash_values_of_files: dict[str, FileHash] = {},use_file_history: bool = False,timeout: int | None = None,+ is_api_execution: bool = False,) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")@@ -756,12 +757,15 @@ class WorkflowHelper:org_schema = UserContext.get_organization_identifier()if execution_mode == WorkflowExecution.Mode.INSTANT:# Instant request from UX (Sync now in ETL and Workflow page)+ # Route to API deployment queue if this is an API execution+ queue = CeleryQueue.CELERY_API_DEPLOYMENTS if is_api_execution else Noneresponse: ExecutionResponse = WorkflowHelper.execute_workflow_async(workflow_id=workflow.id,pipeline_id=pipeline_id,execution_id=execution_id,hash_values_of_files=hash_values_of_files,use_file_history=use_file_history,+ queue=queue,)return responseelse:@@ -805,12 +809,15 @@ class WorkflowHelper:or workflow_execution.execution_type != WorkflowExecution.Type.COMPLETE):raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)+ # Route to API deployment queue if this is an API execution+ queue = CeleryQueue.CELERY_API_DEPLOYMENTS if is_api_execution else Noneresult: ExecutionResponse = WorkflowHelper.execute_workflow_async(workflow_id=str(workflow.id) if workflow else None,pipeline_id=str(pipeline_id) if pipeline_id else None,execution_id=str(execution_id) if execution_id else None,hash_values_of_files=hash_values_of_files,use_file_history=use_file_history,+ queue=queue,)result = WorkflowHelper.wait_for_execution(result, timeout=timeout)return resultdiff --git a/workers/callback/tasks.py b/workers/callback/tasks.pyindex cccfb78c3..1e83559bb 100644--- a/workers/callback/tasks.py+++ b/workers/callback/tasks.py@@ -1615,23 +1615,27 @@ def process_batch_callback_api(# Get pipeline name and type (simplified approach)if not pipeline_id:- error_msg = f"No pipeline_id provided for API callback. execution_id={execution_id}, workflow_id={workflow_id}"- logger.error(error_msg)- raise ValueError(error_msg)-- # Use simplified pipeline data fetching- pipeline_name, pipeline_type = _fetch_pipeline_data_simplified(- pipeline_id, schema_name, api_client, is_api_deployment=True- )-- if pipeline_name:logger.info(- f"✅ Found pipeline: name='{pipeline_name}', type='{pipeline_type}'"+ f"No pipeline_id provided for API callback (testing mode). "+ f"execution_id={execution_id}, workflow_id={workflow_id}. "+ f"APIDeployment-specific updates and notifications will be skipped.")+ pipeline_name = None+ pipeline_type = Noneelse:- logger.warning(f"Could not fetch pipeline data for {pipeline_id}")- pipeline_name = "Unknown API"- pipeline_type = PipelineType.API.value+ # Use simplified pipeline data fetching+ pipeline_name, pipeline_type = _fetch_pipeline_data_simplified(+ pipeline_id, schema_name, api_client, is_api_deployment=True+ )++ if pipeline_name:+ logger.info(+ f"✅ Found pipeline: name='{pipeline_name}', type='{pipeline_type}'"+ )+ else:+ logger.warning(f"Could not fetch pipeline data for {pipeline_id}")+ pipeline_name = "Unknown API"+ pipeline_type = PipelineType.API.value# Use unified status determination with timeout detectionaggregated_results, execution_status, expected_files = (@@ -1676,10 +1680,20 @@ def process_batch_callback_api(# Handle resource cleanup (matching ETL workflow behavior)cleanup_result = _cleanup_execution_resources(context)- # Handle pipeline updates (skip for API deployments)- pipeline_result = _handle_pipeline_updates_unified(- context=context, final_status=execution_status, is_api_deployment=True- )+ # Handle pipeline updates (only if pipeline_id exists)+ if context.pipeline_id:+ pipeline_result = _handle_pipeline_updates_unified(+ context=context, final_status=execution_status, is_api_deployment=True+ )+ else:+ logger.info(+ f"Skipping pipeline status update for execution {execution_id} "+ f"since no APIDeployment record"+ )+ pipeline_result = {+ "skipped": True,+ "reason": "execute without APIDeployment record",+ }# Track subscription usage if plugin is presentsubscription_tracking_result = _track_subscription_usage_if_available(