LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2867 [FIX] Fixed webhook notifications showing incorrect pipeline name for SUCCESS/COMPLETED events (#1577)
ali2aebf04905ee
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.pyindex 088cdc6f8..8dc66aa53 100644--- a/workers/callback/tasks.py+++ b/workers/callback/tasks.py@@ -620,6 +620,51 @@ def _handle_notifications_unified(}+def _fetch_pipeline_name_from_api(+ pipeline_id: str,+ api_client: InternalAPIClient,+) -> str | None:+ """Fetch actual pipeline name from Pipeline/APIDeployment models.++ Args:+ pipeline_id: Pipeline or API deployment ID+ api_client: API client instance++ Returns:+ Pipeline name from API, or None if fetch failed+ """+ try:+ from shared.models.pipeline_models import PipelineApiResponse++ logger.info(+ f"Fetching pipeline data for pipeline_id={pipeline_id} to get correct pipeline name"+ )+ pipeline_response = api_client.get_pipeline_data(+ pipeline_id=pipeline_id, check_active=False+ )++ if pipeline_response.success:+ # Parse response using type-safe dataclass+ pipeline_api_data = PipelineApiResponse.from_dict(pipeline_response.data)+ pipeline_name = pipeline_api_data.pipeline.pipeline_name++ logger.info(+ f"Fetched pipeline name from API: '{pipeline_name}' for pipeline_id={pipeline_id}"+ )+ return pipeline_name+ else:+ logger.warning(+ f"Could not fetch pipeline data for {pipeline_id}: {pipeline_response.error}"+ )+ return None+ except Exception:+ logger.exception(+ f"Error fetching pipeline name for {pipeline_id}."+ f"Will use 'Unknown API' or 'Unknown Pipeline' in notifications."+ )+ return None++def _extract_callback_parameters(task_instance, results: list, kwargs: dict[str, Any]) -> CallbackContext:@@ -748,37 +793,40 @@ def _extract_callback_parameters(# Use existing API detection from source_config (no additional API calls needed)is_api_deployment = source_config.get("is_api", False)+ # Fetch actual pipeline name from Pipeline/APIDeployment models+ pipeline_name_from_api = None+ if context.pipeline_id:+ pipeline_name_from_api = _fetch_pipeline_name_from_api(+ context.pipeline_id, api_client+ )+if is_api_deployment:# This is an API deployment+ # Use fetched pipeline name if available, otherwise use "Unknown API"+ resolved_pipeline_name = pipeline_name_from_api or "Unknown API"context.pipeline_data = {"is_api": True,"resolved_pipeline_type": "API",- "resolved_pipeline_name": workflow_definition.get(- "workflow_name", "Unknown API"- ),+ "resolved_pipeline_name": resolved_pipeline_name,}context.pipeline_type = "API"- context.pipeline_name = workflow_definition.get(- "workflow_name", "Unknown API"- )+ context.pipeline_name = resolved_pipeline_namelogger.info(- f"Detected API deployment from source_config: {context.pipeline_id}"+ f"Detected API deployment from source_config: {context.pipeline_id}, pipeline_name='{resolved_pipeline_name}'")else:# This is ETL/TASK/APP workflow+ # Use fetched pipeline name if available, otherwise use "Unknown Pipeline"+ resolved_pipeline_name = pipeline_name_from_api or "Unknown Pipeline"context.pipeline_data = {"is_api": False,"resolved_pipeline_type": "ETL",- "resolved_pipeline_name": workflow_definition.get(- "workflow_name", "Unknown Workflow"- ),+ "resolved_pipeline_name": resolved_pipeline_name,}context.pipeline_type = "ETL"- context.pipeline_name = workflow_definition.get(- "workflow_name", "Unknown Workflow"- )+ context.pipeline_name = resolved_pipeline_namelogger.info(- f"Detected ETL workflow from source_config: {context.pipeline_id}"+ f"Detected ETL workflow from source_config: {context.pipeline_id}, pipeline_name='{resolved_pipeline_name}'")logger.info(@@ -789,7 +837,7 @@ def _extract_callback_parameters()except Exception as e:- logger.error(f"Failed to fetch workflow execution context: {e}")+ logger.exception("Failed to fetch workflow execution context")raise ValueError(f"Could not get execution context: {e}")# 4. Validate required context is now available@@ -943,7 +991,7 @@ def _cleanup_directory()except Exception as cleanup_error:- logger.error(+ logger.exception(f"Failed to clean {cleanup_type} execution directory: {cleanup_error}")return _create_cleanup_result(@@ -963,7 +1011,7 @@ def _cleanup_directory()except Exception as e:- logger.error(f"Directory cleanup failed for {cleanup_type}: {e}")+ logger.exception(f"Directory cleanup failed for {cleanup_type}")return _create_cleanup_result(cleanup_type=cleanup_type, status="failed", error=e)diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.pyindex d8cb88d7a..158ec7278 100644--- a/workers/file_processing/tasks.py+++ b/workers/file_processing/tasks.py@@ -979,7 +979,7 @@ def _check_file_already_active(except Exception as redis_error:# Redis check failed, fall back to DB- logger.warning(+ logger.exception(f"Redis check failed for '{file_name}': {redis_error}. Falling back to DB check")@@ -1177,7 +1177,7 @@ def _pre_create_file_executions()except Exception as e:- logger.error(+ logger.exception(f"Failed to pre-create WorkflowFileExecution for '{file_name}': {str(e)}")# Continue with other files even if one fails@@ -1523,8 +1523,8 @@ def process_file_batch_api(logger.info(f"Successfully processed API file batch {batch_id}")return batch_result- except Exception as e:- logger.error(f"API file batch processing failed for {batch_id}: {e}")+ except Exception:+ logger.exception(f"API file batch processing failed for {batch_id}")raise@@ -1642,7 +1642,7 @@ def _process_single_file_api(except Exception as e:processing_time = time.time() - start_time- logger.error(+ logger.exception(f"Failed to process file {file_name} after {processing_time:.2f}s: {e}")@@ -1653,8 +1653,8 @@ def _process_single_file_api(status=ExecutionStatus.ERROR.value,error_message=str(e),)- except Exception as update_error:- logger.error(f"Failed to update file execution status: {update_error}")+ except Exception:+ logger.exception("Failed to update file execution status")return {"file_execution_id": file_execution_id,@@ -1779,8 +1779,8 @@ def _call_runner_service(f"Runner service call failed after {retry_count} attempts: {e}")raise- except Exception as e:- logger.error(f"Unexpected error calling runner service: {e}")+ except Exception:+ logger.exception("Unexpected error calling runner service")raiseraise Exception(f"Failed to call runner service after {retry_count} attempts")