Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2867 [FIX] Fixed webhook notifications showing incorrect pipeline name for SUCCESS/COMPLETED events (#1577)

ali2aebf04905ee
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py
index 088cdc6f8..8dc66aa53 100644
--- a/workers/callback/tasks.py
+++ b/workers/callback/tasks.py
@@ -620,6 +620,51 @@ def _handle_notifications_unified(
}
+def _fetch_pipeline_name_from_api(
+ pipeline_id: str,
+ api_client: InternalAPIClient,
+) -> str | None:
+ """Fetch actual pipeline name from Pipeline/APIDeployment models.
+
+ Args:
+ pipeline_id: Pipeline or API deployment ID
+ api_client: API client instance
+
+ Returns:
+ Pipeline name from API, or None if fetch failed
+ """
+ try:
+ from shared.models.pipeline_models import PipelineApiResponse
+
+ logger.info(
+ f"Fetching pipeline data for pipeline_id={pipeline_id} to get correct pipeline name"
+ )
+ pipeline_response = api_client.get_pipeline_data(
+ pipeline_id=pipeline_id, check_active=False
+ )
+
+ if pipeline_response.success:
+ # Parse response using type-safe dataclass
+ pipeline_api_data = PipelineApiResponse.from_dict(pipeline_response.data)
+ pipeline_name = pipeline_api_data.pipeline.pipeline_name
+
+ logger.info(
+ f"Fetched pipeline name from API: '{pipeline_name}' for pipeline_id={pipeline_id}"
+ )
+ return pipeline_name
+ else:
+ logger.warning(
+ f"Could not fetch pipeline data for {pipeline_id}: {pipeline_response.error}"
+ )
+ return None
+ except Exception:
+ logger.exception(
+ f"Error fetching pipeline name for {pipeline_id}."
+ f"Will use 'Unknown API' or 'Unknown Pipeline' in notifications."
+ )
+ return None
+
+
def _extract_callback_parameters(
task_instance, results: list, kwargs: dict[str, Any]
) -> CallbackContext:
@@ -748,37 +793,40 @@ def _extract_callback_parameters(
# Use existing API detection from source_config (no additional API calls needed)
is_api_deployment = source_config.get("is_api", False)
+ # Fetch actual pipeline name from Pipeline/APIDeployment models
+ pipeline_name_from_api = None
+ if context.pipeline_id:
+ pipeline_name_from_api = _fetch_pipeline_name_from_api(
+ context.pipeline_id, api_client
+ )
+
if is_api_deployment:
# This is an API deployment
+ # Use fetched pipeline name if available, otherwise use "Unknown API"
+ resolved_pipeline_name = pipeline_name_from_api or "Unknown API"
context.pipeline_data = {
"is_api": True,
"resolved_pipeline_type": "API",
- "resolved_pipeline_name": workflow_definition.get(
- "workflow_name", "Unknown API"
- ),
+ "resolved_pipeline_name": resolved_pipeline_name,
}
context.pipeline_type = "API"
- context.pipeline_name = workflow_definition.get(
- "workflow_name", "Unknown API"
- )
+ context.pipeline_name = resolved_pipeline_name
logger.info(
- f"Detected API deployment from source_config: {context.pipeline_id}"
+ f"Detected API deployment from source_config: {context.pipeline_id}, pipeline_name='{resolved_pipeline_name}'"
)
else:
# This is ETL/TASK/APP workflow
+ # Use fetched pipeline name if available, otherwise use "Unknown Pipeline"
+ resolved_pipeline_name = pipeline_name_from_api or "Unknown Pipeline"
context.pipeline_data = {
"is_api": False,
"resolved_pipeline_type": "ETL",
- "resolved_pipeline_name": workflow_definition.get(
- "workflow_name", "Unknown Workflow"
- ),
+ "resolved_pipeline_name": resolved_pipeline_name,
}
context.pipeline_type = "ETL"
- context.pipeline_name = workflow_definition.get(
- "workflow_name", "Unknown Workflow"
- )
+ context.pipeline_name = resolved_pipeline_name
logger.info(
- f"Detected ETL workflow from source_config: {context.pipeline_id}"
+ f"Detected ETL workflow from source_config: {context.pipeline_id}, pipeline_name='{resolved_pipeline_name}'"
)
logger.info(
@@ -789,7 +837,7 @@ def _extract_callback_parameters(
)
except Exception as e:
- logger.error(f"Failed to fetch workflow execution context: {e}")
+ logger.exception("Failed to fetch workflow execution context")
raise ValueError(f"Could not get execution context: {e}")
# 4. Validate required context is now available
@@ -943,7 +991,7 @@ def _cleanup_directory(
)
except Exception as cleanup_error:
- logger.error(
+ logger.exception(
f"Failed to clean {cleanup_type} execution directory: {cleanup_error}"
)
return _create_cleanup_result(
@@ -963,7 +1011,7 @@ def _cleanup_directory(
)
except Exception as e:
- logger.error(f"Directory cleanup failed for {cleanup_type}: {e}")
+ logger.exception(f"Directory cleanup failed for {cleanup_type}")
return _create_cleanup_result(cleanup_type=cleanup_type, status="failed", error=e)
diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.py
index d8cb88d7a..158ec7278 100644
--- a/workers/file_processing/tasks.py
+++ b/workers/file_processing/tasks.py
@@ -979,7 +979,7 @@ def _check_file_already_active(
except Exception as redis_error:
# Redis check failed, fall back to DB
- logger.warning(
+ logger.exception(
f"Redis check failed for '{file_name}': {redis_error}. Falling back to DB check"
)
@@ -1177,7 +1177,7 @@ def _pre_create_file_executions(
)
except Exception as e:
- logger.error(
+ logger.exception(
f"Failed to pre-create WorkflowFileExecution for '{file_name}': {str(e)}"
)
# Continue with other files even if one fails
@@ -1523,8 +1523,8 @@ def process_file_batch_api(
logger.info(f"Successfully processed API file batch {batch_id}")
return batch_result
- except Exception as e:
- logger.error(f"API file batch processing failed for {batch_id}: {e}")
+ except Exception:
+ logger.exception(f"API file batch processing failed for {batch_id}")
raise
@@ -1642,7 +1642,7 @@ def _process_single_file_api(
except Exception as e:
processing_time = time.time() - start_time
- logger.error(
+ logger.exception(
f"Failed to process file {file_name} after {processing_time:.2f}s: {e}"
)
@@ -1653,8 +1653,8 @@ def _process_single_file_api(
status=ExecutionStatus.ERROR.value,
error_message=str(e),
)
- except Exception as update_error:
- logger.error(f"Failed to update file execution status: {update_error}")
+ except Exception:
+ logger.exception("Failed to update file execution status")
return {
"file_execution_id": file_execution_id,
@@ -1779,8 +1779,8 @@ def _call_runner_service(
f"Runner service call failed after {retry_count} attempts: {e}"
)
raise
- except Exception as e:
- logger.error(f"Unexpected error calling runner service: {e}")
+ except Exception:
+ logger.exception("Unexpected error calling runner service")
raise
raise Exception(f"Failed to call runner service after {retry_count} attempts")