Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix file skipping while API-deloyment execution (#1230)

ali56c902846edc
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 1ef31ee93..63ed2218c 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -74,6 +74,9 @@ class DestinationConnector(BaseConnector):
self.api_results: list[dict[str, Any]] = []
self.queue_results: list[dict[str, Any]] = []
self.execution_service = execution_service
+ self.is_api: bool = (
+ self.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
+ )
def _get_endpoint_for_workflow(
self,
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index fffbed3cf..cfca80ad3 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -229,8 +229,11 @@ class WorkflowHelper:
workflow_file_execution.update(file_hash=file_content_hash)
# Ensure no duplicate file processing
- if FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_content_hash
+ if (
+ FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_content_hash
+ )
+ and not destination.is_api
):
logger.info(f"Skipping '{file_name}', already processed.")
execution_service.publish_log(