Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Avoid storing file history for API based wfs (#752)

Chandrasekharan M844b239a2310
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index 6792d7a4a..0ba7df8e9 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -329,7 +329,13 @@ class SourceConnector(BaseConnector):
workflow=workflow, cache_key=file_hash
)
- if file_history and file_history.is_completed():
+ # In case of ETL pipelines, its necessary to skip files which have
+ # already been processed
+ if (
+ self.execution_service.use_file_history
+ and file_history
+ and file_history.is_completed()
+ ):
self.execution_service.publish_log(
f"Skipping file {file_path} as it has already been processed. "
"Clear the file markers to process it again."
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index e7808b607..59b51306d 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -329,7 +329,13 @@ class SourceConnector(BaseConnector):
workflow=workflow, cache_key=file_hash
)
- if file_history and file_history.is_completed():
+ # In case of ETL pipelines, its necessary to skip files which have
+ # already been processed
+ if (
+ self.execution_service.use_file_history
+ and file_history
+ and file_history.is_completed()
+ ):
self.execution_service.publish_log(
f"Skipping file {file_path} as it has already been processed. "
"Clear the file markers to process it again."
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index 08c1c6f3f..740f5d46d 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -173,11 +173,16 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_action = serializer.get_execution_action(serializer.validated_data)
file_objs = request.FILES.getlist("files")
hashes_of_files: dict[str, FileHash] = {}
+ use_file_history: bool = True
+
+ # API based execution
if file_objs and execution_id and workflow_id:
+ use_file_history = False
hashes_of_files = SourceConnector.add_input_file_to_api_storage(
workflow_id=workflow_id,
execution_id=execution_id,
file_objs=file_objs,
+ use_file_history=False,
)
try:
@@ -190,6 +195,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id=execution_id,
pipeline_guid=pipeline_guid,
hash_values_of_files=hashes_of_files,
+ use_file_history=use_file_history,
)
if (
execution_response.execution_status == "ERROR"
@@ -216,6 +222,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id: Optional[str] = None,
pipeline_guid: Optional[str] = None,
hash_values_of_files: dict[str, FileHash] = {},
+ use_file_history: bool = True,
) -> ExecutionResponse:
if execution_action is not None:
# Step execution
@@ -236,6 +243,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
pipeline_id=pipeline_guid,
execution_mode=WorkflowExecution.Mode.INSTANT,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
else:
execution_response = WorkflowHelper.complete_execution(
@@ -243,6 +251,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id=execution_id,
execution_mode=WorkflowExecution.Mode.INSTANT,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
return execution_response
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index 400bd723a..6d30bcdd0 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -572,6 +572,7 @@ class WorkflowHelper:
pipeline_id: Optional[str] = None,
execution_mode: Optional[WorkflowExecution] = WorkflowExecution.Mode.QUEUE,
hash_values_of_files: dict[str, FileHash] = {},
+ use_file_history: bool = True,
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
@@ -595,6 +596,7 @@ class WorkflowHelper:
pipeline_id=pipeline_id,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
return response
else:
@@ -607,6 +609,7 @@ class WorkflowHelper:
execution_mode=execution_mode,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
+ use_file_history=use_file_history,
)
updated_execution = WorkflowExecution.objects.get(id=execution_id)
@@ -635,6 +638,7 @@ class WorkflowHelper:
workflow=workflow,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 0961e4e1a..cb0e63ac6 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -161,12 +161,16 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id = serializer.get_execution_id(serializer.validated_data)
execution_action = serializer.get_execution_action(serializer.validated_data)
file_objs = request.FILES.getlist("files")
+ use_file_history: bool = True
+
hashes_of_files: dict[str, FileHash] = {}
if file_objs and execution_id and workflow_id:
+ use_file_history = False
hashes_of_files = SourceConnector.add_input_file_to_api_storage(
workflow_id=workflow_id,
execution_id=execution_id,
file_objs=file_objs,
+ use_file_history=False,
)
try:
@@ -179,6 +183,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id=execution_id,
pipeline_guid=pipeline_guid,
hash_values_of_files=hashes_of_files,
+ use_file_history=use_file_history,
)
if (
execution_response.execution_status == "ERROR"
@@ -205,6 +210,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id: Optional[str] = None,
pipeline_guid: Optional[str] = None,
hash_values_of_files: dict[str, FileHash] = {},
+ use_file_history: bool = False,
) -> ExecutionResponse:
if execution_action is not None:
# Step execution
@@ -225,6 +231,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
pipeline_id=pipeline_guid,
execution_mode=WorkflowExecution.Mode.INSTANT,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
else:
execution_response = WorkflowHelper.complete_execution(
@@ -232,6 +239,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id=execution_id,
execution_mode=WorkflowExecution.Mode.INSTANT,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
return execution_response
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 9b2c432de..8df5fc575 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -620,6 +620,7 @@ class WorkflowHelper:
pipeline_id: Optional[str] = None,
execution_mode: Optional[WorkflowExecution] = WorkflowExecution.Mode.QUEUE,
hash_values_of_files: dict[str, FileHash] = {},
+ use_file_history: bool = False,
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
@@ -643,6 +644,7 @@ class WorkflowHelper:
pipeline_id=pipeline_id,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
return response
else:
@@ -655,6 +657,7 @@ class WorkflowHelper:
execution_mode=execution_mode,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
+ use_file_history=use_file_history,
)
updated_execution = WorkflowExecution.objects.get(id=execution_id)
@@ -683,6 +686,7 @@ class WorkflowHelper:
workflow=workflow,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
+ use_file_history=use_file_history,
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(