LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Avoid storing file history for API based wfs (#752)
Chandrasekharan M844b239a2310
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 6792d7a4a..0ba7df8e9 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -329,7 +329,13 @@ class SourceConnector(BaseConnector):workflow=workflow, cache_key=file_hash)- if file_history and file_history.is_completed():+ # In case of ETL pipelines, its necessary to skip files which have+ # already been processed+ if (+ self.execution_service.use_file_history+ and file_history+ and file_history.is_completed()+ ):self.execution_service.publish_log(f"Skipping file {file_path} as it has already been processed. ""Clear the file markers to process it again."diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex e7808b607..59b51306d 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -329,7 +329,13 @@ class SourceConnector(BaseConnector):workflow=workflow, cache_key=file_hash)- if file_history and file_history.is_completed():+ # In case of ETL pipelines, its necessary to skip files which have+ # already been processed+ if (+ self.execution_service.use_file_history+ and file_history+ and file_history.is_completed()+ ):self.execution_service.publish_log(f"Skipping file {file_path} as it has already been processed. ""Clear the file markers to process it again."diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex 08c1c6f3f..740f5d46d 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -173,11 +173,16 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_action = serializer.get_execution_action(serializer.validated_data)file_objs = request.FILES.getlist("files")hashes_of_files: dict[str, FileHash] = {}+ use_file_history: bool = True++ # API based executionif file_objs and execution_id and workflow_id:+ use_file_history = Falsehashes_of_files = SourceConnector.add_input_file_to_api_storage(workflow_id=workflow_id,execution_id=execution_id,file_objs=file_objs,+ use_file_history=False,)try:@@ -190,6 +195,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id=execution_id,pipeline_guid=pipeline_guid,hash_values_of_files=hashes_of_files,+ use_file_history=use_file_history,)if (execution_response.execution_status == "ERROR"@@ -216,6 +222,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id: Optional[str] = None,pipeline_guid: Optional[str] = None,hash_values_of_files: dict[str, FileHash] = {},+ use_file_history: bool = True,) -> ExecutionResponse:if execution_action is not None:# Step execution@@ -236,6 +243,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):pipeline_id=pipeline_guid,execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)else:execution_response = WorkflowHelper.complete_execution(@@ -243,6 +251,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id=execution_id,execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)return execution_responsediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 400bd723a..6d30bcdd0 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -572,6 +572,7 @@ class WorkflowHelper:pipeline_id: Optional[str] = None,execution_mode: Optional[WorkflowExecution] = WorkflowExecution.Mode.QUEUE,hash_values_of_files: dict[str, FileHash] = {},+ use_file_history: bool = True,) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")@@ -595,6 +596,7 @@ class WorkflowHelper:pipeline_id=pipeline_id,execution_id=execution_id,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)return responseelse:@@ -607,6 +609,7 @@ class WorkflowHelper:execution_mode=execution_mode,pipeline_id=pipeline_id,log_events_id=log_events_id,+ use_file_history=use_file_history,)updated_execution = WorkflowExecution.objects.get(id=execution_id)@@ -635,6 +638,7 @@ class WorkflowHelper:workflow=workflow,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 0961e4e1a..cb0e63ac6 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -161,12 +161,16 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id = serializer.get_execution_id(serializer.validated_data)execution_action = serializer.get_execution_action(serializer.validated_data)file_objs = request.FILES.getlist("files")+ use_file_history: bool = True+hashes_of_files: dict[str, FileHash] = {}if file_objs and execution_id and workflow_id:+ use_file_history = Falsehashes_of_files = SourceConnector.add_input_file_to_api_storage(workflow_id=workflow_id,execution_id=execution_id,file_objs=file_objs,+ use_file_history=False,)try:@@ -179,6 +183,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id=execution_id,pipeline_guid=pipeline_guid,hash_values_of_files=hashes_of_files,+ use_file_history=use_file_history,)if (execution_response.execution_status == "ERROR"@@ -205,6 +210,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id: Optional[str] = None,pipeline_guid: Optional[str] = None,hash_values_of_files: dict[str, FileHash] = {},+ use_file_history: bool = False,) -> ExecutionResponse:if execution_action is not None:# Step execution@@ -225,6 +231,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):pipeline_id=pipeline_guid,execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)else:execution_response = WorkflowHelper.complete_execution(@@ -232,6 +239,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id=execution_id,execution_mode=WorkflowExecution.Mode.INSTANT,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)return execution_responsediff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 9b2c432de..8df5fc575 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -620,6 +620,7 @@ class WorkflowHelper:pipeline_id: Optional[str] = None,execution_mode: Optional[WorkflowExecution] = WorkflowExecution.Mode.QUEUE,hash_values_of_files: dict[str, FileHash] = {},+ use_file_history: bool = False,) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")@@ -643,6 +644,7 @@ class WorkflowHelper:pipeline_id=pipeline_id,execution_id=execution_id,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)return responseelse:@@ -655,6 +657,7 @@ class WorkflowHelper:execution_mode=execution_mode,pipeline_id=pipeline_id,log_events_id=log_events_id,+ use_file_history=use_file_history,)updated_execution = WorkflowExecution.objects.get(id=execution_id)@@ -683,6 +686,7 @@ class WorkflowHelper:workflow=workflow,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,+ use_file_history=use_file_history,)except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(