Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

MT-v2 changes for param to toggle file history usage (#748)

Chandrasekharan M9d72420b202b
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index 5750f6b7f..c3265e75d 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -49,10 +49,9 @@ class DeploymentExecution(views.APIView):
file_objs = request.FILES.getlist(ApiExecution.FILES_FORM_DATA)
serializer = ExecutionRequestSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
- timeout = serializer.get_timeout(serializer.validated_data)
- include_metadata = (
- request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"
- )
+ timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)
+ include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)
+ use_file_history = serializer.validated_data.get(ApiExecution.USE_FILE_HISTORY)
if not file_objs or len(file_objs) == 0:
raise InvalidAPIRequest("File shouldn't be empty")
response = DeploymentHelper.execute_workflow(
@@ -61,6 +60,7 @@ class DeploymentExecution(views.APIView):
file_objs=file_objs,
timeout=timeout,
include_metadata=include_metadata,
+ use_file_history=use_file_history,
)
if "error" in response and response["error"]:
return Response(
diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.py
index 0ec324cc9..2de41efb2 100644
--- a/backend/api_v2/constants.py
+++ b/backend/api_v2/constants.py
@@ -4,3 +4,4 @@ class ApiExecution:
FILES_FORM_DATA: str = "files"
TIMEOUT_FORM_DATA: str = "timeout"
INCLUDE_METADATA: str = "include_metadata"
+ USE_FILE_HISTORY: str = "use_file_history" # Undocumented parameter
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index 47ec0615e..b35bb0a3a 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -131,6 +131,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
file_objs: list[UploadedFile],
timeout: int,
include_metadata: bool = False,
+ use_file_history: bool = False,
) -> ReturnDict:
"""Execute workflow by api.
@@ -138,6 +139,8 @@ class DeploymentHelper(BaseAPIKeyValidator):
organization_name (str): organization name
api (APIDeployment): api model object
file_obj (UploadedFile): input file
+ use_file_history (bool): Use FileHistory table to return results on already
+ processed files. Defaults to False
Returns:
ReturnDict: execution status/ result
@@ -150,6 +153,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
workflow_id=workflow_id,
execution_id=execution_id,
file_objs=file_objs,
+ use_file_history=use_file_history,
)
try:
result = WorkflowHelper.execute_workflow_async(
@@ -159,6 +163,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
timeout=timeout,
execution_id=execution_id,
include_metadata=include_metadata,
+ use_file_history=use_file_history,
)
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py
index 4273b6093..88cc22dfb 100644
--- a/backend/api_v2/serializers.py
+++ b/backend/api_v2/serializers.py
@@ -6,6 +6,7 @@ from api_v2.models import APIDeployment, APIKey
from django.core.validators import RegexValidator
from pipeline_v2.models import Pipeline
from rest_framework.serializers import (
+ BooleanField,
CharField,
IntegerField,
JSONField,
@@ -80,26 +81,22 @@ class APIKeySerializer(AuditSerializer):
class ExecutionRequestSerializer(Serializer):
- """Execution request serializer
- timeout: 0: maximum value of timeout, -1: async execution
+ """Execution request serializer.
+
+ Attributes:
+ timeout (int): Timeout for the API deployment, maximum value can be 300s.
+ If -1 it corresponds to async execution. Defaults to -1
+ include_metadata (bool): Flag to include metadata in API response
+ use_file_history (bool): Flag to use FileHistory to save and retrieve
+ responses quickly. This is undocumented to the user and can be
+ helpful for demos.
"""
timeout = IntegerField(
min_value=-1, max_value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC, default=-1
)
-
- def validate_timeout(self, value: Any) -> int:
- if not isinstance(value, int):
- raise ValidationError("timeout must be a integer.")
- if value == 0:
- value = ApiExecution.MAXIMUM_TIMEOUT_IN_SEC
- return value
-
- def get_timeout(self, validated_data: dict[str, Union[int, None]]) -> int:
- value = validated_data.get(ApiExecution.TIMEOUT_FORM_DATA, -1)
- if not isinstance(value, int):
- raise ValidationError("timeout must be a integer.")
- return value
+ include_metadata = BooleanField(default=False)
+ use_file_history = BooleanField(default=False)
class APIDeploymentListSerializer(ModelSerializer):
diff --git a/backend/usage_v2/helper.py b/backend/usage_v2/helper.py
index 20182fc3d..fd217cd6f 100644
--- a/backend/usage_v2/helper.py
+++ b/backend/usage_v2/helper.py
@@ -61,4 +61,4 @@ class UsageHelper:
except Exception as e:
# Handle any other exceptions that might occur during the execution
logger.error(f"An unexpected error occurred for run_id {run_id}: {str(e)}")
- raise APIException("An unexpected error occurred")
+ raise APIException("Error while aggregating token counts")
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 6f3ec1d75..39a989d70 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -152,8 +152,9 @@ class DestinationConnector(BaseConnector):
file_name: str,
file_hash: FileHash,
workflow: Workflow,
+ input_file_path: str,
error: Optional[str] = None,
- input_file_path: Optional[str] = None,
+ use_file_history: bool = True,
) -> None:
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
@@ -163,9 +164,12 @@ class DestinationConnector(BaseConnector):
if connection_type == WorkflowEndpoint.ConnectionType.API:
self._handle_api_result(file_name=file_name, error=error, result=result)
return
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash.file_hash
- )
+
+ file_history = None
+ if use_file_history:
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash.file_hash
+ )
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
@@ -188,7 +192,7 @@ class DestinationConnector(BaseConnector):
self.execution_service.publish_log(
message=f"File '{file_name}' processed successfully"
)
- if not file_history:
+ if use_file_history and not file_history:
FileHistoryHelper.create_file_history(
cache_key=file_hash.file_hash,
workflow=workflow,
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 4ba76a488..e7808b607 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -485,9 +485,7 @@ class SourceConnector(BaseConnector):
shutil.copyfile(input_file_path, infile_path)
shutil.copyfile(input_file_path, source_path)
- def add_file_to_volume(
- self, input_file_path: str, file_hash: FileHash
- ) -> tuple[str, str]:
+ def add_file_to_volume(self, input_file_path: str, file_hash: FileHash) -> str:
"""Add input file to execution directory.
Args:
@@ -555,7 +553,11 @@ class SourceConnector(BaseConnector):
@classmethod
def add_input_file_to_api_storage(
- cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]
+ cls,
+ workflow_id: str,
+ execution_id: str,
+ file_objs: list[UploadedFile],
+ use_file_history: bool = False,
) -> dict[str, FileHash]:
"""Add input file to api storage.
@@ -563,6 +565,8 @@ class SourceConnector(BaseConnector):
workflow_id (str): UUID of the worklfow
execution_id (str): UUID of the execution
file_objs (list[UploadedFile]): List of uploaded files
+ use_file_history (bool): Use FileHistory table to return results on already
+ processed files. Defaults to False
Returns:
dict[str, FileHash]: Dict containing file name and its corresponding hash
"""
@@ -582,9 +586,12 @@ class SourceConnector(BaseConnector):
f.write(buffer)
file_hash = cls.hash_str(buffer)
connection_type = WorkflowEndpoint.ConnectionType.API
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash
- )
+
+ file_history = None
+ if use_file_history:
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash
+ )
is_executed = (
True if file_history and file_history.is_completed() else False
)
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index 3af07d748..e71b198f4 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -34,6 +34,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
scheduled: bool = False,
mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
workflow_execution: Optional[WorkflowExecution] = None,
+ use_file_history: bool = True,
) -> None:
tool_instances_as_dto = []
for tool_instance in tool_instances:
@@ -95,6 +96,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.project_settings = project_settings
self.pipeline_id = pipeline_id
self.execution_id = str(workflow_execution.id)
+ self.use_file_history = use_file_history
logger.info(
f"Executing for Pipeline ID: {pipeline_id}, "
f"workflow ID: {self.workflow_id}, execution ID: {self.execution_id}, "
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 0ac9b79ec..ca41c9f95 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -92,6 +92,7 @@ class WorkflowHelper:
scheduled: bool,
execution_mode: tuple[str, str],
workflow_execution: Optional[WorkflowExecution],
+ use_file_history: bool = True, # Will be False for API deployment alone
) -> WorkflowExecutionServiceHelper:
workflow_execution_service = WorkflowExecutionServiceHelper(
organization_id=organization_id,
@@ -102,6 +103,7 @@ class WorkflowHelper:
scheduled=scheduled,
mode=execution_mode,
workflow_execution=workflow_execution,
+ use_file_history=use_file_history,
)
workflow_execution_service.build()
return workflow_execution_service
@@ -218,8 +220,9 @@ class WorkflowHelper:
file_name=file_name,
file_hash=file_hash,
workflow=workflow,
- error=error,
input_file_path=input_file,
+ error=error,
+ use_file_history=execution_service.use_file_history,
)
execution_service.publish_update_log(
LogState.SUCCESS,
@@ -249,6 +252,7 @@ class WorkflowHelper:
single_step: bool = False,
workflow_execution: Optional[WorkflowExecution] = None,
execution_mode: Optional[tuple[str, str]] = None,
+ use_file_history: bool = True,
) -> ExecutionResponse:
tool_instances: list[ToolInstance] = (
ToolInstanceHelper.get_tool_instances_by_workflow(
@@ -267,6 +271,7 @@ class WorkflowHelper:
scheduled=scheduled,
execution_mode=execution_mode,
workflow_execution=workflow_execution,
+ use_file_history=use_file_history,
)
execution_id = execution_service.execution_id
source = SourceConnector(
@@ -391,6 +396,7 @@ class WorkflowHelper:
timeout: int = -1,
pipeline_id: Optional[str] = None,
queue: Optional[str] = None,
+ use_file_history: bool = True,
) -> ExecutionResponse:
"""Adding a workflow to the queue for execution.
@@ -399,6 +405,9 @@ class WorkflowHelper:
execution_id (str): Execution ID
timeout (int): Celery timeout (timeout -1 : async execution)
pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.
+ queue (Optional[str]): Name of the celery queue to push into
+ use_file_history (bool): Use FileHistory table to return results on already
+ processed files. Defaults to True
Returns:
ExecutionResponse: Existing status of execution
@@ -421,6 +430,7 @@ class WorkflowHelper:
"execution_mode": None,
"pipeline_id": pipeline_id,
"log_events_id": log_events_id,
+ "use_file_history": use_file_history,
},
queue=queue,
)
@@ -482,6 +492,7 @@ class WorkflowHelper:
scheduled: bool = False,
execution_mode: Optional[tuple[str, str]] = None,
pipeline_id: Optional[str] = None,
+ use_file_history: bool = True,
**kwargs: dict[str, Any],
) -> Optional[list[Any]]:
"""Asynchronous Execution By celery.
@@ -495,6 +506,8 @@ class WorkflowHelper:
execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution Mode
Defaults to None
pipeline_id (Optional[str], optional): Id of pipeline. Defaults to None
+ use_file_history (bool): Use FileHistory table to return results on already
+ processed files. Defaults to True
Kwargs:
log_events_id (str): Session ID of the user,
@@ -515,6 +528,7 @@ class WorkflowHelper:
scheduled=scheduled,
execution_mode=execution_mode,
pipeline_id=pipeline_id,
+ use_file_history=use_file_history,
**kwargs,
)