LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
MT-v2 changes for param to toggle file history usage (#748)
Chandrasekharan M9d72420b202b
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex 5750f6b7f..c3265e75d 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -49,10 +49,9 @@ class DeploymentExecution(views.APIView):file_objs = request.FILES.getlist(ApiExecution.FILES_FORM_DATA)serializer = ExecutionRequestSerializer(data=request.data)serializer.is_valid(raise_exception=True)- timeout = serializer.get_timeout(serializer.validated_data)- include_metadata = (- request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"- )+ timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)+ include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)+ use_file_history = serializer.validated_data.get(ApiExecution.USE_FILE_HISTORY)if not file_objs or len(file_objs) == 0:raise InvalidAPIRequest("File shouldn't be empty")response = DeploymentHelper.execute_workflow(@@ -61,6 +60,7 @@ class DeploymentExecution(views.APIView):file_objs=file_objs,timeout=timeout,include_metadata=include_metadata,+ use_file_history=use_file_history,)if "error" in response and response["error"]:return Response(diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.pyindex 0ec324cc9..2de41efb2 100644--- a/backend/api_v2/constants.py+++ b/backend/api_v2/constants.py@@ -4,3 +4,4 @@ class ApiExecution:FILES_FORM_DATA: str = "files"TIMEOUT_FORM_DATA: str = "timeout"INCLUDE_METADATA: str = "include_metadata"+ USE_FILE_HISTORY: str = "use_file_history" # Undocumented parameterdiff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex 47ec0615e..b35bb0a3a 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -131,6 +131,7 @@ class DeploymentHelper(BaseAPIKeyValidator):file_objs: list[UploadedFile],timeout: int,include_metadata: bool = False,+ use_file_history: bool = False,) -> ReturnDict:"""Execute workflow by api.@@ -138,6 +139,8 @@ class DeploymentHelper(BaseAPIKeyValidator):organization_name (str): organization nameapi (APIDeployment): api model objectfile_obj (UploadedFile): input file+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to FalseReturns:ReturnDict: execution status/ result@@ -150,6 +153,7 @@ class DeploymentHelper(BaseAPIKeyValidator):workflow_id=workflow_id,execution_id=execution_id,file_objs=file_objs,+ use_file_history=use_file_history,)try:result = WorkflowHelper.execute_workflow_async(@@ -159,6 +163,7 @@ class DeploymentHelper(BaseAPIKeyValidator):timeout=timeout,execution_id=execution_id,include_metadata=include_metadata,+ use_file_history=use_file_history,)result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_iddiff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.pyindex 4273b6093..88cc22dfb 100644--- a/backend/api_v2/serializers.py+++ b/backend/api_v2/serializers.py@@ -6,6 +6,7 @@ from api_v2.models import APIDeployment, APIKeyfrom django.core.validators import RegexValidatorfrom pipeline_v2.models import Pipelinefrom rest_framework.serializers import (+ BooleanField,CharField,IntegerField,JSONField,@@ -80,26 +81,22 @@ class APIKeySerializer(AuditSerializer):class ExecutionRequestSerializer(Serializer):- """Execution request serializer- timeout: 0: maximum value of timeout, -1: async execution+ """Execution request serializer.++ Attributes:+ timeout (int): Timeout for the API deployment, maximum value can be 300s.+ If -1 it corresponds to async execution. Defaults to -1+ include_metadata (bool): Flag to include metadata in API response+ use_file_history (bool): Flag to use FileHistory to save and retrieve+ responses quickly. This is undocumented to the user and can be+ helpful for demos."""timeout = IntegerField(min_value=-1, max_value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC, default=-1)-- def validate_timeout(self, value: Any) -> int:- if not isinstance(value, int):- raise ValidationError("timeout must be a integer.")- if value == 0:- value = ApiExecution.MAXIMUM_TIMEOUT_IN_SEC- return value-- def get_timeout(self, validated_data: dict[str, Union[int, None]]) -> int:- value = validated_data.get(ApiExecution.TIMEOUT_FORM_DATA, -1)- if not isinstance(value, int):- raise ValidationError("timeout must be a integer.")- return value+ include_metadata = BooleanField(default=False)+ use_file_history = BooleanField(default=False)class APIDeploymentListSerializer(ModelSerializer):diff --git a/backend/usage_v2/helper.py b/backend/usage_v2/helper.pyindex 20182fc3d..fd217cd6f 100644--- a/backend/usage_v2/helper.py+++ b/backend/usage_v2/helper.py@@ -61,4 +61,4 @@ class UsageHelper:except Exception as e:# Handle any other exceptions that might occur during the executionlogger.error(f"An unexpected error occurred for run_id {run_id}: {str(e)}")- raise APIException("An unexpected error occurred")+ raise APIException("Error while aggregating token counts")diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 6f3ec1d75..39a989d70 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -152,8 +152,9 @@ class DestinationConnector(BaseConnector):file_name: str,file_hash: FileHash,workflow: Workflow,+ input_file_path: str,error: Optional[str] = None,- input_file_path: Optional[str] = None,+ use_file_history: bool = True,) -> None:"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_type@@ -163,9 +164,12 @@ class DestinationConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.API:self._handle_api_result(file_name=file_name, error=error, result=result)return- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash.file_hash- )++ file_history = None+ if use_file_history:+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash.file_hash+ )if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:@@ -188,7 +192,7 @@ class DestinationConnector(BaseConnector):self.execution_service.publish_log(message=f"File '{file_name}' processed successfully")- if not file_history:+ if use_file_history and not file_history:FileHistoryHelper.create_file_history(cache_key=file_hash.file_hash,workflow=workflow,diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 4ba76a488..e7808b607 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -485,9 +485,7 @@ class SourceConnector(BaseConnector):shutil.copyfile(input_file_path, infile_path)shutil.copyfile(input_file_path, source_path)- def add_file_to_volume(- self, input_file_path: str, file_hash: FileHash- ) -> tuple[str, str]:+ def add_file_to_volume(self, input_file_path: str, file_hash: FileHash) -> str:"""Add input file to execution directory.Args:@@ -555,7 +553,11 @@ class SourceConnector(BaseConnector):@classmethoddef add_input_file_to_api_storage(- cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]+ cls,+ workflow_id: str,+ execution_id: str,+ file_objs: list[UploadedFile],+ use_file_history: bool = False,) -> dict[str, FileHash]:"""Add input file to api storage.@@ -563,6 +565,8 @@ class SourceConnector(BaseConnector):workflow_id (str): UUID of the worklfowexecution_id (str): UUID of the executionfile_objs (list[UploadedFile]): List of uploaded files+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to FalseReturns:dict[str, FileHash]: Dict containing file name and its corresponding hash"""@@ -582,9 +586,12 @@ class SourceConnector(BaseConnector):f.write(buffer)file_hash = cls.hash_str(buffer)connection_type = WorkflowEndpoint.ConnectionType.API- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash- )++ file_history = None+ if use_file_history:+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash+ )is_executed = (True if file_history and file_history.is_completed() else False)diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 3af07d748..e71b198f4 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -34,6 +34,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):scheduled: bool = False,mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,workflow_execution: Optional[WorkflowExecution] = None,+ use_file_history: bool = True,) -> None:tool_instances_as_dto = []for tool_instance in tool_instances:@@ -95,6 +96,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.project_settings = project_settingsself.pipeline_id = pipeline_idself.execution_id = str(workflow_execution.id)+ self.use_file_history = use_file_historylogger.info(f"Executing for Pipeline ID: {pipeline_id}, "f"workflow ID: {self.workflow_id}, execution ID: {self.execution_id}, "diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 0ac9b79ec..ca41c9f95 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -92,6 +92,7 @@ class WorkflowHelper:scheduled: bool,execution_mode: tuple[str, str],workflow_execution: Optional[WorkflowExecution],+ use_file_history: bool = True, # Will be False for API deployment alone) -> WorkflowExecutionServiceHelper:workflow_execution_service = WorkflowExecutionServiceHelper(organization_id=organization_id,@@ -102,6 +103,7 @@ class WorkflowHelper:scheduled=scheduled,mode=execution_mode,workflow_execution=workflow_execution,+ use_file_history=use_file_history,)workflow_execution_service.build()return workflow_execution_service@@ -218,8 +220,9 @@ class WorkflowHelper:file_name=file_name,file_hash=file_hash,workflow=workflow,- error=error,input_file_path=input_file,+ error=error,+ use_file_history=execution_service.use_file_history,)execution_service.publish_update_log(LogState.SUCCESS,@@ -249,6 +252,7 @@ class WorkflowHelper:single_step: bool = False,workflow_execution: Optional[WorkflowExecution] = None,execution_mode: Optional[tuple[str, str]] = None,+ use_file_history: bool = True,) -> ExecutionResponse:tool_instances: list[ToolInstance] = (ToolInstanceHelper.get_tool_instances_by_workflow(@@ -267,6 +271,7 @@ class WorkflowHelper:scheduled=scheduled,execution_mode=execution_mode,workflow_execution=workflow_execution,+ use_file_history=use_file_history,)execution_id = execution_service.execution_idsource = SourceConnector(@@ -391,6 +396,7 @@ class WorkflowHelper:timeout: int = -1,pipeline_id: Optional[str] = None,queue: Optional[str] = None,+ use_file_history: bool = True,) -> ExecutionResponse:"""Adding a workflow to the queue for execution.@@ -399,6 +405,9 @@ class WorkflowHelper:execution_id (str): Execution IDtimeout (int): Celery timeout (timeout -1 : async execution)pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.+ queue (Optional[str]): Name of the celery queue to push into+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to TrueReturns:ExecutionResponse: Existing status of execution@@ -421,6 +430,7 @@ class WorkflowHelper:"execution_mode": None,"pipeline_id": pipeline_id,"log_events_id": log_events_id,+ "use_file_history": use_file_history,},queue=queue,)@@ -482,6 +492,7 @@ class WorkflowHelper:scheduled: bool = False,execution_mode: Optional[tuple[str, str]] = None,pipeline_id: Optional[str] = None,+ use_file_history: bool = True,**kwargs: dict[str, Any],) -> Optional[list[Any]]:"""Asynchronous Execution By celery.@@ -495,6 +506,8 @@ class WorkflowHelper:execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution ModeDefaults to Nonepipeline_id (Optional[str], optional): Id of pipeline. Defaults to None+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to TrueKwargs:log_events_id (str): Session ID of the user,@@ -515,6 +528,7 @@ class WorkflowHelper:scheduled=scheduled,execution_mode=execution_mode,pipeline_id=pipeline_id,+ use_file_history=use_file_history,**kwargs,)