LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Added option to API deployment: use_cache, toggles usage of FileHistory (#745)
Chandrasekharan M68f72f5f0cad
diff --git a/backend/api/api_deployment_views.py b/backend/api/api_deployment_views.pyindex bce65aabe..99749988e 100644--- a/backend/api/api_deployment_views.py+++ b/backend/api/api_deployment_views.py@@ -49,10 +49,9 @@ class DeploymentExecution(views.APIView):file_objs = request.FILES.getlist(ApiExecution.FILES_FORM_DATA)serializer = ExecutionRequestSerializer(data=request.data)serializer.is_valid(raise_exception=True)- timeout = serializer.get_timeout(serializer.validated_data)- include_metadata = (- request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"- )+ timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)+ include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)+ use_file_history = serializer.validated_data.get(ApiExecution.USE_FILE_HISTORY)if not file_objs or len(file_objs) == 0:raise InvalidAPIRequest("File shouldn't be empty")response = DeploymentHelper.execute_workflow(@@ -61,6 +60,7 @@ class DeploymentExecution(views.APIView):file_objs=file_objs,timeout=timeout,include_metadata=include_metadata,+ use_file_history=use_file_history,)if "error" in response and response["error"]:return Response(diff --git a/backend/api/constants.py b/backend/api/constants.pyindex 0ec324cc9..2de41efb2 100644--- a/backend/api/constants.py+++ b/backend/api/constants.py@@ -4,3 +4,4 @@ class ApiExecution:FILES_FORM_DATA: str = "files"TIMEOUT_FORM_DATA: str = "timeout"INCLUDE_METADATA: str = "include_metadata"+ USE_FILE_HISTORY: str = "use_file_history" # Undocumented parameterdiff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.pyindex 494eb002e..c301f56d1 100644--- a/backend/api/deployment_helper.py+++ b/backend/api/deployment_helper.py@@ -148,6 +148,7 @@ class DeploymentHelper(BaseAPIKeyValidator):file_objs: list[UploadedFile],timeout: int,include_metadata: bool = False,+ use_file_history: bool = False,) -> ReturnDict:"""Execute workflow by api.@@ -155,6 +156,8 @@ class DeploymentHelper(BaseAPIKeyValidator):organization_name (str): organization nameapi (APIDeployment): api model objectfile_obj (UploadedFile): input file+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to FalseReturns:ReturnDict: execution status/ result@@ -167,6 +170,7 @@ class DeploymentHelper(BaseAPIKeyValidator):workflow_id=workflow_id,execution_id=execution_id,file_objs=file_objs,+ use_file_history=use_file_history,)try:result = WorkflowHelper.execute_workflow_async(@@ -176,6 +180,7 @@ class DeploymentHelper(BaseAPIKeyValidator):timeout=timeout,execution_id=execution_id,queue=CeleryQueue.CELERY_API_DEPLOYMENTS,+ use_file_history=use_file_history,)result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_iddiff --git a/backend/api/serializers.py b/backend/api/serializers.pyindex 88af3b740..35216e85f 100644--- a/backend/api/serializers.py+++ b/backend/api/serializers.py@@ -6,6 +6,7 @@ from api.models import APIDeployment, APIKeyfrom django.core.validators import RegexValidatorfrom pipeline.models import Pipelinefrom rest_framework.serializers import (+ BooleanField,CharField,IntegerField,JSONField,@@ -80,26 +81,22 @@ class APIKeySerializer(AuditSerializer):class ExecutionRequestSerializer(Serializer):- """Execution request serializer- timeout: 0: maximum value of timeout, -1: async execution+ """Execution request serializer.++ Attributes:+ timeout (int): Timeout for the API deployment, maximum value can be 300s.+ If -1 it corresponds to async execution. Defaults to -1+ include_metadata (bool): Flag to include metadata in API response+ use_file_history (bool): Flag to use FileHistory to save and retrieve+ responses quickly. This is undocumented to the user and can be+ helpful for demos."""timeout = IntegerField(min_value=-1, max_value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC, default=-1)-- def validate_timeout(self, value: Any) -> int:- if not isinstance(value, int):- raise ValidationError("timeout must be a integer.")- if value == 0:- value = ApiExecution.MAXIMUM_TIMEOUT_IN_SEC- return value-- def get_timeout(self, validated_data: dict[str, Union[int, None]]) -> int:- value = validated_data.get(ApiExecution.TIMEOUT_FORM_DATA, -1)- if not isinstance(value, int):- raise ValidationError("timeout must be a integer.")- return value+ include_metadata = BooleanField(default=False)+ use_file_history = BooleanField(default=False)class APIDeploymentListSerializer(ModelSerializer):diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 27a01dd23..2a90eaf14 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -155,20 +155,24 @@ class DestinationConnector(BaseConnector):file_name: str,file_hash: FileHash,workflow: Workflow,+ input_file_path: str,error: Optional[str] = None,- input_file_path: Optional[str] = None,+ use_file_history: bool = True,) -> None:"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_typeresult: Optional[str] = None- meta_data: Optional[str] = None+ metadata: Optional[str] = Noneif error:if connection_type == WorkflowEndpoint.ConnectionType.API:self._handle_api_result(file_name=file_name, error=error, result=result)return- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash.file_hash- )++ file_history = None+ if use_file_history:+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash.file_hash+ )if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:@@ -181,9 +185,9 @@ class DestinationConnector(BaseConnector):self.insert_into_db(input_file_path=input_file_path)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)- meta_data = self.get_metadata(file_history)+ exec_metadata = self.get_metadata(file_history)self._handle_api_result(- file_name=file_name, error=error, result=result, meta_data=meta_data+ file_name=file_name, error=error, result=result, metadata=exec_metadata)elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:self._push_data_to_queue(file_name, workflow, input_file_path)@@ -191,13 +195,14 @@ class DestinationConnector(BaseConnector):self.execution_service.publish_log(message=f"File '{file_name}' processed successfully")- if not file_history:++ if use_file_history and not file_history:FileHistoryHelper.create_file_history(cache_key=file_hash.file_hash,workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,- metadata=meta_data,+ metadata=metadata,file_name=file_name,)@@ -325,7 +330,7 @@ class DestinationConnector(BaseConnector):file_name: str,error: Optional[str] = None,result: Optional[str] = None,- meta_data: Optional[dict[str, Any]] = None,+ metadata: Optional[dict[str, Any]] = None,) -> None:"""Handle the API result.@@ -351,7 +356,7 @@ class DestinationConnector(BaseConnector):{"status": ApiDeploymentResultStatus.SUCCESS,"result": result,- "metadata": meta_data,+ "metadata": metadata,})else:diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 7c855e4a0..6792d7a4a 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -486,9 +486,7 @@ class SourceConnector(BaseConnector):shutil.copyfile(input_file_path, infile_path)shutil.copyfile(input_file_path, source_path)- def add_file_to_volume(- self, input_file_path: str, file_hash: FileHash- ) -> tuple[str, str]:+ def add_file_to_volume(self, input_file_path: str, file_hash: FileHash) -> str:"""Add input file to execution directory.Args:@@ -556,7 +554,11 @@ class SourceConnector(BaseConnector):@classmethoddef add_input_file_to_api_storage(- cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]+ cls,+ workflow_id: str,+ execution_id: str,+ file_objs: list[UploadedFile],+ use_file_history: bool = False,) -> dict[str, FileHash]:"""Add input file to api storage.@@ -564,6 +566,8 @@ class SourceConnector(BaseConnector):workflow_id (str): UUID of the worklfowexecution_id (str): UUID of the executionfile_objs (list[UploadedFile]): List of uploaded files+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to Falsereturns:dict[str, FileHash]: Dict containing file name and its corresponding hash"""@@ -583,9 +587,12 @@ class SourceConnector(BaseConnector):f.write(buffer)file_hash = cls.hash_str(buffer)connection_type = WorkflowEndpoint.ConnectionType.API- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash- )++ file_history = None+ if use_file_history:+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash+ )is_executed = (True if file_history and file_history.is_completed() else False)diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex ef6a6a7c1..e04f0bb8c 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -34,6 +34,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):scheduled: bool = False,mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,workflow_execution: Optional[WorkflowExecution] = None,+ use_file_history: bool = True,) -> None:tool_instances_as_dto = []for tool_instance in tool_instances:@@ -95,6 +96,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.project_settings = project_settingsself.pipeline_id = pipeline_idself.execution_id = str(workflow_execution.id)+ self.use_file_history = use_file_historylogger.info(f"Executing for Pipeline ID: {pipeline_id}, "f"workflow ID: {self.workflow_id}, execution ID: {self.execution_id}, "diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex e9a420c5c..400bd723a 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -92,6 +92,7 @@ class WorkflowHelper:scheduled: bool,execution_mode: tuple[str, str],workflow_execution: Optional[WorkflowExecution],+ use_file_history: bool = True, # Will be False for API deployment alone) -> WorkflowExecutionServiceHelper:workflow_execution_service = WorkflowExecutionServiceHelper(organization_id=organization_id,@@ -102,6 +103,7 @@ class WorkflowHelper:scheduled=scheduled,mode=execution_mode,workflow_execution=workflow_execution,+ use_file_history=use_file_history,)workflow_execution_service.build()return workflow_execution_service@@ -218,8 +220,9 @@ class WorkflowHelper:file_name=file_name,file_hash=file_hash,workflow=workflow,- error=error,input_file_path=input_file,+ error=error,+ use_file_history=execution_service.use_file_history,)execution_service.publish_update_log(LogState.SUCCESS,@@ -249,6 +252,7 @@ class WorkflowHelper:single_step: bool = False,workflow_execution: Optional[WorkflowExecution] = None,execution_mode: Optional[tuple[str, str]] = None,+ use_file_history: bool = True,) -> ExecutionResponse:tool_instances: list[ToolInstance] = (ToolInstanceHelper.get_tool_instances_by_workflow(@@ -267,6 +271,7 @@ class WorkflowHelper:scheduled=scheduled,execution_mode=execution_mode,workflow_execution=workflow_execution,+ use_file_history=use_file_history,)execution_id = execution_service.execution_idsource = SourceConnector(@@ -391,6 +396,7 @@ class WorkflowHelper:timeout: int = -1,pipeline_id: Optional[str] = None,queue: Optional[str] = None,+ use_file_history: bool = True,) -> ExecutionResponse:"""Adding a workflow to the queue for execution.@@ -399,6 +405,9 @@ class WorkflowHelper:execution_id (str): Execution IDtimeout (int): Celery timeout (timeout -1 : async execution)pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.+ queue (Optional[str]): Name of the celery queue to push into+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to TrueReturns:ExecutionResponse: Existing status of execution@@ -421,6 +430,7 @@ class WorkflowHelper:"execution_mode": None,"pipeline_id": pipeline_id,"log_events_id": log_events_id,+ "use_file_history": use_file_history,},queue=queue,)@@ -482,6 +492,7 @@ class WorkflowHelper:scheduled: bool = False,execution_mode: Optional[tuple[str, str]] = None,pipeline_id: Optional[str] = None,+ use_file_history: bool = True,**kwargs: dict[str, Any],) -> Optional[list[Any]]:"""Asynchronous Execution By celery.@@ -495,6 +506,8 @@ class WorkflowHelper:execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution ModeDefaults to Nonepipeline_id (Optional[str], optional): Id of pipeline. Defaults to None+ use_file_history (bool): Use FileHistory table to return results on already+ processed files. Defaults to TrueKwargs:log_events_id (str): Session ID of the user,@@ -539,6 +552,7 @@ class WorkflowHelper:workflow_execution=workflow_execution,execution_mode=execution_mode,hash_values_of_files=hash_values,+ use_file_history=use_file_history,)except Exception as error:error_message = traceback.format_exc()diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 77075ad1e..0f601bb06 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -346,7 +346,7 @@ class WorkflowExecutionService:with self.redis_con as r:r.delete(self.execution_id)- log_message = f"Executing workflow {self.workflow_id} Done."+ log_message = f"Executed workflow {self.workflow_id} successfully."self.publish_log(log_message)def _handle_execution_type(self, execution_type: ExecutionType) -> None: