LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
First version of file Execution model (#891)
ali3c491e51a07a
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex f8341cdaf..025da5eb5 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -232,6 +232,7 @@ v2_apps = ("connector_v2","adapter_processor_v2","file_management",+ "workflow_manager.file_execution","workflow_manager.endpoint_v2","workflow_manager.workflow_v2","tool_instance_v2",diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex e0c89e04c..4a4f17849 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -112,8 +112,13 @@ def execute_pipeline_task_v2(PipelineProcessor.update_pipeline(pipeline_id, Pipeline.PipelineStatus.INPROGRESS)+ # Mark the File in file history to avoid duplicate execution+ # only for ETL and TASK execution+ use_file_history: bool = Trueexecution_response = WorkflowHelper.complete_execution(- workflow=workflow, pipeline_id=pipeline_id+ workflow=workflow,+ pipeline_id=pipeline_id,+ use_file_history=use_file_history,)execution_response.remove_result_metadata_keys()logger.info(diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.pyindex 0e7aa6fd9..6c701f2e9 100644--- a/backend/workflow_manager/endpoint_v2/dto.py+++ b/backend/workflow_manager/endpoint_v2/dto.py@@ -9,6 +9,8 @@ class FileHash:file_hash: strfile_name: strsource_connection_type: str+ file_size: Optional[int] = None+ mime_type: Optional[str] = Nonefile_destination: Optional[tuple[str, str]] = (None # To which destination this file wants to go for MRQ percentage)@@ -22,6 +24,8 @@ class FileHash:"source_connection_type": self.source_connection_type,"file_destination": self.file_destination,"is_executed": self.is_executed,+ "file_size": self.file_size,+ "mime_type": self.mime_type,}@staticmethoddiff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex c004f3397..273962787 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -8,6 +8,7 @@ from itertools import islicefrom typing import Any, Optionalimport fsspec+import magicfrom connector_processor.constants import ConnectorKeysfrom connector_v2.models import ConnectorInstancefrom django.core.files.uploadedfile import UploadedFile@@ -313,10 +314,19 @@ class SourceConnector(BaseConnector):breakif self._should_process_file(file, patterns):file_path = str(os.path.join(root, file))+ file_content, file_size = self.get_file_content(+ input_file_path=file_path+ )if self._is_new_file(- file_path=file_path, workflow=self.endpoint.workflow+ file_path=file_path,+ file_content=file_content,+ workflow=self.endpoint.workflow,):- matched_files[file_path] = self._create_file_hash(file_path)+ matched_files[file_path] = self._create_file_hash(+ file_path=file_path,+ file_content=file_content,+ file_size=file_size,+ )count += 1return matched_files, count@@ -327,10 +337,11 @@ class SourceConnector(BaseConnector):fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns)- def _is_new_file(self, file_path: str, workflow: Workflow) -> bool:+ def _is_new_file(+ self, file_path: str, file_content: bytes, workflow: Workflow+ ) -> bool:"""Check if the file is new or already processed."""- file_content = self.get_file_content(input_file_path=file_path)- file_hash = self.get_hash_value(file_content)+ file_hash = self.get_file_content_hash(file_content)file_history = FileHistoryHelper.get_file_history(workflow=workflow, cache_key=file_hash)@@ -350,18 +361,21 @@ class SourceConnector(BaseConnector):return True- def _create_file_hash(self, file_path: str) -> FileHash:+ def _create_file_hash(+ self, file_path: str, file_content: bytes, file_size: Optional[int]+ ) -> FileHash:"""Create a FileHash object for the matched file."""file_name = os.path.basename(file_path)- file_content = self.get_file_content(input_file_path=file_path)- file_hash = self.get_hash_value(file_content)+ file_hash = self.get_file_content_hash(file_content)connection_type = self.endpoint.connection_type-+ file_type = magic.from_buffer(file_content, mime=True)return FileHash(file_path=file_path,source_connection_type=connection_type,file_name=file_name,file_hash=file_hash,+ file_size=file_size,+ mime_type=file_type,)def list_files_from_source(@@ -408,7 +422,9 @@ class SourceConnector(BaseConnector):else:raise ValueError(f"Unsupported hash_method: {hash_method}")- def get_file_content(self, input_file_path: str, chunk_size: int = 8192) -> bytes:+ def get_file_content(+ self, input_file_path: str, chunk_size: int = 8192+ ) -> tuple[bytes, Optional[int]]:"""Read the content of a file from a remote filesystem in chunks.Args:@@ -417,21 +433,30 @@ class SourceConnector(BaseConnector):(default is 8192 bytes).Returns:- bytes: The content of the file.+ tuple[bytes, int]:+ A tuple containing the content of the file as bytes and+ its size in bytes."""connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatasource_fs = self.get_fsspec(settings=connector_settings, connector_id=connector.connector_id)++ # Get file size+ file_metadata = source_fs.stat(input_file_path)+ file_size: Optional[int] = file_metadata.get("size")+ if file_size is None:+ logger.warning(f"File size for {input_file_path} could not be determined.")+file_content = bytearray() # Use bytearray for efficient byte concatenationwith source_fs.open(input_file_path, "rb") as remote_file:while chunk := remote_file.read(chunk_size):file_content.extend(chunk)- return bytes(file_content)+ return bytes(file_content), file_size- def get_hash_value(self, file_content: bytes) -> str:+ def get_file_content_hash(self, file_content: bytes) -> str:"""Generate a hash value from the file content.Args:@@ -469,8 +494,8 @@ class SourceConnector(BaseConnector):source_file = f"file://{source_file_path}"# Get file content and hash value- file_content = self.get_file_content(input_file_path)- hash_value_of_file_content = self.get_hash_value(file_content)+ file_content, _ = self.get_file_content(input_file_path)+ hash_value_of_file_content = self.get_file_content_hash(file_content)logger.info(f"hash_value_of_file {source_file} is : {hash_value_of_file_content}"@@ -695,6 +720,8 @@ class SourceConnector(BaseConnector):file_name=file_name,file_hash=file_hash,is_executed=is_executed,+ file_size=file.size,+ mime_type=file.content_type,)file_hashes.update({file_name: file_hash})return file_hashesdiff --git a/backend/workflow_manager/file_execution/__init__.py b/backend/workflow_manager/file_execution/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/workflow_manager/file_execution/apps.py b/backend/workflow_manager/file_execution/apps.pynew file mode 100644index 000000000..1beb32531--- /dev/null+++ b/backend/workflow_manager/file_execution/apps.py@@ -0,0 +1,6 @@+from django.apps import AppConfig+++class FileExecutionConfig(AppConfig):+ default_auto_field = "django.db.models.BigAutoField"+ name = "workflow_manager.file_execution"diff --git a/backend/workflow_manager/file_execution/migrations/0001_initial.py b/backend/workflow_manager/file_execution/migrations/0001_initial.pynew file mode 100644index 000000000..1b8a06d68--- /dev/null+++ b/backend/workflow_manager/file_execution/migrations/0001_initial.py@@ -0,0 +1,122 @@+# Generated by Django 4.2.1 on 2024-12-12 05:41++import uuid++import django.db.models.deletion+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ ("workflow_v2", "0002_remove_workflow_llm_response_and_more"),+ ]++ operations = [+ migrations.CreateModel(+ name="WorkflowFileExecution",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "file_name",+ models.CharField(db_comment="Name of the file", max_length=255),+ ),+ (+ "file_path",+ models.CharField(+ db_comment="Full Path of the file", max_length=255, null=True+ ),+ ),+ (+ "file_size",+ models.BigIntegerField(+ db_comment="Size of the file in bytes", null=True+ ),+ ),+ (+ "file_hash",+ models.CharField(+ db_comment="Hash of the file content", max_length=64+ ),+ ),+ (+ "mime_type",+ models.CharField(+ blank=True,+ db_comment="MIME type of the file",+ max_length=128,+ null=True,+ ),+ ),+ (+ "status",+ models.TextField(+ choices=[+ ("PENDING", "PENDING"),+ ("INITIATED", "INITIATED"),+ ("QUEUED", "QUEUED"),+ ("READY", "READY"),+ ("EXECUTING", "EXECUTING"),+ ("COMPLETED", "COMPLETED"),+ ("STOPPED", "STOPPED"),+ ("ERROR", "ERROR"),+ ],+ db_comment="Current status of the execution",+ ),+ ),+ (+ "execution_time",+ models.FloatField(+ db_comment="Execution time in seconds", null=True+ ),+ ),+ (+ "execution_error",+ models.TextField(+ blank=True,+ db_comment="Error message if execution failed",+ null=True,+ ),+ ),+ (+ "workflow_execution",+ models.ForeignKey(+ db_comment="Foreign key from WorkflowExecution model",+ editable=False,+ on_delete=django.db.models.deletion.CASCADE,+ to="workflow_v2.workflowexecution",+ ),+ ),+ ],+ options={+ "verbose_name": "Workflow File Execution",+ "verbose_name_plural": "Workflow File Executions",+ "db_table": "workflow_file_execution",+ "indexes": [+ models.Index(+ fields=["workflow_execution", "file_hash"],+ name="workflow_file_hash_idx",+ )+ ],+ },+ ),+ migrations.AddConstraint(+ model_name="workflowfileexecution",+ constraint=models.UniqueConstraint(+ fields=("workflow_execution", "file_hash", "file_path"),+ name="unique_workflow_file_hash_path",+ ),+ ),+ ]diff --git a/backend/workflow_manager/file_execution/migrations/__init__.py b/backend/workflow_manager/file_execution/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.pynew file mode 100644index 000000000..9f5edaaa1--- /dev/null+++ b/backend/workflow_manager/file_execution/models.py@@ -0,0 +1,139 @@+import uuid+from typing import Optional++from django.db import models+from utils.models.base_model import BaseModel+from workflow_manager.workflow_v2.enums import ExecutionStatus+from workflow_manager.workflow_v2.models.execution import WorkflowExecution++FILE_NAME_LENGTH = 255+FILE_PATH_LENGTH = 255+HASH_LENGTH = 64+MIME_TYPE_LENGTH = 128+++class WorkflowFileExecutionManager(models.Manager):+ def get_or_create_file_execution(+ self,+ workflow_execution: WorkflowExecution,+ file_name: str,+ file_size: int,+ file_hash: str,+ file_path: Optional[str] = None,+ mime_type: Optional[str] = None,+ ):+ """+ Retrieves or creates a new input file record for a workflow execution.++ Args:+ workflow_execution: The `WorkflowExecution` object associated with this file+ file_name: The name of the input file+ file_size: The size of the file in bytes+ file_hash: The hash of the file content+ file_path: (Optional) The full path of the input file+ mime_type: (Optional) MIME type of the file++ return:+ The `WorkflowFileExecution` object+ """+ execution_file: WorkflowFileExecution+ execution_file, is_created = self.get_or_create(+ workflow_execution=workflow_execution,+ file_hash=file_hash,+ file_path=file_path,+ )++ if is_created:+ execution_file.file_name = file_name+ execution_file.file_size = file_size+ execution_file.mime_type = mime_type+ execution_file.save()+ return execution_file+++class WorkflowFileExecution(BaseModel):+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ workflow_execution = models.ForeignKey(+ WorkflowExecution,+ on_delete=models.CASCADE,+ db_index=True,+ editable=False,+ db_comment="Foreign key from WorkflowExecution model",+ )+ file_name = models.CharField(+ max_length=FILE_NAME_LENGTH, db_comment="Name of the file"+ )+ file_path = models.CharField(+ max_length=FILE_PATH_LENGTH, null=True, db_comment="Full Path of the file"+ )+ file_size = models.BigIntegerField(+ null=True, db_comment="Size of the file in bytes"+ )+ file_hash = models.CharField(+ max_length=HASH_LENGTH, db_comment="Hash of the file content"+ )+ mime_type = models.CharField(+ max_length=MIME_TYPE_LENGTH,+ blank=True,+ null=True,+ db_comment="MIME type of the file",+ )+ status = models.TextField(+ choices=ExecutionStatus.choices(),+ db_comment="Current status of the execution",+ )+ execution_time = models.FloatField(+ null=True, db_comment="Execution time in seconds"+ )+ execution_error = models.TextField(+ blank=True, null=True, db_comment="Error message if execution failed"+ )++ # Custom manager+ objects = WorkflowFileExecutionManager()++ def __str__(self):+ return (+ f"WorkflowFileExecution: {self.file_name} "+ f"(WorkflowExecution: {self.workflow_execution})"+ )++ def update_status(+ self,+ status: ExecutionStatus,+ execution_time: int = 0,+ execution_error: str = None,+ ) -> None:+ """+ Updates the status and execution details of an input file.++ Args:+ execution_file: The `WorkflowExecutionFile` object to update+ status: The new status of the file+ execution_time: The execution time for processing the file+ execution_error: (Optional) Error message if processing failed++ return:+ The updated `WorkflowExecutionInputFile` object+ """+ self.status = status+ self.execution_time = execution_time+ self.execution_error = execution_error+ self.save()++ class Meta:+ verbose_name = "Workflow File Execution"+ verbose_name_plural = "Workflow File Executions"+ db_table = "workflow_file_execution"+ indexes = [+ models.Index(+ fields=["workflow_execution", "file_hash"],+ name="workflow_file_hash_idx",+ ),+ ]+ constraints = [+ models.UniqueConstraint(+ fields=["workflow_execution", "file_hash", "file_path"],+ name="unique_workflow_file_hash_path",+ ),+ ]diff --git a/backend/workflow_manager/file_execution/views.py b/backend/workflow_manager/file_execution/views.pynew file mode 100644index 000000000..60f00ef0e--- /dev/null+++ b/backend/workflow_manager/file_execution/views.py@@ -0,0 +1 @@+# Create your views here.diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 7d302f03a..994300d01 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -14,6 +14,7 @@ from unstract.workflow_execution.enums import ExecutionType, LogComponent, LogStfrom unstract.workflow_execution.exceptions import StopExecutionfrom utils.local_context import StateStorefrom utils.user_context import UserContext+from workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.workflow_v2.constants import WorkflowKeyfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.exceptions import WorkflowExecutionError@@ -84,6 +85,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):)workflow_execution.save()else:+ self.workflow_execution = workflow_executionself.execution_mode = workflow_execution.execution_modeself.execution_method = workflow_execution.execution_methodself.execution_type = workflow_execution.execution_type@@ -309,6 +311,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):run_id: str,file_name: str,single_step: bool,+ workflow_file_execution: WorkflowFileExecution,) -> None:"""Executes the input file.@@ -330,6 +333,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):message=f"{file_name} Sent for execution",component=LogComponent.SOURCE,)+ workflow_file_execution.update_status(ExecutionStatus.EXECUTING)+self.execute(run_id, file_name, single_step)self.publish_log(f"Tool executed successfully for '{file_name}'")self._handle_execution_type(execution_type)diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 3e1e4c80b..afb7962db 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -3,7 +3,6 @@ import loggingimport osimport tracebackfrom typing import Any, Optional-from uuid import uuid4from account_v2.constants import Commonfrom api_v2.models import APIDeployment@@ -27,7 +26,9 @@ from utils.local_context import StateStorefrom utils.user_context import UserContextfrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.dto import FileHash+from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.endpoint_v2.source import SourceConnector+from workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.workflow_v2.constants import (CeleryConfigurations,WorkflowErrors,@@ -109,7 +110,27 @@ class WorkflowHelper:return workflow_execution_service@staticmethod+ def _get_or_create_workflow_execution_file(+ execution_service: WorkflowExecutionServiceHelper,+ file_hash: FileHash,+ source: SourceConnector,+ ) -> WorkflowFileExecution:+ is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API+ # Determine file path based on connection type+ execution_file_path = file_hash.file_path if not is_api else None+ # Create or retrieve the workflow execution file+ return WorkflowFileExecution.objects.get_or_create_file_execution(+ workflow_execution=execution_service.workflow_execution,+ file_name=file_hash.file_name,+ file_hash=file_hash.file_hash,+ file_path=execution_file_path,+ file_size=file_hash.file_size,+ mime_type=file_hash.mime_type,+ )++ @classmethoddef process_input_files(+ cls,workflow: Workflow,source: SourceConnector,destination: DestinationConnector,@@ -129,6 +150,12 @@ class WorkflowHelper:q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)for index, (file_path, file_hash) in enumerate(input_files.items()):+ # Get workflow execution file+ workflow_execution_file = cls._get_or_create_workflow_execution_file(+ execution_service=execution_service,+ file_hash=file_hash,+ source=source,+ )file_number = index + 1file_hash = WorkflowUtil.add_file_destination_filehash(file_number,@@ -136,7 +163,7 @@ class WorkflowHelper:file_hash,)try:- error = WorkflowHelper.process_file(+ error = cls._process_file(current_file_idx=file_number,total_files=total_files,input_file=file_hash.file_path,@@ -146,20 +173,33 @@ class WorkflowHelper:execution_service=execution_service,single_step=single_step,file_hash=file_hash,+ workflow_file_execution=workflow_execution_file,)if error:failed_files += 1+ workflow_execution_file.update_status(+ status=ExecutionStatus.ERROR,+ execution_error=error,+ )else:successful_files += 1+ workflow_execution_file.update_status(ExecutionStatus.COMPLETED)except StopExecution as e:execution_service.update_execution(ExecutionStatus.STOPPED, error=str(e))+ workflow_execution_file.update_status(+ status=ExecutionStatus.STOPPED, execution_error=str(e)+ )breakexcept Exception as e:failed_files += 1error_message = f"Error processing file '{file_path}'. {e}"logger.error(error_message, stack_info=True, exc_info=True)+ workflow_execution_file.update_status(+ status=ExecutionStatus.ERROR,+ execution_error=error_message,+ )execution_service.publish_log(message=error_message, level=LogLevel.ERROR)@@ -178,7 +218,7 @@ class WorkflowHelper:return execution_service.get_execution_instance()@staticmethod- def process_file(+ def _process_file(current_file_idx: int,total_files: int,input_file: str,@@ -188,6 +228,7 @@ class WorkflowHelper:execution_service: WorkflowExecutionServiceHelper,single_step: bool,file_hash: FileHash,+ workflow_file_execution: WorkflowFileExecution,) -> Optional[str]:error: Optional[str] = Nonefile_name = source.add_file_to_volume(@@ -197,14 +238,17 @@ class WorkflowHelper:execution_service.initiate_tool_execution(current_file_idx, total_files, file_name, single_step)+ workflow_file_execution.update_status(status=ExecutionStatus.INITIATED)if not file_hash.is_executed:# Multiple run_ids are linked to an execution_id# Each run_id corresponds to workflow runs for a single file- run_id = str(uuid4())+ # It should e uuid of workflow_file_execution+ run_id = str(workflow_file_execution.id)execution_service.execute_input_file(run_id=run_id,file_name=file_name,single_step=single_step,+ workflow_file_execution=workflow_file_execution,)except StopExecution:raise