LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Un 2271 optimization of workflow file listing (#1221)
ali648ee6d15b31
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.pyindex cb28b1332..b2613ba73 100644--- a/backend/connector_v2/models.py+++ b/backend/connector_v2/models.py@@ -15,7 +15,6 @@ from utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,)-from workflow_manager.workflow_v2.models import Workflowfrom backend.constants import FieldLengthConstants as FLC@@ -43,7 +42,7 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):max_length=CONNECTOR_NAME_SIZE, null=False, blank=False)workflow = models.ForeignKey(- Workflow,+ "workflow_v2.Workflow",on_delete=models.CASCADE,related_name="connector_workflow",null=False,diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 4105e0912..1ef31ee93 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -135,6 +135,43 @@ class DestinationConnector(BaseConnector):):raise DestinationConnectorNotConfigured()+ # Validate database connection if it's a database destination+ if connection_type == WorkflowEndpoint.ConnectionType.DATABASE and connector:+ try:+ # Get database class and test connection+ db_class = DatabaseUtils.get_db_class(+ connector_id=connector.connector_id,+ connector_settings=connector.metadata,+ )+ engine = db_class.get_engine()+ if hasattr(engine, "close"):+ engine.close()+ except Exception as e:+ logger.error(f"Database connection failed: {str(e)}")+ raise++ def _handle_hitl(+ self,+ file_name: str,+ file_hash: FileHash,+ workflow: Workflow,+ input_file_path: str,+ file_execution_id: str,+ ) -> bool:+ """Handles HITL processing, returning True if data was pushed to the queue."""+ execution_result = self.get_result()+ if WorkflowUtil.validate_db_rule(+ execution_result, workflow, file_hash.file_destination+ ):+ self._push_data_to_queue(+ file_name=file_name,+ workflow=workflow,+ input_file_path=input_file_path,+ file_execution_id=file_execution_id,+ )+ return True+ return False+def _push_data_to_queue(self,file_name: str,@@ -180,17 +217,13 @@ class DestinationConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:- result = self.get_result(file_history)- if WorkflowUtil.validate_db_rule(- result, workflow, file_hash.file_destination+ if not self._handle_hitl(+ file_name=file_name,+ file_hash=file_hash,+ workflow=workflow,+ input_file_path=input_file_path,+ file_execution_id=file_execution_id,):- self._push_data_to_queue(- file_name,- workflow,- input_file_path,- file_execution_id,- )- else:self.insert_into_db(input_file_path=input_file_path)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)@@ -212,7 +245,7 @@ class DestinationConnector(BaseConnector):if use_file_history and not file_history:FileHistoryHelper.create_file_history(- cache_key=file_hash.file_hash,+ file_hash=file_hash,workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,@@ -316,29 +349,53 @@ class DestinationConnector(BaseConnector):file_path=input_file_path,execution_id=self.execution_id,)- db_class = DatabaseUtils.get_db_class(- connector_id=connector_instance.connector_id,- connector_settings=connector_settings,- )- engine = db_class.get_engine()- DatabaseUtils.create_table_if_not_exists(- db_class=db_class,- engine=engine,- table_name=table_name,- database_entry=values,- )- sql_columns_and_values = DatabaseUtils.get_sql_query_data(- conn_cls=db_class,- table_name=table_name,- values=values,- )- DatabaseUtils.execute_write_query(- db_class=db_class,- engine=engine,- table_name=table_name,- sql_keys=list(sql_columns_and_values.keys()),- sql_values=list(sql_columns_and_values.values()),- )+ engine = None+ try:+ db_class = DatabaseUtils.get_db_class(+ connector_id=connector_instance.connector_id,+ connector_settings=connector_settings,+ )+ engine = db_class.get_engine()+ DatabaseUtils.create_table_if_not_exists(+ db_class=db_class,+ engine=engine,+ table_name=table_name,+ database_entry=values,+ )+ sql_columns_and_values = DatabaseUtils.get_sql_query_data(+ conn_cls=db_class,+ table_name=table_name,+ values=values,+ )+ DatabaseUtils.execute_write_query(+ db_class=db_class,+ engine=engine,+ table_name=table_name,+ sql_keys=list(sql_columns_and_values.keys()),+ sql_values=list(sql_columns_and_values.values()),+ )+ except ConnectorError as e:+ error_msg = f"Database connection failed for {input_file_path}: {str(e)}"+ logger.error(error_msg)+ raise+ except Exception as e:+ error_msg = (+ f"Failed to insert data into database for {input_file_path}: {str(e)}"+ )+ logger.error(error_msg)+ raise+ finally:+ self._close_engine(engine, input_file_path)++ def _close_engine(self, engine: Any, input_file_path: str) -> None:+ """Safely close database engine."""+ if engine:+ try:+ engine.close()+ except Exception as e:+ logger.error(+ f"Failed to close database engine for {input_file_path}: {str(e)}"+ )def _handle_api_result(self,@@ -488,6 +545,14 @@ class DestinationConnector(BaseConnector):file_storage = file_system.get_file_storage()if file_storage.exists(self.execution_dir):file_storage.rm(self.execution_dir, recursive=True)++ def delete_execution_and_api_storage_dir(self) -> None:+ """Delete the execution and api storage directories.++ Returns:+ None+ """+ self.delete_execution_directory()self.delete_api_storage_dir(self.workflow_id, self.execution_id)@classmethoddiff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.pyindex 6c701f2e9..9576f376a 100644--- a/backend/workflow_manager/endpoint_v2/dto.py+++ b/backend/workflow_manager/endpoint_v2/dto.py@@ -6,11 +6,13 @@ from typing import Any, Optional@dataclassclass FileHash:file_path: str- file_hash: strfile_name: strsource_connection_type: str+ file_hash: Optional[str] = Nonefile_size: Optional[int] = None+ provider_file_uuid: Optional[str] = Nonemime_type: Optional[str] = None+ fs_metadata: Optional[dict[str, Any]] = Nonefile_destination: Optional[tuple[str, str]] = (None # To which destination this file wants to go for MRQ percentage)@@ -26,6 +28,8 @@ class FileHash:"is_executed": self.is_executed,"file_size": self.file_size,"mime_type": self.mime_type,+ "provider_file_uuid": self.provider_file_uuid,+ "fs_metadata": self.fs_metadata,}@staticmethoddiff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 619621b7f..75fb52139 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -2,16 +2,16 @@ import fnmatchimport loggingimport osimport shutil-from hashlib import md5, sha256+from hashlib import sha256from io import BytesIOfrom itertools import islicefrom typing import Any, Optionalimport fsspec-import magicfrom connector_processor.constants import ConnectorKeysfrom connector_v2.models import ConnectorInstancefrom django.core.files.uploadedfile import UploadedFile+from unstract.sdk.file_storage import FileStoragefrom unstract.workflow_execution.enums import LogStatefrom utils.user_context import UserContextfrom workflow_manager.endpoint_v2.base_connector import BaseConnector@@ -25,8 +25,6 @@ from workflow_manager.endpoint_v2.constants import ()from workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.endpoint_v2.exceptions import (- FileHashMismatched,- FileHashNotFound,InvalidInputDirectory,InvalidSourceConnectionType,MissingSourceConnectionType,@@ -37,8 +35,10 @@ from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper+from workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemfrom unstract.filesystem import FileStorageType, FileSystemlogger = logging.getLogger(__name__)@@ -59,6 +59,8 @@ class SourceConnector(BaseConnector):workflow (Workflow): The workflow associated with the source connector."""+ READ_CHUNK_SIZE = 4194304 # Chunk size for reading files+def __init__(self,workflow: Workflow,@@ -208,7 +210,7 @@ class SourceConnector(BaseConnector):)logger.debug(f"Listing files from: {input_directory}")matched_files, count = self._get_matched_files(- source_fs_fsspec, input_directory, patterns, recursive, limit+ source_fs, input_directory, patterns, recursive, limit)self.publish_user_sys_log(f"Matched '{count}' files from '{input_directory}'"@@ -277,7 +279,7 @@ class SourceConnector(BaseConnector):def _get_matched_files(self,- source_fs: Any,+ source_fs: UnstractFileSystem,input_directory: str,patterns: list[str],recursive: bool,@@ -305,28 +307,23 @@ class SourceConnector(BaseConnector):matched_files: dict[str, FileHash] = {}count = 0max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1-- for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):+ fs_fsspec = source_fs.get_fsspec_fs()+ for root, dirs, files in fs_fsspec.walk(input_directory, maxdepth=max_depth):for file in files:if count >= limit:breakif self._should_process_file(file, patterns):file_path = str(os.path.join(root, file))- file_content, file_size = self.get_file_content(- input_file_path=file_path- )if self._is_new_file(file_path=file_path,- file_content=file_content,workflow=self.endpoint.workflow,+ source_fs=source_fs,):matched_files[file_path] = self._create_file_hash(file_path=file_path,- file_content=file_content,- file_size=file_size,+ source_fs=source_fs,)count += 1-return matched_files, countdef _should_process_file(self, file: str, patterns: list[str]) -> bool:@@ -375,14 +372,10 @@ class SourceConnector(BaseConnector):return Truedef _is_new_file(- self, file_path: str, file_content: bytes, workflow: Workflow+ self, file_path: str, workflow: Workflow, source_fs: UnstractFileSystem) -> bool:"""Check if the file is new or already processed."""- file_hash = self.get_file_content_hash(file_content)- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash- )-+ file_history = self._get_file_history(workflow, source_fs, file_path)# In case of ETL pipelines, its necessary to skip files which have# already been processedif (@@ -398,21 +391,69 @@ class SourceConnector(BaseConnector):return True+ def _get_file_history(+ self, workflow: Workflow, source_fs: UnstractFileSystem, file_path: str+ ) -> Optional[FileHistory]:+ """Retrieve file history using provider UUID or legacy cache key."""++ provider_file_uuid = source_fs.get_file_system_uuid(file_path)++ if provider_file_uuid:+ logger.info(+ f"Checking file history for provider UUID: {provider_file_uuid}"+ )+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, provider_file_uuid=provider_file_uuid+ )++ if file_history:+ return file_history # Early return if history exists++ # The provider_file_uuid was recently integrated,+ # so we also check the cache_key for backward compatibility.+ # This ensures older files without a provider UUID+ # can still be identified.+ # In the future, this check can be removed as file history validation+ # is already handled during file execution.+ file_content_hash = self.get_file_content_hash(source_fs, file_path)+ logger.info(+ f"Checking file history for legacy cache key: {file_content_hash}"+ )+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_content_hash+ )++ if file_history and file_history.is_completed():+ file_history.update(provider_file_uuid=provider_file_uuid)++ # Fallback for connectors that do not support provider_file_uuid+ else:+ file_content_hash = self.get_file_content_hash(source_fs, file_path)+ logger.info(+ f"Checking file history for legacy cache key: {file_content_hash}"+ )+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_content_hash+ )++ return file_history+def _create_file_hash(- self, file_path: str, file_content: bytes, file_size: Optional[int]+ self, file_path: str, source_fs: UnstractFileSystem) -> FileHash:"""Create a FileHash object for the matched file."""file_name = os.path.basename(file_path)- file_hash = self.get_file_content_hash(file_content)+ provider_file_uuid = source_fs.get_file_system_uuid(file_path)+ file_size = source_fs.get_file_size(file_path)+ fs_metadata = source_fs.get_file_metadata(file_path)connection_type = self.endpoint.connection_type- file_type = magic.from_buffer(file_content, mime=True)return FileHash(file_path=file_path,source_connection_type=connection_type,file_name=file_name,- file_hash=file_hash,file_size=file_size,- mime_type=file_type,+ provider_file_uuid=provider_file_uuid,+ fs_metadata=fs_metadata,)def list_files_from_source(@@ -433,76 +474,25 @@ class SourceConnector(BaseConnector):return self.list_file_from_api_storage(file_hashes)raise InvalidSourceConnectionType()- @classmethod- def hash_str(cls, string_to_hash: Any, hash_method: str = "sha256") -> str:- """Computes the hash for a given input string.-- Useful to hash strings needed for caching and other purposes.- Hash method defaults to "md5"-- Args:- string_to_hash (str): String to be hashed- hash_method (str): Hash hash_method to use, supported ones- - "md5"-- Returns:- str: Hashed string- """- if hash_method == "md5":- if isinstance(string_to_hash, bytes):- return str(md5(string_to_hash).hexdigest())- return str(md5(string_to_hash.encode()).hexdigest())- elif hash_method == "sha256":- if isinstance(string_to_hash, (bytes, bytearray)):- return str(sha256(string_to_hash).hexdigest())- return str(sha256(string_to_hash.encode()).hexdigest())- else:- raise ValueError(f"Unsupported hash_method: {hash_method}")-- def get_file_content(- self, input_file_path: str, chunk_size: int = 4194304- ) -> tuple[bytes, Optional[int]]:- """Read the content of a file from a remote filesystem in chunks.-- Args:- input_file_path (str): The path of the input file.- chunk_size (int): The size of the chunks to read at a time- (default is 4194304 bytes = 4 MB).-- Returns:- tuple[bytes, int]:- A tuple containing the content of the file as bytes and- its size in bytes.- """- connector: ConnectorInstance = self.endpoint.connector_instance- connector_settings: dict[str, Any] = connector.connector_metadata- source_fs = self.get_fsspec(- settings=connector_settings, connector_id=connector.connector_id- )-- # Get file size- file_metadata = source_fs.stat(input_file_path)- file_size: Optional[int] = file_metadata.get("size")- if file_size is None:- logger.warning(f"File size for {input_file_path} could not be determined.")-- file_content = bytearray() # Use bytearray for efficient byte concatenation- with source_fs.open(input_file_path, "rb") as remote_file:- while chunk := remote_file.read(chunk_size):- file_content.extend(chunk)-- return bytes(file_content), file_size-- def get_file_content_hash(self, file_content: bytes) -> str:+ def get_file_content_hash(+ self, source_fs: UnstractFileSystem, file_path: str+ ) -> str:"""Generate a hash value from the file content.Args:- file_content (bytes): The content of the file.+ source_fs (UnstractFileSystem): The file system object used for+ reading the file.+ file_path (str): The path of the file.Returns:str: The hash value of the file content."""- return self.hash_str(file_content)+ file_content_hash = sha256()+ source = source_fs.get_fsspec_fs()+ with source.open(file_path, "rb") as remote_file:+ while chunk := remote_file.read(self.READ_CHUNK_SIZE):+ file_content_hash.update(chunk)+ return file_content_hash.hexdigest()def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:"""Copy the source file to the infile directory.@@ -514,11 +504,11 @@ class SourceConnector(BaseConnector):shutil.copyfile(source_file_path, infile_path)logger.info(f"File copied from {source_file_path} to {infile_path}")- def add_input_from_connector_to_volume(self, input_file_path: str) -> str:+ def add_input_from_connector_to_volume(self, file_hash: FileHash) -> str:"""Add input file to execution directory.Args:- input_file_path (str): The path of the input file.+ file_hash (FileHash): The file hash object.Returns:str: The hash value of the file content.@@ -526,32 +516,49 @@ class SourceConnector(BaseConnector):Raises:FileHashNotFound: If the hash value of the file content is not found."""- source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)- infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)- source_file = f"file://{source_file_path}"+ input_file_path = file_hash.file_path+ source_file_path = self.source_file+ infile_path = self.infile+ connector: ConnectorInstance = self.endpoint.connector_instance+ connector_settings: dict[str, Any] = connector.connector_metadata+ source_fs = self.get_fsspec(+ settings=connector_settings, connector_id=connector.connector_id+ )- # Get file content and hash value- file_content, _ = self.get_file_content(input_file_path)- hash_value_of_file_content = self.get_file_content_hash(file_content)+ workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ workflow_file_storage = workflow_file_system.get_file_storage()- logger.info(- f"hash_value_of_file {source_file} is : {hash_value_of_file_content}"- )+ first_iteration = True+ input_log = ""+ file_hash = sha256()+ with source_fs.open(input_file_path, "rb") as remote_file:+ while chunk := remote_file.read(self.READ_CHUNK_SIZE):+ file_hash.update(chunk)+ if first_iteration:+ input_log = (+ chunk[:500].decode("utf-8", errors="replace") + "...(truncated)"+ )+ first_iteration = False+ # write the chunk in to execution directory+ workflow_file_storage.write(+ path=source_file_path, mode="ab", data=chunk+ )+ workflow_file_storage.write(path=infile_path, mode="ab", data=chunk)- input_log = (- file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"- )+ # publish input file content+ # TODO: Consider removing this since the input is not extracted text.+ # This function is typically relevant for extracted text content,+ # may not be necessary for PDFs, images, or other non-text formats.self.publish_input_file_content(input_file_path, input_log)-- file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)- file_storage = file_system.get_file_storage()- file_storage.write(path=source_file_path, mode="wb", data=file_content)- file_storage.write(path=infile_path, mode="wb", data=file_content)+ hash_value_of_file_content = file_hash.hexdigest()+ logger.info(+ f"hash_value_of_file {source_file_path} is : {hash_value_of_file_content}"+ )logger.info(f"{input_file_path} is added to execution directory")return hash_value_of_file_content- def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:+ def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:"""Add input file to execution directory from api storage."""infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)@@ -560,22 +567,23 @@ class SourceConnector(BaseConnector):api_file_storage = api_file_system.get_file_storage()workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)workflow_file_storage = workflow_file_system.get_file_storage()- self._copy_file_to_destination(+ file_content_hash = self._copy_file_to_destination(source_storage=api_file_storage,destination_storage=workflow_file_storage,source_path=input_file_path,destination_paths=[infile_path, source_path],)+ logger.info(f"File {input_file_path} added to execution directory")+ return file_content_hash# TODO: replace it with method from SDK Utilsdef _copy_file_to_destination(self,- source_storage: Any,- destination_storage: Any,+ source_storage: FileStorage,+ destination_storage: FileStorage,source_path: str,destination_paths: list[str],- chunk_size: int = 4194304,- ) -> None:+ ) -> str:"""Copy a file from a source storage to one or more paths in adestination storage.@@ -592,49 +600,43 @@ class SourceConnector(BaseConnector):source_path (str): The path of the file in the source storage.destination_paths (list[str]): A list of paths where the file will becopied in the destination storage.- chunk_size (int, optional): The number of bytes to read per chunk.- (default is 4194304 bytes = 4 MB).+ Returns:+ str: The SHA-256 hash of the file content."""seek_position = 0 # Start from the beginning- end_of_file = False-+ file_content_hash = sha256()# Loop to read and write in chunks until the end of the file- while not end_of_file:- # Read a chunk from the source file- chunk = source_storage.read(- path=source_path,- mode="rb",- seek_position=seek_position,- length=chunk_size,- )- # Check if the end of the file has been reached- if not chunk:- end_of_file = True- else:- # Write the chunk to each destination path- for destination_file in destination_paths:- destination_storage.write(- path=destination_file,- mode="ab",- seek_position=seek_position,- data=chunk,- )+ while chunk := source_storage.read(+ path=source_path,+ mode="rb",+ seek_position=seek_position,+ length=self.READ_CHUNK_SIZE,+ ):+ file_content_hash.update(chunk)+ # Write the chunk to each destination path+ for destination_file in destination_paths:+ destination_storage.write(+ path=destination_file,+ mode="ab",+ data=chunk,+ )- # Update the seek position- seek_position += len(chunk)+ # Update the seek position+ seek_position += len(chunk)+ return file_content_hash.hexdigest()def add_file_to_volume(self,- input_file_path: str,workflow_file_execution: WorkflowFileExecution,- tags: list[str] = [],+ tags: list[str],+ file_hash: FileHash,) -> str:"""Add input file to execution directory.Args:- input_file_path (str): source fileworkflow_file_execution: WorkflowFileExecution modeltags (list[str]): Tag names associated with the workflow execution.+ file_hash: FileHash modelRaises:InvalidSource: _description_@@ -643,18 +645,15 @@ class SourceConnector(BaseConnector):str: file_name"""connection_type = self.endpoint.connection_type- file_name = os.path.basename(input_file_path)+ input_file_path = file_hash.file_pathif connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:file_content_hash = self.add_input_from_connector_to_volume(- input_file_path=input_file_path,+ file_hash=file_hash)- if file_content_hash != workflow_file_execution.file_hash:- raise FileHashMismatched()elif connection_type == WorkflowEndpoint.ConnectionType.API:- self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)- if file_name != workflow_file_execution.file_name:- raise FileHashNotFound()- file_content_hash = workflow_file_execution.file_hash+ file_content_hash = self.add_input_from_api_storage_to_volume(+ input_file_path=input_file_path+ )else:raise InvalidSourceConnectionType()@@ -664,7 +663,7 @@ class SourceConnector(BaseConnector):source_hash=file_content_hash,tags=tags,)- return file_name+ return file_content_hashdef handle_final_result(self,@@ -729,11 +728,11 @@ class SourceConnector(BaseConnector):file_system = FileSystem(FileStorageType.API_EXECUTION)file_storage = file_system.get_file_storage()- buffer = bytearray()- for chunk in file.chunks():- buffer.extend(chunk)- file_storage.write(path=destination_path, mode="wb", data=buffer)- file_hash = cls.hash_str(buffer)+ file_hash = sha256()+ for chunk in file.chunks(chunk_size=cls.READ_CHUNK_SIZE):+ file_hash.update(chunk)+ file_storage.write(path=destination_path, mode="ab", data=chunk)+ file_hash = file_hash.hexdigest()connection_type = WorkflowEndpoint.ConnectionType.APIfile_history = Nonediff --git a/backend/workflow_manager/file_execution/migrations/0005_workflowfileexecution_fs_metadata_and_more.py b/backend/workflow_manager/file_execution/migrations/0005_workflowfileexecution_fs_metadata_and_more.pynew file mode 100644index 000000000..9ff5b951c--- /dev/null+++ b/backend/workflow_manager/file_execution/migrations/0005_workflowfileexecution_fs_metadata_and_more.py@@ -0,0 +1,51 @@+# Generated by Django 4.2.1 on 2025-04-02 13:22++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("file_execution", "0004_alter_workflowfileexecution_status"),+ ]++ operations = [+ migrations.AddField(+ model_name="workflowfileexecution",+ name="fs_metadata",+ field=models.JSONField(+ db_comment="Complete metadata of the file retrieved from the file system.",+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="workflowfileexecution",+ name="provider_file_uuid",+ field=models.CharField(+ db_comment="Unique identifier assigned by the file storage provider",+ max_length=64,+ null=True,+ ),+ ),+ migrations.AlterField(+ model_name="workflowfileexecution",+ name="file_hash",+ field=models.CharField(+ db_comment="Hash of the file content", max_length=64, null=True+ ),+ ),+ migrations.AddIndex(+ model_name="workflowfileexecution",+ index=models.Index(+ fields=["workflow_execution", "provider_file_uuid"],+ name="workflow_exec_p_uuid_idx",+ ),+ ),+ migrations.AddConstraint(+ model_name="workflowfileexecution",+ constraint=models.UniqueConstraint(+ fields=("workflow_execution", "provider_file_uuid", "file_path"),+ name="unique_workflow_provider_uuid_path",+ ),+ ),+ ]diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.pyindex 9c58a9158..20850909c 100644--- a/backend/workflow_manager/file_execution/models.py+++ b/backend/workflow_manager/file_execution/models.py@@ -1,12 +1,13 @@import uuidfrom datetime import timedelta-from typing import Optional+from typing import Anyfrom django.db import modelsfrom utils.common_utils import CommonUtilsfrom utils.models.base_model import BaseModel+from workflow_manager.endpoint_v2.dto import FileHash+from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.workflow_v2.enums import ExecutionStatus-from workflow_manager.workflow_v2.models.execution import WorkflowExecutionFILE_NAME_LENGTH = 255FILE_PATH_LENGTH = 255@@ -17,46 +18,59 @@ MIME_TYPE_LENGTH = 128class WorkflowFileExecutionManager(models.Manager):def get_or_create_file_execution(self,- workflow_execution: WorkflowExecution,- file_name: str,- file_size: int,- file_hash: str,- file_path: Optional[str] = None,- mime_type: Optional[str] = None,+ workflow_execution: Any,+ file_hash: FileHash,+ connection_type: WorkflowEndpoint.ConnectionType,) -> "WorkflowFileExecution":"""Retrieves or creates a new input file record for a workflow execution.Args:- workflow_execution: The `WorkflowExecution` object associated with this file- file_name: The name of the input file- file_size: The size of the file in bytes- file_hash: The hash of the file content- file_path: (Optional) The full path of the input file- mime_type: (Optional) MIME type of the file+ workflow_execution: The `WorkflowExecution` object+ associated with this file.+ file_hash: The `FileHash` object containing file metadata.+ file_path: (Optional) The full path of the input file.- return:- The `WorkflowFileExecution` object+ Returns:+ The `WorkflowFileExecution` object."""- execution_file: WorkflowFileExecution- execution_file, is_created = self.get_or_create(- workflow_execution=workflow_execution,- file_hash=file_hash,- file_path=file_path,- )+ is_api = connection_type == WorkflowEndpoint.ConnectionType.API+ # Determine file path based on connection type+ execution_file_path = file_hash.file_path if not is_api else None++ lookup_fields = {+ "workflow_execution": workflow_execution,+ "file_path": execution_file_path,+ }++ if file_hash.file_hash:+ lookup_fields["file_hash"] = file_hash.file_hash+ elif file_hash.provider_file_uuid:+ lookup_fields["provider_file_uuid"] = file_hash.provider_file_uuid++ execution_file, is_created = self.get_or_create(**lookup_fields)if is_created:- execution_file.file_name = file_name- execution_file.file_size = file_size- execution_file.mime_type = mime_type- execution_file.save()+ self._update_execution_file(execution_file, file_hash)+return execution_file+ def _update_execution_file(+ self, execution_file: "WorkflowFileExecution", file_hash: FileHash+ ) -> None:+ """Updates the attributes of a newly created WorkflowFileExecution object."""+ execution_file.file_name = file_hash.file_name+ execution_file.file_size = file_hash.file_size+ execution_file.mime_type = file_hash.mime_type+ execution_file.provider_file_uuid = file_hash.provider_file_uuid+ execution_file.fs_metadata = file_hash.fs_metadata+ execution_file.save()+class WorkflowFileExecution(BaseModel):id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)workflow_execution = models.ForeignKey(- WorkflowExecution,+ "workflow_v2.WorkflowExecution",on_delete=models.CASCADE,db_index=True,editable=False,@@ -73,7 +87,16 @@ class WorkflowFileExecution(BaseModel):null=True, db_comment="Size of the file in bytes")file_hash = models.CharField(- max_length=HASH_LENGTH, db_comment="Hash of the file content"+ max_length=HASH_LENGTH, null=True, db_comment="Hash of the file content"+ )+ provider_file_uuid = models.CharField(+ max_length=HASH_LENGTH,+ null=True,+ db_comment="Unique identifier assigned by the file storage provider",+ )+ fs_metadata = models.JSONField(+ null=True,+ db_comment="Complete metadata of the file retrieved from the file system.",)mime_type = models.CharField(max_length=MIME_TYPE_LENGTH,@@ -167,10 +190,52 @@ class WorkflowFileExecution(BaseModel):fields=["workflow_execution", "file_hash"],name="workflow_file_hash_idx",),+ models.Index(+ fields=["workflow_execution", "provider_file_uuid"],+ name="workflow_exec_p_uuid_idx",+ ),]constraints = [models.UniqueConstraint(fields=["workflow_execution", "file_hash", "file_path"],name="unique_workflow_file_hash_path",),+ models.UniqueConstraint(+ fields=["workflow_execution", "provider_file_uuid", "file_path"],+ name="unique_workflow_provider_uuid_path",+ ),]++ @property+ def is_completed(self) -> bool:+ """Check if the execution status is completed.++ Returns:+ bool: True if the execution status is completed, False otherwise.+ """+ return self.status is not None and self.status == ExecutionStatus.COMPLETED++ def update(+ self,+ file_hash: str = None,+ fs_metadata: dict[str, Any] = None,+ ) -> None:+ """+ Updates the file execution details.++ Args:+ file_hash: (Optional) Hash of the file content++ Returns:+ None+ """+ update_fields = []++ if file_hash is not None:+ self.file_hash = file_hash+ update_fields.append("file_hash")+ if fs_metadata is not None:+ self.fs_metadata = fs_metadata+ update_fields.append("fs_metadata")+ if update_fields: # Save only if there's an actual update+ self.save(update_fields=update_fields)diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex 820146004..7d9c26872 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -2,6 +2,7 @@ import loggingfrom typing import Any, Optionalfrom django.db.utils import IntegrityError+from workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -14,36 +15,41 @@ class FileHistoryHelper:@staticmethoddef get_file_history(- workflow: Workflow, cache_key: Optional[str] = None+ workflow: Workflow,+ cache_key: Optional[str] = None,+ provider_file_uuid: Optional[str] = None,) -> Optional[FileHistory]:"""Retrieve a file history record based on the cache key.Args:cache_key (Optional[str]): The cache key to search for.+ provider_file_uuid (Optional[str]): The provider file UUID to search for.Returns:Optional[FileHistory]: The matching file history record, if found."""- if not cache_key:++ if not cache_key and not provider_file_uuid:return Nonetry:- file_history: FileHistory = FileHistory.objects.get(- cache_key=cache_key, workflow=workflow- )+ if not cache_key:+ return FileHistory.objects.get(+ provider_file_uuid=provider_file_uuid, workflow=workflow+ )+ return FileHistory.objects.get(cache_key=cache_key, workflow=workflow)except FileHistory.DoesNotExist:return None- return file_history@staticmethoddef create_file_history(- cache_key: str,+ file_hash: FileHash,workflow: Workflow,status: ExecutionStatus,result: Any,metadata: Any,error: Optional[str] = None,file_name: Optional[str] = None,- ) -> FileHistory:+ ) -> None:"""Create a new file history record.Args:@@ -51,31 +57,23 @@ class FileHistoryHelper:workflow (Workflow): The associated workflow.status (ExecutionStatus): The execution status.result (Any): The result from the execution.-- Returns:- FileHistory: The newly created file history record."""try:- file_history: FileHistory = FileHistory.objects.create(+ FileHistory.objects.create(workflow=workflow,- cache_key=cache_key,+ cache_key=file_hash.file_hash,+ provider_file_uuid=file_hash.provider_file_uuid,status=status,result=str(result),metadata=str(metadata),error=str(error) if error else "",)- except IntegrityError:+ except IntegrityError as e:# TODO: Need to find why duplicate insert is cominglogger.warning(- "Trying to insert duplication data for filename %s for workflow %s",- file_name,- workflow,+ f"Trying to insert duplication data for filename {file_name} "+ f"for workflow {workflow}. Error: {str(e)} with metadata {metadata}",)- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=cache_key- )-- return file_history@staticmethoddef clear_history_for_workflow(diff --git a/backend/workflow_manager/workflow_v2/migrations/0011_remove_filehistory_unique_workflow_cachekey_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0011_remove_filehistory_unique_workflow_cachekey_and_more.pynew file mode 100644index 000000000..a4ed9c412--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0011_remove_filehistory_unique_workflow_cachekey_and_more.py@@ -0,0 +1,42 @@+# Generated by Django 4.2.1 on 2025-04-02 13:22++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow_v2", "0010_executionlog_wf_execution_and_more"),+ ]++ operations = [+ migrations.RemoveConstraint(+ model_name="filehistory",+ name="unique_workflow_cacheKey",+ ),+ migrations.AddField(+ model_name="filehistory",+ name="provider_file_uuid",+ field=models.CharField(+ db_comment="Unique identifier assigned by the file storage provider",+ max_length=64,+ null=True,+ ),+ ),+ migrations.AddConstraint(+ model_name="filehistory",+ constraint=models.UniqueConstraint(+ condition=models.Q(("cache_key__isnull", False)),+ fields=("workflow", "cache_key"),+ name="unique_workflow_cacheKey",+ ),+ ),+ migrations.AddConstraint(+ model_name="filehistory",+ constraint=models.UniqueConstraint(+ condition=models.Q(("provider_file_uuid__isnull", False)),+ fields=("workflow", "provider_file_uuid"),+ name="unique_workflow_providerFileUUID",+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex dcf3043e3..67a099b1e 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -104,21 +104,21 @@ class WorkflowExecution(BaseModel):return list(self.tags.values_list("name", flat=True))@property- def workflow(self) -> Optional[str]:- """Obtains the workflow associated to this execution."""+ def workflow_name(self) -> Optional[str]:+ """Obtains the workflow's name associated to this execution."""try:- return Workflow.objects.get(id=self.workflow_id)+ return Workflow.objects.get(id=self.workflow_id).workflow_nameexcept ObjectDoesNotExist:logger.warning(- f"Expected workflow '{self.workflow_id}' to exist but missing"+ f"Expected workflow ID '{self.workflow_id}' to exist but missing")return None@property- def workflow_name(self) -> Optional[str]:- """Obtains the workflow's name associated to this execution."""+ def workflow(self) -> Optional[Workflow]:+ """Obtains the workflow associated to this execution."""try:- return Workflow.objects.get(id=self.workflow_id).workflow_name+ return Workflow.objects.get(id=self.workflow_id)except ObjectDoesNotExist:logger.warning(f"Expected workflow ID '{self.workflow_id}' to exist but missing"diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.pyindex 0093d6529..acecfe66a 100644--- a/backend/workflow_manager/workflow_v2/models/execution_log.py+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py@@ -2,8 +2,6 @@ import uuidfrom django.db import modelsfrom utils.models.base_model import BaseModel-from workflow_manager.file_execution.models import WorkflowFileExecution-from workflow_manager.workflow_v2.models import WorkflowExecutionclass ExecutionLog(BaseModel):@@ -17,7 +15,7 @@ class ExecutionLog(BaseModel):null=True,)wf_execution = models.ForeignKey(- WorkflowExecution,+ "workflow_v2.WorkflowExecution",on_delete=models.CASCADE,editable=False,db_comment="Foreign key from WorkflowExecution model",@@ -26,7 +24,7 @@ class ExecutionLog(BaseModel):blank=True,)file_execution = models.ForeignKey(- WorkflowFileExecution,+ "file_execution.WorkflowFileExecution",on_delete=models.CASCADE,db_index=True,editable=False,diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.pyindex afbf8587f..17a270228 100644--- a/backend/workflow_manager/workflow_v2/models/file_history.py+++ b/backend/workflow_manager/workflow_v2/models/file_history.py@@ -1,4 +1,5 @@import uuid+from typing import Optionalfrom django.db import modelsfrom utils.models.base_model import BaseModel@@ -22,6 +23,11 @@ class FileHistory(BaseModel):max_length=HASH_LENGTH,db_comment="Hash value of file contents, WF and tool modified times",)+ provider_file_uuid = models.CharField(+ max_length=HASH_LENGTH,+ null=True,+ db_comment="Unique identifier assigned by the file storage provider",+ )workflow = models.ForeignKey(Workflow,on_delete=models.CASCADE,@@ -47,5 +53,32 @@ class FileHistory(BaseModel):models.UniqueConstraint(fields=["workflow", "cache_key"],name="unique_workflow_cacheKey",+ condition=models.Q(cache_key__isnull=False),+ ),+ models.UniqueConstraint(+ fields=["workflow", "provider_file_uuid"],+ name="unique_workflow_providerFileUUID",+ condition=models.Q(provider_file_uuid__isnull=False),),]++ def update(+ self,+ provider_file_uuid: Optional[str] = None,+ ) -> None:+ """+ Updates the file execution details.++ Args:+ provider_file_uuid: (Optional) UUID of the file in the storage provider++ Returns:+ None+ """+ update_fields = []++ if provider_file_uuid is not None:+ self.provider_file_uuid = provider_file_uuid+ update_fields.append("provider_file_uuid")+ if update_fields: # Save only if there's an actual update+ self.save(update_fields=update_fields)diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 1417ff2ec..fffbed3cf 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -27,7 +27,6 @@ from utils.local_context import StateStorefrom utils.user_context import UserContextfrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.dto import FileHash-from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.endpoint_v2.source import SourceConnectorfrom workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.workflow_v2.constants import (@@ -42,6 +41,7 @@ from workflow_manager.workflow_v2.exceptions import (InvalidRequest,TaskDoesNotExistError,WorkflowDoesNotExistError,+ WorkflowExecutionError,WorkflowExecutionNotExist,)from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper@@ -109,25 +109,6 @@ class WorkflowHelper:workflow_execution_service.build()return workflow_execution_service- @staticmethod- def _get_or_create_workflow_execution_file(- execution_service: WorkflowExecutionServiceHelper,- file_hash: FileHash,- source: SourceConnector,- ) -> WorkflowFileExecution:- is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API- # Determine file path based on connection type- execution_file_path = file_hash.file_path if not is_api else None- # Create or retrieve the workflow execution file- return WorkflowFileExecution.objects.get_or_create_file_execution(- workflow_execution=execution_service.workflow_execution,- file_name=file_hash.file_name,- file_hash=file_hash.file_hash,- file_path=execution_file_path,- file_size=file_hash.file_size,- mime_type=file_hash.mime_type,- )-@classmethoddef process_input_files(cls,@@ -146,16 +127,12 @@ class WorkflowHelper:execution_service.update_execution(ExecutionStatus.EXECUTING, increment_attempt=True)- if total_files > 0:- q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)+ q_file_no_list = (+ WorkflowUtil.get_q_no_list(workflow, total_files) if total_files > 0 else []+ )for index, (file_name, file_hash) in enumerate(input_files.items()):# Get workflow execution file- workflow_execution_file = cls._get_or_create_workflow_execution_file(- execution_service=execution_service,- file_hash=file_hash,- source=source,- )file_number = index + 1file_hash = WorkflowUtil.add_file_destination_filehash(file_number,@@ -173,44 +150,31 @@ class WorkflowHelper:execution_service=execution_service,single_step=single_step,file_hash=file_hash,- workflow_file_execution=workflow_execution_file,)if error:failed_files += 1- workflow_execution_file.update_status(- status=ExecutionStatus.ERROR,- execution_error=error,- )else:successful_files += 1- workflow_execution_file.update_status(ExecutionStatus.COMPLETED)except StopExecution as e:execution_service.update_execution(ExecutionStatus.STOPPED, error=str(e))- workflow_execution_file.update_status(- status=ExecutionStatus.STOPPED, execution_error=str(e)- )breakexcept Exception as e:failed_files += 1error_message = f"Error processing file '{file_name}'. {e}"logger.error(error_message, stack_info=True, exc_info=True)- workflow_execution_file.update_status(- status=ExecutionStatus.ERROR,- execution_error=error_message,- )execution_service.publish_log(message=error_message, level=LogLevel.ERROR)# TODO: Store only generic WF errors here (concerning all failed files)# TODO: Review if we need partial success- if failed_files and failed_files >= total_files:- execution_service.update_execution(- ExecutionStatus.ERROR, error=error_message- )- else:- execution_service.update_execution(ExecutionStatus.COMPLETED)+ final_status = (+ ExecutionStatus.ERROR+ if total_files > 0 and failed_files >= total_files+ else ExecutionStatus.COMPLETED+ )+ execution_service.update_execution(final_status, error=error_message)workflow_execution = execution_service.get_execution_instance()@@ -222,8 +186,9 @@ class WorkflowHelper:return workflow_execution- @staticmethod+ @classmethoddef _process_file(+ cls,current_file_idx: int,total_files: int,input_file: str,@@ -233,18 +198,46 @@ class WorkflowHelper:execution_service: WorkflowExecutionServiceHelper,single_step: bool,file_hash: FileHash,- workflow_file_execution: WorkflowFileExecution,) -> Optional[str]:- error: Optional[str] = None++ # Delete previous execution directory to avoid conflicts+ destination.delete_execution_directory()+# Multiple run_ids are linked to an execution_id# Each run_id corresponds to workflow runs for a single file# It should e uuid of workflow_file_execution+ workflow_file_execution: WorkflowFileExecution = (+ WorkflowFileExecution.objects.get_or_create_file_execution(+ workflow_execution=execution_service.workflow_execution,+ file_hash=file_hash,+ connection_type=source.endpoint.connection_type,+ )+ )+ workflow_file_execution.update_status(status=ExecutionStatus.EXECUTING)file_execution_id = str(workflow_file_execution.id)- file_name = source.add_file_to_volume(- input_file_path=input_file,+ file_name = file_hash.file_name++ # This will add the file to the volume+ file_content_hash = source.add_file_to_volume(workflow_file_execution=workflow_file_execution,tags=execution_service.tags,+ file_hash=file_hash,)++ # Update file_hash after adding to volume+ file_hash.file_hash = file_content_hash+ workflow_file_execution.update(file_hash=file_content_hash)++ # Ensure no duplicate file processing+ if FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_content_hash+ ):+ logger.info(f"Skipping '{file_name}', already processed.")+ execution_service.publish_log(+ f"Skipping '{file_name}', already processed.", LogLevel.INFO+ )+ return None+try:execution_service.file_execution_id = file_execution_idexecution_service.initiate_tool_execution(@@ -259,35 +252,55 @@ class WorkflowHelper:workflow_file_execution=workflow_file_execution,)except StopExecution:+ workflow_file_execution.update_status(status=ExecutionStatus.STOPPED)raise+ except WorkflowExecutionError as e:+ error_message = f"Error processing file '{file_name}': {e}"+ execution_service.publish_log(error_message, level=LogLevel.ERROR)+ error = error_messageexcept Exception as e:- error = f"Error processing file '{os.path.basename(input_file)}'. {str(e)}"- execution_service.publish_log(error, level=LogLevel.ERROR)+ error_message = f"Unexpected error processing file '{file_name}': {str(e)}"+ logger.error(error_message, stack_info=True, exc_info=True)+ execution_service.publish_log(error_message, level=LogLevel.ERROR)+ error = error_message# Handling error based on destination and continuing for other files+ else:+ error = Noneexecution_service.publish_update_log(LogState.RUNNING,f"Processing output for {file_name}",LogComponent.DESTINATION,)- destination.handle_output(- file_name=file_name,- file_hash=file_hash,- workflow=workflow,- input_file_path=input_file,- error=error,- use_file_history=execution_service.use_file_history,- file_execution_id=file_execution_id,- )-- execution_service.log_total_cost_per_file(- run_id=file_execution_id, file_name=file_name- )-- execution_service.publish_update_log(- LogState.SUCCESS,- f"{file_name}'s output is processed successfully",- LogComponent.DESTINATION,- )+ try:+ destination.handle_output(+ file_name=file_name,+ file_hash=file_hash,+ workflow=workflow,+ input_file_path=input_file,+ error=error,+ use_file_history=execution_service.use_file_history,+ file_execution_id=file_execution_id,+ )+ except Exception as e:+ error_message = (+ f"Error during output processing for '{file_name}': {str(e)}"+ )+ logger.error(error_message, stack_info=True, exc_info=True)+ execution_service.publish_log(error_message, level=LogLevel.ERROR)+ error = error_message+ finally:+ execution_service.log_total_cost_per_file(+ run_id=file_execution_id, file_name=file_name+ )+ execution_service.publish_update_log(+ LogState.SUCCESS,+ f"{file_name}'s output is processed successfully",+ LogComponent.DESTINATION,+ )+ workflow_file_execution.update_status(+ status=ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED,+ execution_error=error if error else None,+ )return error@staticmethod@@ -344,11 +357,11 @@ class WorkflowHelper:execution_id=execution_id,execution_service=execution_service,)- # Validating endpoints- source.validate()- destination.validate()- # Execution Processtry:+ # Validating endpoints+ source.validate()+ destination.validate()+ # Execution Processinput_files, total_files = source.list_files_from_source(hash_values_of_files)@@ -387,7 +400,7 @@ class WorkflowHelper:finally:# TODO: Handle error gracefully during delete# Mark status as an ERROR correctly- destination.delete_execution_directory()+ destination.delete_execution_and_api_storage_dir()@staticmethoddef _update_pipeline_status(diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.pyindex a9bc4da2b..b63f09eb4 100644--- a/unstract/connectors/src/unstract/connectors/databases/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py@@ -111,3 +111,9 @@ class ColumnMissingException(UnstractDBConnectorException):f"as per the destination DB configuration.\n")super().__init__(detail=default_detail)+++class OperationalException(UnstractDBConnectorException):+ def __init__(self, detail: Any, database: str) -> None:+ default_detail = f"Error creating/writing to {database}. Operational error. "+ super().__init__(detail=default_detail)diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.pyindex 3c2b5c4bd..976b2dbd2 100644--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py@@ -9,6 +9,13 @@ from unstract.connectors.databases.unstract_db import UnstractDBclass PostgreSQL(UnstractDB, PsycoPgHandler):+ # Connection timeout settings (in seconds)+ CONNECT_TIMEOUT = 30 # Time to establish connection+ STATEMENT_TIMEOUT = 300 # Time for query execution (5 minutes)+ KEEPALIVE_IDLE = 30 # Time before sending keepalive+ KEEPALIVE_INTERVAL = 10 # Time between keepalive probes+ KEEPALIVE_COUNT = 3 # Number of keepalive failures before dropping+def __init__(self, settings: dict[str, Any]):super().__init__("PostgreSQL")@@ -60,17 +67,51 @@ class PostgreSQL(UnstractDB, PsycoPgHandler):return Truedef get_engine(self) -> connection:+ # Set timeouts via options+ timeout_options = (+ f"-c connect_timeout={self.CONNECT_TIMEOUT} "+ f"-c statement_timeout={self.STATEMENT_TIMEOUT * 1000} "+ f"-c tcp_keepalives_idle={self.KEEPALIVE_IDLE} "+ f"-c tcp_keepalives_interval={self.KEEPALIVE_INTERVAL} "+ f"-c tcp_keepalives_count={self.KEEPALIVE_COUNT}"+ )++ # Base connection parameters+ conn_params = {+ "keepalives": 1,+ "keepalives_idle": self.KEEPALIVE_IDLE,+ "keepalives_interval": self.KEEPALIVE_INTERVAL,+ "keepalives_count": self.KEEPALIVE_COUNT,+ "connect_timeout": self.CONNECT_TIMEOUT,+ "application_name": "unstract_connector",+ }++ # Determine SSL mode based on connection URL+ if self.connection_url and (+ "neon.tech" in self.connection_url or "amazonaws.com" in self.connection_url+ ):+ # Cloud hosted PostgreSQL (Neon, AWS RDS etc)+ conn_params.update({"sslmode": "verify-full", "sslrootcert": "system"})+ else:+ # Standard PostgreSQL - use basic SSL if available+ conn_params["sslmode"] = "prefer"+if self.connection_url:- con = psycopg2.connect(dsn=self.connection_url)+ conn_params.update({"dsn": self.connection_url, "options": timeout_options})+ con = psycopg2.connect(**conn_params)else:- con = psycopg2.connect(- host=self.host,- port=self.port,- database=self.database,- user=self.user,- password=self.password,- options=f"-c search_path={self.schema}",+ conn_params.update(+ {+ "host": self.host,+ "port": self.port,+ "database": self.database,+ "user": self.user,+ "password": self.password,+ "options": f"{timeout_options} -c search_path={self.schema}",+ })+ con = psycopg2.connect(**conn_params)+return condef execute_query(diff --git a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.pyindex 34b089cd3..08bae5564 100644--- a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py+++ b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py@@ -8,6 +8,7 @@ from unstract.connectors.databases.exceptions import (FeatureNotSupportedException,InvalidSchemaException,InvalidSyntaxException,+ OperationalException,UnderfinedTableException,ValueTooLongException,)@@ -62,3 +63,6 @@ class PsycoPgHandler:schema=schema,table_name=table_name,) from e+ except PsycopgError.OperationalError as e:+ logger.error(f"Operational error in creating/inserting data: {e.pgerror}")+ raise OperationalException(detail=e.pgerror, database=database) from ediff --git a/unstract/connectors/src/unstract/connectors/exceptions.py b/unstract/connectors/src/unstract/connectors/exceptions.pyindex cfeeb6f4c..827da3b8e 100644--- a/unstract/connectors/src/unstract/connectors/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/exceptions.py@@ -55,3 +55,11 @@ class PermissionDeniedError(ConnectorError):Args:ConnectorError (Class): Inherits class ConnectorError"""+++class FileHashNotFound(ConnectorError):+ """Handles file hash not found error from fs connector.++ Args:+ ConnectorError (Class): Inherits class ConnectorError+ """diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.pyindex e4cc47ab1..4419b9324 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py@@ -1,6 +1,6 @@import loggingimport os-from typing import Any+from typing import Any, Optionalimport azure.core.exceptions as AzureExceptionfrom adlfs import AzureBlobFileSystem@@ -67,6 +67,25 @@ class AzureCloudStorageFS(UnstractFileSystem):def get_fsspec_fs(self) -> AzureBlobFileSystem:return self.azure_fs+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ # Extracts content_md5 (Bytearray) for Azure Blob Storage+ content_md5 = metadata.get("content_settings", {}).get("content_md5")+ if content_md5:+ return content_md5.hex()+ logger.error(+ f"[Azure Blob Storage] File hash not found for the metadata: {metadata}"+ )+ return None+def test_credentials(self) -> bool:"""To test credentials for Azure Cloud Storage."""try:diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.pyindex 2989e7916..133641280 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py@@ -2,7 +2,7 @@ import jsonimport loggingimport osfrom json import JSONDecodeError-from typing import Any+from typing import Any, Optionalfrom boxfs import BoxFileSystemfrom boxsdk import JWTAuth@@ -110,6 +110,19 @@ class BoxFS(UnstractFileSystem):def get_fsspec_fs(self) -> BoxFileSystem:return self.box_fs+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ logger.error(f"[Box] File hash not found for the metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""To test credentials for the Box connector."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.pyindex 025ede70c..07569898c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py@@ -1,7 +1,8 @@+import base64import jsonimport loggingimport os-from typing import Any+from typing import Any, Optionalfrom gcsfs import GCSFileSystem@@ -61,6 +62,23 @@ class GoogleCloudStorageFS(UnstractFileSystem):def get_fsspec_fs(self) -> GCSFileSystem:return self.gcs_fs+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ # Extracts md5Hash (Base64) for GCS+ file_hash = metadata.get("md5Hash")+ if file_hash:+ return base64.b64decode(file_hash).hex()+ logger.error(f"[GCS] File hash not found for the metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""To test credentials for Google Cloud Storage."""try:diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.pyindex 1743246dd..649dfb8c6 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py@@ -2,7 +2,7 @@ import jsonimport loggingimport osfrom pathlib import Path-from typing import Any+from typing import Any, Optionalimport google.api_core.exceptions as GoogleApiExceptionfrom oauth2client.client import OAuth2Credentials@@ -98,6 +98,19 @@ class GoogleDriveFS(UnstractFileSystem):def get_fsspec_fs(self) -> GDriveFileSystem:return self.drive+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ logger.error(f"[Google Drive] File hash not found for the metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""To test credentials for Google Drive."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.pyindex bf0a29dd6..b5e859209 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py@@ -1,6 +1,6 @@import loggingimport os-from typing import Any+from typing import Any, Optionalimport aiohttpfrom fsspec.implementations.http import HTTPFileSystem@@ -69,6 +69,19 @@ class HttpFS(UnstractFileSystem):def get_fsspec_fs(self) -> HTTPFileSystem:return self.http_fs+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ logger.error(f"[HTTP] File hash not found for the metadata: {metadata}")+ return None+def test_credentials(self) -> bool:"""To test credentials for HTTP(S)."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.pyindex 783803fe3..ce15a2c44 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py@@ -59,6 +59,19 @@ class LocalStorageFS(UnstractFileSystem):def get_fsspec_fs(self) -> Any:return self.local+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ logger.error(f"[LocalStorage] File hash not found for the metadata: {metadata}")+ return None+def test_credentials(self, *args, **kwargs) -> bool: # type:ignore"""To test credentials for LocalStorage."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex 7f99a6711..2c51ee589 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -1,6 +1,6 @@import loggingimport os-from typing import Any+from typing import Any, Optionalfrom s3fs.core import S3FileSystem@@ -70,6 +70,30 @@ class MinioFS(UnstractFileSystem):def can_read() -> bool:return True+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ # Extracts ETag for MinIO+ file_hash = metadata.get("ETag")+ if file_hash:+ file_hash = file_hash.strip('"')+ if "-" in file_hash:+ logger.warning(+ f"[S3/MinIO] Multipart upload detected. ETag may not be an "+ f"MD5 hash. Full metadata: {metadata}"+ )+ return None+ return file_hash+ logger.error(f"[MinIO] File hash not found for the metadata: {metadata}")+ return None+def get_fsspec_fs(self) -> S3FileSystem:return self.s3diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.pyindex feef4d62a..c8b9d290d 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py@@ -1,6 +1,6 @@import loggingimport os-from typing import Any+from typing import Any, Optionalfrom fsspec.implementations.sftp import SFTPFileSystem@@ -65,6 +65,19 @@ class SftpFS(UnstractFileSystem):def get_fsspec_fs(self) -> SFTPFileSystem:return self.sftp_fs+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ logger.error(f"[SFTP] File hash not found for the metadata: {metadata}")+ return None+# TODO: Check if this method can be removed, and use it from parent class# (class UnstractFileSystem)@staticmethoddiff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.pyindex b2162420c..fcaf3409c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py@@ -1,7 +1,9 @@+import base64import loggingimport osfrom abc import ABC, abstractmethod-from typing import Any+from datetime import date, datetime+from typing import Any, Optionalfrom fsspec import AbstractFileSystem@@ -75,11 +77,121 @@ class UnstractFileSystem(UnstractConnector, ABC):"""Override to test credentials for a connector."""pass+ @abstractmethod+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ This method must be implemented in subclasses.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ pass+@staticmethoddef get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:"""Override to get root dir of a connector."""return f"{input_dir.strip('/')}/"+ def get_file_size(self, file_path: str) -> Optional[int]:+ """Get the size of a file."""+ fs_fsspec = self.get_fsspec_fs()+ metadata = fs_fsspec.stat(file_path)+ if not metadata or not isinstance(metadata, dict):+ return None+ return metadata.get("size")++ def _serialize_metadata_value(self, value: Any, depth: int = 0) -> Any:+ """+ Recursively serialize metadata values to ensure JSON compatibility.++ Args:+ value: Any value that needs to be serialized+ depth: Current recursion depth (default: 0)++ Returns:+ JSON serializable version of the value+ """+ # Prevent infinite recursion+ if depth > 10: # Max recursion depth+ return str(value)++ if value is None:+ return None+ elif isinstance(value, (str, int, float, bool)):+ return value+ elif isinstance(value, (datetime, date)):+ return value.isoformat()+ elif isinstance(value, (bytes, bytearray)):+ return base64.b64encode(value).decode("utf-8")+ elif isinstance(value, dict):+ return {+ k: self._serialize_metadata_value(v, depth + 1)+ for k, v in value.items()+ }+ elif isinstance(value, (list, tuple)):+ return [self._serialize_metadata_value(v, depth + 1) for v in value]+ else:+ # For custom objects like ContentSettings, convert to dict if possible+ if hasattr(value, "__dict__"):+ try:+ return self._serialize_metadata_value(value.__dict__, depth + 1)+ except Exception:+ return str(value)+ # Last resort: convert to string+ return str(value)++ def get_file_metadata(self, file_path: str) -> dict[str, Any]:+ """+ Get the metadata of a file.++ Args:+ file_path (str): Path of the file.++ Returns:+ dict[str, Any]: Metadata of the file or empty dict+ if metadata cannot be retrieved.+ All values are guaranteed to be JSON serializable.+ """+ try:+ fs_fsspec = self.get_fsspec_fs()+ metadata = fs_fsspec.stat(file_path)+ if not metadata or not isinstance(metadata, dict):+ return {}++ # Recursively serialize all metadata values+ serialized_metadata = {}+ for key, value in metadata.items():+ serialized_metadata[key] = self._serialize_metadata_value(value)++ return serialized_metadata+ except Exception as e:+ logger.error(+ f"Error getting file system metadata for {file_path}: {str(e)}"+ )+ return {}++ def get_file_system_uuid(self, file_path: str) -> Optional[str]:+ """+ Get the UUID of a file.++ Args:+ file_path (str): Path of the file.++ Returns:+ Optional[str]: UUID of the file in hex format or None if not found.+ """+ fs_fsspec = self.get_fsspec_fs()+ metadata = fs_fsspec.stat(file_path)+ file_hash = self.extract_metadata_file_hash(metadata)+ if not file_hash:+ logger.error(f"File hash not found for {file_path}.")+ return file_hash+def create_dir_if_not_exists(self, input_dir: str) -> None:"""Method to create dir of a connector if not exists.diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.pyindex e7d5a7d37..8930855e5 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py@@ -1,6 +1,6 @@import loggingimport os-from typing import Any+from typing import Any, Optionalfrom dropbox.exceptions import ApiError as DropBoxApiErrorfrom dropbox.exceptions import DropboxException@@ -63,6 +63,24 @@ class DropboxFS(UnstractFileSystem):def get_fsspec_fs(self) -> DropboxDriveFileSystem:return self.dropbox_fs+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:+ """+ Extracts a unique file hash from metadata.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ Optional[str]: The file hash in hexadecimal format or None if not found.+ """+ file_hash = metadata.get("content_hash")+ if file_hash:+ return file_hash+ logger.error(+ f"[Dropbox] File hash (content_hash) not found for the metadata: {metadata}"+ )+ return None+def test_credentials(self) -> bool:"""To test credentials for Dropbox."""try: