LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Optimization and fix of file execution - parallel execution (#1326)
ali472e725cc326
diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.pyindex b1a9aab4e..d2365a18b 100644--- a/backend/workflow_manager/endpoint_v2/dto.py+++ b/backend/workflow_manager/endpoint_v2/dto.py@@ -113,7 +113,7 @@ class DestinationConfig:@dataclassclass FileExecutionResult:file: str- file_execution_id: str+ file_execution_id: str | None = Nonestatus: str | None = Noneresult: str | None = Noneerror: str | None = Nonediff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 688e762a0..5cfb8a036 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -185,6 +185,7 @@ class SourceConnector(BaseConnector):source_fs_fsspec = source_fs.get_fsspec_fs()# Checking if folders exist at source before processing# TODO: Validate while receiving this input configuration as well+ valid_directories = []for input_directory in folders_to_process:# TODO: Move to connector class for better error handlingtry:@@ -193,6 +194,7 @@ class SourceConnector(BaseConnector):)if not source_fs_fsspec.isdir(input_directory):raise InvalidInputDirectory(dir=input_directory)+ valid_directories.append(input_directory)except Exception as e:msg = f"Error while validating path '{input_directory}'. {str(e)}"self.publish_user_sys_log(msg)@@ -202,14 +204,12 @@ class SourceConnector(BaseConnector):total_files_to_process = 0total_matched_files = {}+ unique_file_hashes: set[str] = set()- for input_directory in folders_to_process:- input_directory = source_fs.get_connector_root_dir(- input_dir=input_directory, root_path=root_dir_path- )+ for input_directory in valid_directories:logger.debug(f"Listing files from: {input_directory}")matched_files, count = self._get_matched_files(- source_fs, input_directory, patterns, recursive, limit+ source_fs, input_directory, patterns, recursive, limit, unique_file_hashes)self.publish_user_sys_log(f"Matched '{count}' files from '{input_directory}'")total_matched_files.update(matched_files)@@ -274,6 +274,7 @@ class SourceConnector(BaseConnector):patterns: list[str],recursive: bool,limit: int,+ unique_file_hashes: set[str],) -> tuple[dict[str, FileHash], int]:"""Get a dictionary of matched files based on patterns in a directory.@@ -298,24 +299,180 @@ class SourceConnector(BaseConnector):count = 0max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1fs_fsspec = source_fs.get_fsspec_fs()- for root, dirs, files in fs_fsspec.walk(input_directory, maxdepth=max_depth):- for file in files:- if count >= limit:- break- if self._should_process_file(file, patterns):- file_path = str(os.path.join(root, file))- if self._is_new_file(- file_path=file_path,- workflow=self.endpoint.workflow,- source_fs=source_fs,- ):- matched_files[file_path] = self._create_file_hash(- file_path=file_path,- source_fs=source_fs,- )- count += 1+ for root, dirs, _ in fs_fsspec.walk(input_directory, maxdepth=max_depth):+ try:+ fs_metadata_list: list[dict[str, Any]] = fs_fsspec.listdir(+ root+ ) # Single call for file system metadata+ except Exception as e:+ logger.warning(f"Failed to list directory from path: {root}, error: {e}")+ continue++ count = self._process_file_fs_directory(+ fs_metadata_list=fs_metadata_list,+ count=count,+ limit=limit,+ unique_file_hashes=unique_file_hashes,+ matched_files=matched_files,+ patterns=patterns,+ source_fs=source_fs,+ dirs=dirs,+ )return matched_files, count+ def _process_file_fs_directory(+ self,+ fs_metadata_list: list[dict[str, Any]],+ count: int,+ limit: int,+ unique_file_hashes: set[str],+ matched_files: dict[str, FileHash],+ patterns: list[str],+ source_fs: UnstractFileSystem,+ dirs: list[str],+ ) -> int:+ for fs_metadata in fs_metadata_list:+ if count >= limit:+ logger.info(+ f"[Matched Files] Maximum limit {limit} of files reached, count: {count}"+ )+ break++ file_path: str | None = fs_metadata.get("name")+ file_size = fs_metadata.get("size", 0)++ if not file_path or self._is_directory(+ source_fs, file_path, fs_metadata, dirs+ ):+ continue++ if self._should_skip_file(file_path, fs_metadata, patterns, source_fs):+ continue++ file_hash = self._create_file_hash(+ file_path=file_path,+ source_fs=source_fs,+ file_size=file_size,+ fs_metadata=fs_metadata,+ )++ # Skip duplicate files+ if self._is_duplicate(file_hash, unique_file_hashes):+ logger.info(+ f"[Matched Files] Skipping execution of duplicate file: {file_path}"+ )+ continue+ self._update_unique_file_hashes(file_hash, unique_file_hashes)++ matched_files[file_path] = file_hash+ count += 1+ return count++ def _should_skip_file(+ self,+ file_path: str,+ fs_metadata: dict[str, Any],+ patterns: list[str],+ source_fs: UnstractFileSystem,+ ) -> bool:+ """Check if the given file should be skipped.++ Args:+ file_path (str): The path of the file.+ fs_metadata (dict[str, Any]): The metadata of the file.+ patterns (list[str]): The patterns to match against file names.+ source_fs (UnstractFileSystem): The file system object used for+ reading the file.++ Returns:+ bool: True if the file should be skipped, False otherwise.+ """+ file_name = os.path.basename(file_path)+ return not self._should_process_file(+ file_name, patterns+ ) or not self._is_new_file(+ file_path=file_path,+ fs_metadata=fs_metadata,+ workflow=self.endpoint.workflow,+ source_fs=source_fs,+ )++ def _is_duplicate(self, file_hash: FileHash, unique_file_hashes: set[str]) -> bool:+ return (+ file_hash.provider_file_uuid in unique_file_hashes+ or file_hash.file_hash in unique_file_hashes+ )++ def _update_unique_file_hashes(+ self, file_hash: FileHash, unique_file_hashes: set[str]+ ) -> None:+ if file_hash.provider_file_uuid:+ unique_file_hashes.add(file_hash.provider_file_uuid)+ elif file_hash.file_hash:+ unique_file_hashes.add(file_hash.file_hash)++ def _is_directory(+ self,+ source_fs: UnstractFileSystem,+ file_path: str,+ metadata: dict[str, Any],+ dirs: list[str],+ ) -> bool:+ """Check if the given path is a directory.++ Args:+ source_fs (UnstractFileSystem): The file system object used for+ reading the file.+ file_path (str): The path of the file.+ metadata (dict[str, Any]): The metadata of the file.+ dirs (list[str]): The list of directories.++ Returns:+ bool: True if the file is a directory, False otherwise.+ """+ try:+ # Check if the path is a directory using metadata first.+ # Some connectors incorrectly label directories as files, so if metadata is inconclusive or fails,+ # fall back to other checks: directory listing, path suffix ("/"), or zero file size.+ if source_fs.is_dir_by_metadata(metadata):+ logger.info(+ f"[Directory Check] '{file_path}' identified as a directory via metadata."+ )+ return True+ except NotImplementedError:+ logger.debug(+ f"[Directory Check] Metadata-based check not implemented for '{file_path}'."+ )+ except Exception as e:+ logger.warning(+ f"[Directory Check] Error while checking metadata for '{file_path}': {e}"+ )++ file_name = os.path.basename(file_path)++ # Fallback 1: Check if the file is explicitly listed in directory entries+ if file_name in dirs:+ logger.info(+ f"[Directory Check] '{file_path}' identified as a directory via checking list of directories."+ )+ return True++ # Fallback 2: Check if the path ends with a slash+ if file_path.endswith("/"):+ logger.info(+ f"[Directory Check] '{file_path}' identified as a directory based on path suffix '/'."+ )+ return True++ # Fallback 3: Check if the file has size zero+ if source_fs.get_file_size(metadata=metadata) == 0:+ logger.info(+ f"[Directory Check] '{file_path}' identified as a directory based on file size = 0."+ )+ return True++ return False+def _should_process_file(self, file: str, patterns: list[str]) -> bool:"""Check if the file should be processed based on the patterns.@@ -360,10 +517,14 @@ class SourceConnector(BaseConnector):return Truedef _is_new_file(- self, file_path: str, workflow: Workflow, source_fs: UnstractFileSystem+ self,+ file_path: str,+ fs_metadata: dict[str, Any],+ workflow: Workflow,+ source_fs: UnstractFileSystem,) -> bool:"""Check if the file is new or already processed."""- file_history = self._get_file_history(workflow, source_fs, file_path)+ file_history = self._get_file_history(fs_metadata, workflow, source_fs, file_path)# In case of ETL pipelines, its necessary to skip files which have# already been processedif self.use_file_history and file_history and file_history.is_completed():@@ -376,57 +537,36 @@ class SourceConnector(BaseConnector):return Truedef _get_file_history(- self, workflow: Workflow, source_fs: UnstractFileSystem, file_path: str+ self,+ fs_metadata: dict[str, Any],+ workflow: Workflow,+ source_fs: UnstractFileSystem,+ file_path: str,) -> FileHistory | None:"""Retrieve file history using provider UUID or legacy cache key."""- provider_file_uuid = source_fs.get_file_system_uuid(file_path)+ provider_file_uuid = source_fs.get_file_system_uuid(file_path, fs_metadata)if provider_file_uuid:logger.info(f"Checking file history for provider UUID: {provider_file_uuid}")file_history = FileHistoryHelper.get_file_history(workflow=workflow, provider_file_uuid=provider_file_uuid)-- if file_history:- return file_history # Early return if history exists-- # The provider_file_uuid was recently integrated,- # so we also check the cache_key for backward compatibility.- # This ensures older files without a provider UUID- # can still be identified.- # In the future, this check can be removed as file history validation- # is already handled during file execution.- file_content_hash = self.get_file_content_hash(source_fs, file_path)- logger.info(- f"Checking file history for legacy cache key: {file_content_hash}"- )- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_content_hash- )-- if file_history and file_history.is_completed():- file_history.update(provider_file_uuid=provider_file_uuid)-- # Fallback for connectors that do not support provider_file_uuid- else:- file_content_hash = self.get_file_content_hash(source_fs, file_path)- logger.info(- f"Checking file history for legacy cache key: {file_content_hash}"- )- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_content_hash- )-- return file_history+ return file_history+ return Nonedef _create_file_hash(- self, file_path: str, source_fs: UnstractFileSystem+ self,+ file_path: str,+ source_fs: UnstractFileSystem,+ file_size: int,+ fs_metadata: dict[str, Any],) -> FileHash:"""Create a FileHash object for the matched file."""file_name = os.path.basename(file_path)- provider_file_uuid = source_fs.get_file_system_uuid(file_path)- file_size = source_fs.get_file_size(file_path)- fs_metadata = source_fs.get_file_metadata(file_path)+ provider_file_uuid = source_fs.get_file_system_uuid(+ file_path=file_path, metadata=fs_metadata+ )+ serialized_metadata = source_fs.serialize_metadata_value(value=fs_metadata)connection_type = self.endpoint.connection_typereturn FileHash(file_path=file_path,@@ -434,7 +574,7 @@ class SourceConnector(BaseConnector):file_name=file_name,file_size=file_size,provider_file_uuid=provider_file_uuid,- fs_metadata=fs_metadata,+ fs_metadata=serialized_metadata,)def list_files_from_source(@@ -501,6 +641,9 @@ class SourceConnector(BaseConnector):Raises:FileHashNotFound: If the hash value of the file content is not found."""+ logger.info(+ f"Adding input file from source connector to execution directory: {file_hash.file_name}"+ )input_file_path = file_hash.file_pathsource_file_path = self.source_fileinfile_path = self.infile@@ -545,6 +688,9 @@ class SourceConnector(BaseConnector):def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:"""Add input file to execution directory from api storage."""+ logger.info(+ f"Adding input file from api storage to execution directory: {input_file_path}"+ )source_file_path = self.source_fileinfile_path = self.infileif not source_file_path or not infile_path:@@ -631,6 +777,7 @@ class SourceConnector(BaseConnector):Returns:str: file_name"""+ logger.info(f"Adding input file to execution directory: {file_hash.file_name}")connection_type = self.endpoint.connection_typeinput_file_path = file_hash.file_pathif connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:@@ -641,6 +788,12 @@ class SourceConnector(BaseConnector):file_content_hash = self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)+ # Filehash created from Django requests files might be different+ # from file_content_hash created from file content here.+ # So use file_content_hash only when file_hash is not available+ file_content_hash = (+ file_hash.file_hash if file_hash.file_hash else file_content_hash+ )else:raise InvalidSourceConnectionType()@@ -709,6 +862,7 @@ class SourceConnector(BaseConnector):)workflow: Workflow = Workflow.objects.get(id=workflow_id)file_hashes: dict[str, FileHash] = {}+ unique_file_hashes: set[str] = set()for file in file_objs:file_name = file.namedestination_path = os.path.join(api_storage_dir, file_name)@@ -722,6 +876,14 @@ class SourceConnector(BaseConnector):file_hash = file_hash.hexdigest()connection_type = WorkflowEndpoint.ConnectionType.API+ # Skip duplicate files+ if file_hash in unique_file_hashes:+ logger.info(+ f"[Matched Files] Skipping execution of duplicate file: {file_name}"+ )+ continue+ unique_file_hashes.add(file_hash)+file_history = Noneif use_file_history:file_history = FileHistoryHelper.get_file_history(diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.pyindex 299fc75d2..854be42c6 100644--- a/backend/workflow_manager/utils/workflow_log.py+++ b/backend/workflow_manager/utils/workflow_log.py@@ -49,8 +49,19 @@ class WorkflowLog:self.publish_log(message, level=LogLevel.ERROR)logger.error(message, exc_info=True, stack_info=True)- def log_info(self, message: str) -> None:+ def log_info(self, logger: logging.Logger, message: str) -> None:+ """Publishes an info log message to the configured logger and to the+ websocket channel if the `execution_service` is configured.++ Args:+ logger (logging.Logger): The logger to use for logging.+ message (str): The log message to be published.++ Returns:+ None+ """self.publish_log(message, level=LogLevel.INFO)+ logger.info(message)def publish_update_log(self,diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.pyindex 91dcad76c..f6e0a0f1a 100644--- a/backend/workflow_manager/workflow_v2/dto.py+++ b/backend/workflow_manager/workflow_v2/dto.py@@ -186,6 +186,22 @@ class FileBatchData:return asdict(self)+@dataclass+class FileBatchResult:+ successful_files: int = 0+ failed_files: int = 0++ @property+ def total_files(self) -> int:+ return self.successful_files + self.failed_files++ def to_dict(self) -> dict[str, int]:+ return {+ "successful_files": self.successful_files,+ "failed_files": self.failed_files,+ }++@dataclassclass ToolExecutionResult:error: str | Nonediff --git a/backend/workflow_manager/workflow_v2/exceptions.py b/backend/workflow_manager/workflow_v2/exceptions.pyindex fa53b3023..6f52081f8 100644--- a/backend/workflow_manager/workflow_v2/exceptions.py+++ b/backend/workflow_manager/workflow_v2/exceptions.py@@ -59,3 +59,7 @@ class InternalException(APIException):class WorkflowExecutionNotExist(APIException):status_code = 404default_detail = "Workflow execution does not exist"+++class ExecutionContextInitializationError(APIException):+ default_detail = "Error initializing execution context."diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 3d04ab850..10fcb96b3 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -31,18 +31,21 @@ from workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.dto import (ExecutionContext,FileBatchData,+ FileBatchResult,FileData,FinalOutputResult,ToolExecutionResult,)from workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.exceptions import (+ ExecutionContextInitializationError,WorkflowDoesNotExistError,WorkflowExecutionError,)from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.execution import WorkflowExecution+from workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflowlogger = logging.getLogger(__name__)@@ -113,22 +116,23 @@ class FileExecutionTasks:- single_step: Whether to process in single step mode"""file_batch_data = FileBatchData.from_dict(file_batch_data)+ logger.debug(f"Batch data received: {file_batch_data}")+file_data = file_batch_data.file_datalogger.info(f"File processing context {file_data}")+ execution_id = file_data.execution_id+ workflow_id = file_data.workflow_idorganization_id = file_data.organization_id- StateStore.set(Account.ORGANIZATION_ID, organization_id)- # Use proper logger instead of print- logger.info(- f"Starting to process file batch for execution {file_data.execution_id}"- )- logger.debug(f"Batch data received: {file_batch_data}")+ StateStore.set(Account.ORGANIZATION_ID, organization_id)successful_files = 0failed_files = 0- execution_id = file_data.execution_id- workflow_id = file_data.workflow_id++ logger.info(+ f"Initializing file batch processing for execution {execution_id}, organization {organization_id}"+ )# Reconstruct necessary objectsworkflow = FileExecutionTasks.get_workflow_by_id(str(workflow_id))workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(@@ -140,13 +144,12 @@ class FileExecutionTasks:total_files = len(file_batch_data.files)q_file_no_list = set(file_data.q_file_no_list)- logger.info(- f"Processing {total_files} files of execution {execution_id} in a batch"- )+ logger.info(f"Processing {total_files} files of execution {execution_id}")for file_number, (file_name, file_hash_dict) in enumerate(file_batch_data.files, 1):+ logger.info(f"[{file_number}/{total_files}] Processing file '{file_name}'")file_hash = FileHash(file_path=file_hash_dict.get("file_path"),file_name=file_hash_dict.get("file_name"),@@ -165,6 +168,7 @@ class FileExecutionTasks:q_file_no_list,file_hash,)+ logger.info(f"File hash for file {file_name}: {file_hash.to_json()}")file_execution_result = FileExecutionTasks._process_file(current_file_idx=file_number,total_files=total_files,@@ -172,6 +176,7 @@ class FileExecutionTasks:file_hash=file_hash,workflow_execution=workflow_execution,)+if file_execution_result.error:failed_files += 1ExecutionCacheUtils.increment_failed_files(@@ -184,10 +189,10 @@ class FileExecutionTasks:workflow_id=workflow.id,execution_id=execution_id,)- return {- "successful_files": successful_files,- "failed_files": failed_files,- }+ return FileBatchResult(+ successful_files=successful_files,+ failed_files=failed_files,+ ).to_dict()@file_processing_app.task(bind=True,@@ -235,8 +240,12 @@ class FileExecutionTasks:total_successful = sum(result["successful_files"] for result in results)total_failed = sum(result["failed_files"] for result in results)- total_files = total_successful + total_failed+ batch_result = FileBatchResult(+ successful_files=total_successful,+ failed_files=total_failed,+ )+ total_files = batch_result.total_files# Update final statusfinal_status = (ExecutionStatus.COMPLETED if total_successful else ExecutionStatus.ERROR@@ -296,6 +305,9 @@ class FileExecutionTasks:FileExecutionResult: Result of the file execution"""try:+ logger.info(+ f"[Execution {workflow_execution.id}] Processing file: '{file_hash.file_name}'"+ )# Initialization Phaseexecution_context = cls._initialize_execution_context(file_data, file_hash, workflow_execution@@ -354,6 +366,26 @@ class FileExecutionTasks:destination,execution_result,)+ except ExecutionContextInitializationError as error:+ # This case is not expected under normal conditions — handled here as a safety net.+ # If it occurs, it likely indicates a logical error (e.g., the same file being processed with a different name under the same execution_id).+ error_msg = f"Error initializing execution context. {error}"+ workflow_log = WorkflowLog(+ execution_id=str(workflow_execution.id),+ log_stage=LogStage.PROCESSING,+ organization_id=file_data.organization_id,+ pipeline_id=str(workflow_execution.pipeline_id),+ )+ workflow_log.log_error(logger=logger, message=error_msg)+ result = FinalOutputResult(output=None, metadata=None, error=error_msg)+ return cls._build_final_result(+ workflow_execution=workflow_execution,+ file_hash=file_hash,+ result=result,+ workflow_file_execution=None,+ error=error_msg,+ is_api=destination.is_api,+ )except Exception as error:error_msg = f"File execution failed: {error}"workflow_log.log_error(logger=logger, message=error_msg)@@ -363,11 +395,11 @@ class FileExecutionTasks:result = FinalOutputResult(output=None, metadata=None, error=error_msg)return cls._build_final_result(workflow_execution=workflow_execution,- workflow_file_execution=workflow_file_execution,file_hash=file_hash,- is_api=destination.is_api,result=result,+ workflow_file_execution=workflow_file_execution,error=error_msg,+ is_api=destination.is_api,)@classmethod@@ -378,45 +410,53 @@ class FileExecutionTasks:workflow_execution: WorkflowExecution,) -> ExecutionContext:"""Set up all required execution context objects and configurations."""- workflow = workflow_execution.workflow- endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(- workflow=workflow,- endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,- )+ try:+ workflow = workflow_execution.workflow+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ )- is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API+ is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API- # Create execution record- file_execution: WorkflowFileExecution = (- WorkflowFileExecution.objects.get_or_create_file_execution(- workflow_execution=workflow_execution,- file_hash=file_hash,- is_api=is_api,+ # Create execution record+ file_execution: WorkflowFileExecution = (+ WorkflowFileExecution.objects.get_or_create_file_execution(+ workflow_execution=workflow_execution,+ file_hash=file_hash,+ is_api=is_api,+ ))- )- # Create configurations- source_config = SourceConfig.from_json(file_data.source_config)- destination_config = DestinationConfig.from_json(file_data.destination_config)- source_config.file_execution_id = destination_config.file_execution_id = str(- file_execution.id- )+ # Create configurations+ source_config = SourceConfig.from_json(file_data.source_config)+ destination_config = DestinationConfig.from_json(file_data.destination_config)+ source_config.file_execution_id = destination_config.file_execution_id = str(+ file_execution.id+ )- # Initialize logging- workflow_log = WorkflowLog(- execution_id=str(workflow_execution.id),- log_stage=LogStage.PROCESSING,- organization_id=file_data.organization_id,- pipeline_id=str(workflow_execution.pipeline_id),- file_execution_id=str(file_execution.id),- )+ # Initialize logging+ workflow_log = WorkflowLog(+ execution_id=str(workflow_execution.id),+ log_stage=LogStage.PROCESSING,+ organization_id=file_data.organization_id,+ pipeline_id=str(workflow_execution.pipeline_id),+ file_execution_id=str(file_execution.id),+ )- return ExecutionContext(- workflow_log=workflow_log,- workflow_file_execution=file_execution,- source_config=source_config,- destination_config=destination_config,- )+ return ExecutionContext(+ workflow_log=workflow_log,+ workflow_file_execution=file_execution,+ source_config=source_config,+ destination_config=destination_config,+ )+ except Exception as e:+ logger.error(+ f"Failed to initialize execution context: {str(e)}", exc_info=True+ )+ raise ExecutionContextInitializationError(+ f"Failed to initialize execution context: {str(e)}"+ ) from e@classmethoddef _prepare_file_for_processing(@@ -431,6 +471,7 @@ class FileExecutionTasks:workflow_file_exec.update_status(ExecutionStatus.EXECUTING)try:+ logger.info(f"Preparing file for processing: {file_hash.file_name}")content_hash = source.add_file_to_volume(workflow_file_execution=workflow_file_exec,tags=workflow_execution.tag_names,@@ -462,13 +503,35 @@ class FileExecutionTasks:if destination.is_api:return None+ # Check for existing file processing history by content hashfile_history = FileHistoryHelper.get_file_history(workflow=workflow, cache_key=content_hash)if not file_history:return None- workflow_log.log_info(f"Skipping duplicate '{file_hash.file_name}'")+ workflow_log.log_info(+ logger=logger,+ message=f"Skipping duplicate file: '{file_hash.file_name}' in workflow '{workflow}' (content hash match)",+ )++ # Check for provider_file_uuid consistency+ if file_history.provider_file_uuid and file_hash.provider_file_uuid:+ logger.warning(+ f"Provider file UUID mismatch for file '{file_hash.file_name}' in workflow '{workflow}'"+ )+ elif file_history.provider_file_uuid and not file_hash.provider_file_uuid:+ logger.warning(+ f"Provider file UUID is missing for file '{file_hash.file_name}' in workflow '{workflow}'"+ )+ elif not file_history.provider_file_uuid and file_hash.provider_file_uuid:+ logger.info(+ f"Backfilling provider UUID for '{file_hash.file_name}' in workflow '{workflow}' with value: "+ f"{file_hash.provider_file_uuid}"+ )+ file_history.provider_file_uuid = file_hash.provider_file_uuid+ file_history.save()+return FileExecutionResult(file=file_hash.file_name,file_execution_id=str(workflow_file_exec.id),@@ -488,6 +551,7 @@ class FileExecutionTasks:total_files: int,) -> ToolExecutionResult:"""Execute main workflow processing steps with proper error handling."""+ logger.info(f"Executing workflow steps for file: '{file_hash.file_name}'")tool_instances: list[ToolInstance] = (ToolInstanceHelper.get_tool_instances_by_workflow(file_data.workflow_id, ToolInstanceKey.STEP@@ -556,6 +620,7 @@ class FileExecutionTasks:execution_result: ToolExecutionResult,) -> FileExecutionResult:"""Handle final processing steps and result generation."""+ logger.info(f"Finalizing execution for file: '{file_hash.file_name}'")result = cls._process_final_output(destination,workflow_execution.workflow,@@ -572,11 +637,11 @@ class FileExecutionTasks:return cls._build_final_result(workflow_execution=workflow_execution,- workflow_file_execution=workflow_file_execution,file_hash=file_hash,- is_api=destination.is_api,result=result,+ workflow_file_execution=workflow_file_execution,error=result.error or execution_result.error,+ is_api=destination.is_api,)@classmethod@@ -614,7 +679,12 @@ class FileExecutionTasks:if destination.is_api:execution_metadata = destination.get_metadata(file_history)- if destination.use_file_history and not file_history:+ if cls._should_create_file_history(+ destination=destination,+ file_history=file_history,+ output_result=output_result,+ processing_error=processing_error,+ ):FileHistoryHelper.create_file_history(file_hash=file_hash,workflow=workflow,@@ -632,6 +702,41 @@ class FileExecutionTasks:output=output_result, metadata=execution_metadata, error=None)+ @classmethod+ def _should_create_file_history(+ cls,+ destination: DestinationConnector,+ file_history: FileHistory | None,+ output_result: str | None,+ processing_error: str | None,+ ) -> bool:+ """Determine whether a new FileHistory record should be created.++ Returns True if:+ - File history is enabled for the destination,+ - No existing file history record is present,+ - There is no processing error (`processing_error` is None),+ - And one of the following conditions is met:+ - The destination is not an API, or+ - The destination is an API and a valid output_result exists.++ Args:+ destination: Destination connector+ file_history: File history+ output_result: Output result+ processing_error: Processing error++ Returns:+ bool: True if file history should be created, False otherwise+ """+ if not destination.use_file_history or file_history:+ return False+ if destination.is_api and not output_result:+ return False+ if processing_error:+ return False+ return True+@classmethoddef _complete_execution(cls,@@ -662,11 +767,11 @@ class FileExecutionTasks:def _build_final_result(cls,workflow_execution: WorkflowExecution,- workflow_file_execution: WorkflowFileExecution,file_hash: FileHash,- is_api: bool,result: FinalOutputResult,- error: str | None,+ workflow_file_execution: WorkflowFileExecution | None = None,+ error: str | None = None,+ is_api: bool = False,) -> FileExecutionResult:"""Construct and cache the final execution result."""final_result = FileExecutionResult(diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.pyindex c91e234d6..17013b5a9 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py@@ -89,6 +89,24 @@ class AzureCloudStorageFS(UnstractFileSystem):)return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ inner_metadata = metadata.get("metadata")+ if not isinstance(inner_metadata, dict):+ inner_metadata = {}++ is_dir = inner_metadata.get("is_directory") == "true"+ if not is_dir:+ is_dir = metadata.get("type") == "directory"+ return is_dir+def test_credentials(self) -> bool:"""To test credentials for Azure Cloud Storage."""try:diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.pyindex 910a5acb1..8a6c03347 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py@@ -122,6 +122,17 @@ class BoxFS(UnstractFileSystem):logger.error(f"[Box] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ raise NotImplementedError+def test_credentials(self) -> bool:"""To test credentials for the Box connector."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.pyindex 3f4fee644..9f983402d 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py@@ -94,6 +94,18 @@ class GoogleCloudStorageFS(UnstractFileSystem):logger.error(f"[GCS] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ # Note: Here Metadata type seems to be always "file" even for directories+ return metadata.get("type") == "directory"+def test_credentials(self) -> bool:"""Test Google Cloud Storage credentials by accessing the root path info.diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.pyindex 3955cabbd..a8cb313bb 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py@@ -107,9 +107,24 @@ class GoogleDriveFS(UnstractFileSystem):Returns:Optional[str]: The file hash in hexadecimal format or None if not found."""+ # Extracts checksum for GDrive+ file_hash = metadata.get("checksum")+ if file_hash:+ return file_hash.lower()logger.error(f"[Google Drive] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ return metadata.get("type") == "directory"+def test_credentials(self) -> bool:"""To test credentials for Google Drive."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.pyindex 2908eb147..e19b24e54 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py@@ -81,6 +81,17 @@ class HttpFS(UnstractFileSystem):logger.error(f"[HTTP] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ raise NotImplementedError+def test_credentials(self) -> bool:"""To test credentials for HTTP(S)."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.pyindex a63d6970a..8a701b25c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py@@ -71,6 +71,17 @@ class LocalStorageFS(UnstractFileSystem):logger.error(f"[LocalStorage] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ raise NotImplementedError+def test_credentials(self, *args, **kwargs) -> bool: # type:ignore"""To test credentials for LocalStorage."""is_dir = Falsediff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex 8531565fd..60a365d16 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -93,6 +93,17 @@ class MinioFS(UnstractFileSystem):logger.error(f"[MinIO] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ return metadata.get("type") == "directory"+def get_fsspec_fs(self) -> S3FileSystem:return self.s3diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.pyindex 3989eeb95..e6ba61b82 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py@@ -77,6 +77,17 @@ class SftpFS(UnstractFileSystem):logger.error(f"[SFTP] File hash not found for the metadata: {metadata}")return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ raise NotImplementedError+# TODO: Check if this method can be removed, and use it from parent class# (class UnstractFileSystem)@staticmethoddiff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.pyindex 02ac90d0d..d7debda77 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py@@ -91,20 +91,37 @@ class UnstractFileSystem(UnstractConnector, ABC):"""pass+ @abstractmethod+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ pass+@staticmethoddef get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:"""Override to get root dir of a connector."""return f"{input_dir.strip('/')}/"- def get_file_size(self, file_path: str) -> int | None:+ def get_file_size(+ self, file_path: str | None = None, metadata: dict[str, Any] | None = None+ ) -> int | None:"""Get the size of a file."""- fs_fsspec = self.get_fsspec_fs()- metadata = fs_fsspec.stat(file_path)+ if not metadata:+ if not file_path:+ return None+ fs_fsspec = self.get_fsspec_fs()+ metadata = fs_fsspec.stat(file_path)if not metadata or not isinstance(metadata, dict):return Nonereturn metadata.get("size")- def _serialize_metadata_value(self, value: Any, depth: int = 0) -> Any:+ def serialize_metadata_value(self, value: Any, depth: int = 0) -> Any:"""Recursively serialize metadata values to ensure JSON compatibility.Args:@@ -128,15 +145,15 @@ class UnstractFileSystem(UnstractConnector, ABC):return base64.b64encode(value).decode("utf-8")elif isinstance(value, dict):return {- k: self._serialize_metadata_value(v, depth + 1) for k, v in value.items()+ k: self.serialize_metadata_value(v, depth + 1) for k, v in value.items()}elif isinstance(value, (list, tuple)):- return [self._serialize_metadata_value(v, depth + 1) for v in value]+ return [self.serialize_metadata_value(v, depth + 1) for v in value]else:# For custom objects like ContentSettings, convert to dict if possibleif hasattr(value, "__dict__"):try:- return self._serialize_metadata_value(value.__dict__, depth + 1)+ return self.serialize_metadata_value(value.__dict__, depth + 1)except Exception:return str(value)# Last resort: convert to string@@ -162,24 +179,29 @@ class UnstractFileSystem(UnstractConnector, ABC):# Recursively serialize all metadata valuesserialized_metadata = {}for key, value in metadata.items():- serialized_metadata[key] = self._serialize_metadata_value(value)+ serialized_metadata[key] = self.serialize_metadata_value(value)return serialized_metadataexcept Exception as e:logger.error(f"Error getting file system metadata for {file_path}: {str(e)}")return {}- def get_file_system_uuid(self, file_path: str) -> str | None:+ def get_file_system_uuid(+ self, file_path: str, metadata: dict[str, Any]+ ) -> str | None:"""Get the UUID of a file.Args:file_path (str): Path of the file.+ metadata (dict[str, Any]): Metadata of the file.Returns:Optional[str]: UUID of the file in hex format or None if not found."""- fs_fsspec = self.get_fsspec_fs()- metadata = fs_fsspec.stat(file_path)+ if not metadata:+ fs_fsspec = self.get_fsspec_fs()+ metadata = fs_fsspec.stat(file_path)+file_hash = self.extract_metadata_file_hash(metadata)if not file_hash:logger.error(f"File hash not found for {file_path}.")diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.pyindex 576fd3f54..306357185 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py@@ -80,6 +80,17 @@ class DropboxFS(UnstractFileSystem):)return None+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:+ """Check if the given path is a directory.++ Args:+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.++ Returns:+ bool: True if the path is a directory, False otherwise.+ """+ return metadata.get("type") == "directory"+def test_credentials(self) -> bool:"""To test credentials for Dropbox."""try: