Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Optimization and fix of file execution - parallel execution (#1326)

ali472e725cc326
diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.py
index b1a9aab4e..d2365a18b 100644
--- a/backend/workflow_manager/endpoint_v2/dto.py
+++ b/backend/workflow_manager/endpoint_v2/dto.py
@@ -113,7 +113,7 @@ class DestinationConfig:
@dataclass
class FileExecutionResult:
file: str
- file_execution_id: str
+ file_execution_id: str | None = None
status: str | None = None
result: str | None = None
error: str | None = None
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 688e762a0..5cfb8a036 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -185,6 +185,7 @@ class SourceConnector(BaseConnector):
source_fs_fsspec = source_fs.get_fsspec_fs()
# Checking if folders exist at source before processing
# TODO: Validate while receiving this input configuration as well
+ valid_directories = []
for input_directory in folders_to_process:
# TODO: Move to connector class for better error handling
try:
@@ -193,6 +194,7 @@ class SourceConnector(BaseConnector):
)
if not source_fs_fsspec.isdir(input_directory):
raise InvalidInputDirectory(dir=input_directory)
+ valid_directories.append(input_directory)
except Exception as e:
msg = f"Error while validating path '{input_directory}'. {str(e)}"
self.publish_user_sys_log(msg)
@@ -202,14 +204,12 @@ class SourceConnector(BaseConnector):
total_files_to_process = 0
total_matched_files = {}
+ unique_file_hashes: set[str] = set()
- for input_directory in folders_to_process:
- input_directory = source_fs.get_connector_root_dir(
- input_dir=input_directory, root_path=root_dir_path
- )
+ for input_directory in valid_directories:
logger.debug(f"Listing files from: {input_directory}")
matched_files, count = self._get_matched_files(
- source_fs, input_directory, patterns, recursive, limit
+ source_fs, input_directory, patterns, recursive, limit, unique_file_hashes
)
self.publish_user_sys_log(f"Matched '{count}' files from '{input_directory}'")
total_matched_files.update(matched_files)
@@ -274,6 +274,7 @@ class SourceConnector(BaseConnector):
patterns: list[str],
recursive: bool,
limit: int,
+ unique_file_hashes: set[str],
) -> tuple[dict[str, FileHash], int]:
"""Get a dictionary of matched files based on patterns in a directory.
@@ -298,24 +299,180 @@ class SourceConnector(BaseConnector):
count = 0
max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
fs_fsspec = source_fs.get_fsspec_fs()
- for root, dirs, files in fs_fsspec.walk(input_directory, maxdepth=max_depth):
- for file in files:
- if count >= limit:
- break
- if self._should_process_file(file, patterns):
- file_path = str(os.path.join(root, file))
- if self._is_new_file(
- file_path=file_path,
- workflow=self.endpoint.workflow,
- source_fs=source_fs,
- ):
- matched_files[file_path] = self._create_file_hash(
- file_path=file_path,
- source_fs=source_fs,
- )
- count += 1
+ for root, dirs, _ in fs_fsspec.walk(input_directory, maxdepth=max_depth):
+ try:
+ fs_metadata_list: list[dict[str, Any]] = fs_fsspec.listdir(
+ root
+ ) # Single call for file system metadata
+ except Exception as e:
+ logger.warning(f"Failed to list directory from path: {root}, error: {e}")
+ continue
+
+ count = self._process_file_fs_directory(
+ fs_metadata_list=fs_metadata_list,
+ count=count,
+ limit=limit,
+ unique_file_hashes=unique_file_hashes,
+ matched_files=matched_files,
+ patterns=patterns,
+ source_fs=source_fs,
+ dirs=dirs,
+ )
return matched_files, count
+ def _process_file_fs_directory(
+ self,
+ fs_metadata_list: list[dict[str, Any]],
+ count: int,
+ limit: int,
+ unique_file_hashes: set[str],
+ matched_files: dict[str, FileHash],
+ patterns: list[str],
+ source_fs: UnstractFileSystem,
+ dirs: list[str],
+ ) -> int:
+ for fs_metadata in fs_metadata_list:
+ if count >= limit:
+ logger.info(
+ f"[Matched Files] Maximum limit {limit} of files reached, count: {count}"
+ )
+ break
+
+ file_path: str | None = fs_metadata.get("name")
+ file_size = fs_metadata.get("size", 0)
+
+ if not file_path or self._is_directory(
+ source_fs, file_path, fs_metadata, dirs
+ ):
+ continue
+
+ if self._should_skip_file(file_path, fs_metadata, patterns, source_fs):
+ continue
+
+ file_hash = self._create_file_hash(
+ file_path=file_path,
+ source_fs=source_fs,
+ file_size=file_size,
+ fs_metadata=fs_metadata,
+ )
+
+ # Skip duplicate files
+ if self._is_duplicate(file_hash, unique_file_hashes):
+ logger.info(
+ f"[Matched Files] Skipping execution of duplicate file: {file_path}"
+ )
+ continue
+ self._update_unique_file_hashes(file_hash, unique_file_hashes)
+
+ matched_files[file_path] = file_hash
+ count += 1
+ return count
+
+ def _should_skip_file(
+ self,
+ file_path: str,
+ fs_metadata: dict[str, Any],
+ patterns: list[str],
+ source_fs: UnstractFileSystem,
+ ) -> bool:
+ """Check if the given file should be skipped.
+
+ Args:
+ file_path (str): The path of the file.
+ fs_metadata (dict[str, Any]): The metadata of the file.
+ patterns (list[str]): The patterns to match against file names.
+ source_fs (UnstractFileSystem): The file system object used for
+ reading the file.
+
+ Returns:
+ bool: True if the file should be skipped, False otherwise.
+ """
+ file_name = os.path.basename(file_path)
+ return not self._should_process_file(
+ file_name, patterns
+ ) or not self._is_new_file(
+ file_path=file_path,
+ fs_metadata=fs_metadata,
+ workflow=self.endpoint.workflow,
+ source_fs=source_fs,
+ )
+
+ def _is_duplicate(self, file_hash: FileHash, unique_file_hashes: set[str]) -> bool:
+ return (
+ file_hash.provider_file_uuid in unique_file_hashes
+ or file_hash.file_hash in unique_file_hashes
+ )
+
+ def _update_unique_file_hashes(
+ self, file_hash: FileHash, unique_file_hashes: set[str]
+ ) -> None:
+ if file_hash.provider_file_uuid:
+ unique_file_hashes.add(file_hash.provider_file_uuid)
+ elif file_hash.file_hash:
+ unique_file_hashes.add(file_hash.file_hash)
+
+ def _is_directory(
+ self,
+ source_fs: UnstractFileSystem,
+ file_path: str,
+ metadata: dict[str, Any],
+ dirs: list[str],
+ ) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ source_fs (UnstractFileSystem): The file system object used for
+ reading the file.
+ file_path (str): The path of the file.
+ metadata (dict[str, Any]): The metadata of the file.
+ dirs (list[str]): The list of directories.
+
+ Returns:
+ bool: True if the file is a directory, False otherwise.
+ """
+ try:
+ # Check if the path is a directory using metadata first.
+ # Some connectors incorrectly label directories as files, so if metadata is inconclusive or fails,
+ # fall back to other checks: directory listing, path suffix ("/"), or zero file size.
+ if source_fs.is_dir_by_metadata(metadata):
+ logger.info(
+ f"[Directory Check] '{file_path}' identified as a directory via metadata."
+ )
+ return True
+ except NotImplementedError:
+ logger.debug(
+ f"[Directory Check] Metadata-based check not implemented for '{file_path}'."
+ )
+ except Exception as e:
+ logger.warning(
+ f"[Directory Check] Error while checking metadata for '{file_path}': {e}"
+ )
+
+ file_name = os.path.basename(file_path)
+
+ # Fallback 1: Check if the file is explicitly listed in directory entries
+ if file_name in dirs:
+ logger.info(
+ f"[Directory Check] '{file_path}' identified as a directory via checking list of directories."
+ )
+ return True
+
+ # Fallback 2: Check if the path ends with a slash
+ if file_path.endswith("/"):
+ logger.info(
+ f"[Directory Check] '{file_path}' identified as a directory based on path suffix '/'."
+ )
+ return True
+
+ # Fallback 3: Check if the file has size zero
+ if source_fs.get_file_size(metadata=metadata) == 0:
+ logger.info(
+ f"[Directory Check] '{file_path}' identified as a directory based on file size = 0."
+ )
+ return True
+
+ return False
+
def _should_process_file(self, file: str, patterns: list[str]) -> bool:
"""Check if the file should be processed based on the patterns.
@@ -360,10 +517,14 @@ class SourceConnector(BaseConnector):
return True
def _is_new_file(
- self, file_path: str, workflow: Workflow, source_fs: UnstractFileSystem
+ self,
+ file_path: str,
+ fs_metadata: dict[str, Any],
+ workflow: Workflow,
+ source_fs: UnstractFileSystem,
) -> bool:
"""Check if the file is new or already processed."""
- file_history = self._get_file_history(workflow, source_fs, file_path)
+ file_history = self._get_file_history(fs_metadata, workflow, source_fs, file_path)
# In case of ETL pipelines, its necessary to skip files which have
# already been processed
if self.use_file_history and file_history and file_history.is_completed():
@@ -376,57 +537,36 @@ class SourceConnector(BaseConnector):
return True
def _get_file_history(
- self, workflow: Workflow, source_fs: UnstractFileSystem, file_path: str
+ self,
+ fs_metadata: dict[str, Any],
+ workflow: Workflow,
+ source_fs: UnstractFileSystem,
+ file_path: str,
) -> FileHistory | None:
"""Retrieve file history using provider UUID or legacy cache key."""
- provider_file_uuid = source_fs.get_file_system_uuid(file_path)
+ provider_file_uuid = source_fs.get_file_system_uuid(file_path, fs_metadata)
if provider_file_uuid:
logger.info(f"Checking file history for provider UUID: {provider_file_uuid}")
file_history = FileHistoryHelper.get_file_history(
workflow=workflow, provider_file_uuid=provider_file_uuid
)
-
- if file_history:
- return file_history # Early return if history exists
-
- # The provider_file_uuid was recently integrated,
- # so we also check the cache_key for backward compatibility.
- # This ensures older files without a provider UUID
- # can still be identified.
- # In the future, this check can be removed as file history validation
- # is already handled during file execution.
- file_content_hash = self.get_file_content_hash(source_fs, file_path)
- logger.info(
- f"Checking file history for legacy cache key: {file_content_hash}"
- )
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_content_hash
- )
-
- if file_history and file_history.is_completed():
- file_history.update(provider_file_uuid=provider_file_uuid)
-
- # Fallback for connectors that do not support provider_file_uuid
- else:
- file_content_hash = self.get_file_content_hash(source_fs, file_path)
- logger.info(
- f"Checking file history for legacy cache key: {file_content_hash}"
- )
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_content_hash
- )
-
- return file_history
+ return file_history
+ return None
def _create_file_hash(
- self, file_path: str, source_fs: UnstractFileSystem
+ self,
+ file_path: str,
+ source_fs: UnstractFileSystem,
+ file_size: int,
+ fs_metadata: dict[str, Any],
) -> FileHash:
"""Create a FileHash object for the matched file."""
file_name = os.path.basename(file_path)
- provider_file_uuid = source_fs.get_file_system_uuid(file_path)
- file_size = source_fs.get_file_size(file_path)
- fs_metadata = source_fs.get_file_metadata(file_path)
+ provider_file_uuid = source_fs.get_file_system_uuid(
+ file_path=file_path, metadata=fs_metadata
+ )
+ serialized_metadata = source_fs.serialize_metadata_value(value=fs_metadata)
connection_type = self.endpoint.connection_type
return FileHash(
file_path=file_path,
@@ -434,7 +574,7 @@ class SourceConnector(BaseConnector):
file_name=file_name,
file_size=file_size,
provider_file_uuid=provider_file_uuid,
- fs_metadata=fs_metadata,
+ fs_metadata=serialized_metadata,
)
def list_files_from_source(
@@ -501,6 +641,9 @@ class SourceConnector(BaseConnector):
Raises:
FileHashNotFound: If the hash value of the file content is not found.
"""
+ logger.info(
+ f"Adding input file from source connector to execution directory: {file_hash.file_name}"
+ )
input_file_path = file_hash.file_path
source_file_path = self.source_file
infile_path = self.infile
@@ -545,6 +688,9 @@ class SourceConnector(BaseConnector):
def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:
"""Add input file to execution directory from api storage."""
+ logger.info(
+ f"Adding input file from api storage to execution directory: {input_file_path}"
+ )
source_file_path = self.source_file
infile_path = self.infile
if not source_file_path or not infile_path:
@@ -631,6 +777,7 @@ class SourceConnector(BaseConnector):
Returns:
str: file_name
"""
+ logger.info(f"Adding input file to execution directory: {file_hash.file_name}")
connection_type = self.endpoint.connection_type
input_file_path = file_hash.file_path
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
@@ -641,6 +788,12 @@ class SourceConnector(BaseConnector):
file_content_hash = self.add_input_from_api_storage_to_volume(
input_file_path=input_file_path
)
+ # Filehash created from Django requests files might be different
+ # from file_content_hash created from file content here.
+ # So use file_content_hash only when file_hash is not available
+ file_content_hash = (
+ file_hash.file_hash if file_hash.file_hash else file_content_hash
+ )
else:
raise InvalidSourceConnectionType()
@@ -709,6 +862,7 @@ class SourceConnector(BaseConnector):
)
workflow: Workflow = Workflow.objects.get(id=workflow_id)
file_hashes: dict[str, FileHash] = {}
+ unique_file_hashes: set[str] = set()
for file in file_objs:
file_name = file.name
destination_path = os.path.join(api_storage_dir, file_name)
@@ -722,6 +876,14 @@ class SourceConnector(BaseConnector):
file_hash = file_hash.hexdigest()
connection_type = WorkflowEndpoint.ConnectionType.API
+ # Skip duplicate files
+ if file_hash in unique_file_hashes:
+ logger.info(
+ f"[Matched Files] Skipping execution of duplicate file: {file_name}"
+ )
+ continue
+ unique_file_hashes.add(file_hash)
+
file_history = None
if use_file_history:
file_history = FileHistoryHelper.get_file_history(
diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.py
index 299fc75d2..854be42c6 100644
--- a/backend/workflow_manager/utils/workflow_log.py
+++ b/backend/workflow_manager/utils/workflow_log.py
@@ -49,8 +49,19 @@ class WorkflowLog:
self.publish_log(message, level=LogLevel.ERROR)
logger.error(message, exc_info=True, stack_info=True)
- def log_info(self, message: str) -> None:
+ def log_info(self, logger: logging.Logger, message: str) -> None:
+ """Publishes an info log message to the configured logger and to the
+ websocket channel if the `execution_service` is configured.
+
+ Args:
+ logger (logging.Logger): The logger to use for logging.
+ message (str): The log message to be published.
+
+ Returns:
+ None
+ """
self.publish_log(message, level=LogLevel.INFO)
+ logger.info(message)
def publish_update_log(
self,
diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.py
index 91dcad76c..f6e0a0f1a 100644
--- a/backend/workflow_manager/workflow_v2/dto.py
+++ b/backend/workflow_manager/workflow_v2/dto.py
@@ -186,6 +186,22 @@ class FileBatchData:
return asdict(self)
+@dataclass
+class FileBatchResult:
+ successful_files: int = 0
+ failed_files: int = 0
+
+ @property
+ def total_files(self) -> int:
+ return self.successful_files + self.failed_files
+
+ def to_dict(self) -> dict[str, int]:
+ return {
+ "successful_files": self.successful_files,
+ "failed_files": self.failed_files,
+ }
+
+
@dataclass
class ToolExecutionResult:
error: str | None
diff --git a/backend/workflow_manager/workflow_v2/exceptions.py b/backend/workflow_manager/workflow_v2/exceptions.py
index fa53b3023..6f52081f8 100644
--- a/backend/workflow_manager/workflow_v2/exceptions.py
+++ b/backend/workflow_manager/workflow_v2/exceptions.py
@@ -59,3 +59,7 @@ class InternalException(APIException):
class WorkflowExecutionNotExist(APIException):
status_code = 404
default_detail = "Workflow execution does not exist"
+
+
+class ExecutionContextInitializationError(APIException):
+ default_detail = "Error initializing execution context."
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 3d04ab850..10fcb96b3 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -31,18 +31,21 @@ from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.dto import (
ExecutionContext,
FileBatchData,
+ FileBatchResult,
FileData,
FinalOutputResult,
ToolExecutionResult,
)
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.exceptions import (
+ ExecutionContextInitializationError,
WorkflowDoesNotExistError,
WorkflowExecutionError,
)
from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
logger = logging.getLogger(__name__)
@@ -113,22 +116,23 @@ class FileExecutionTasks:
- single_step: Whether to process in single step mode
"""
file_batch_data = FileBatchData.from_dict(file_batch_data)
+ logger.debug(f"Batch data received: {file_batch_data}")
+
file_data = file_batch_data.file_data
logger.info(f"File processing context {file_data}")
+ execution_id = file_data.execution_id
+ workflow_id = file_data.workflow_id
organization_id = file_data.organization_id
- StateStore.set(Account.ORGANIZATION_ID, organization_id)
- # Use proper logger instead of print
- logger.info(
- f"Starting to process file batch for execution {file_data.execution_id}"
- )
- logger.debug(f"Batch data received: {file_batch_data}")
+ StateStore.set(Account.ORGANIZATION_ID, organization_id)
successful_files = 0
failed_files = 0
- execution_id = file_data.execution_id
- workflow_id = file_data.workflow_id
+
+ logger.info(
+ f"Initializing file batch processing for execution {execution_id}, organization {organization_id}"
+ )
# Reconstruct necessary objects
workflow = FileExecutionTasks.get_workflow_by_id(str(workflow_id))
workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(
@@ -140,13 +144,12 @@ class FileExecutionTasks:
total_files = len(file_batch_data.files)
q_file_no_list = set(file_data.q_file_no_list)
- logger.info(
- f"Processing {total_files} files of execution {execution_id} in a batch"
- )
+ logger.info(f"Processing {total_files} files of execution {execution_id}")
for file_number, (file_name, file_hash_dict) in enumerate(
file_batch_data.files, 1
):
+ logger.info(f"[{file_number}/{total_files}] Processing file '{file_name}'")
file_hash = FileHash(
file_path=file_hash_dict.get("file_path"),
file_name=file_hash_dict.get("file_name"),
@@ -165,6 +168,7 @@ class FileExecutionTasks:
q_file_no_list,
file_hash,
)
+ logger.info(f"File hash for file {file_name}: {file_hash.to_json()}")
file_execution_result = FileExecutionTasks._process_file(
current_file_idx=file_number,
total_files=total_files,
@@ -172,6 +176,7 @@ class FileExecutionTasks:
file_hash=file_hash,
workflow_execution=workflow_execution,
)
+
if file_execution_result.error:
failed_files += 1
ExecutionCacheUtils.increment_failed_files(
@@ -184,10 +189,10 @@ class FileExecutionTasks:
workflow_id=workflow.id,
execution_id=execution_id,
)
- return {
- "successful_files": successful_files,
- "failed_files": failed_files,
- }
+ return FileBatchResult(
+ successful_files=successful_files,
+ failed_files=failed_files,
+ ).to_dict()
@file_processing_app.task(
bind=True,
@@ -235,8 +240,12 @@ class FileExecutionTasks:
total_successful = sum(result["successful_files"] for result in results)
total_failed = sum(result["failed_files"] for result in results)
- total_files = total_successful + total_failed
+ batch_result = FileBatchResult(
+ successful_files=total_successful,
+ failed_files=total_failed,
+ )
+ total_files = batch_result.total_files
# Update final status
final_status = (
ExecutionStatus.COMPLETED if total_successful else ExecutionStatus.ERROR
@@ -296,6 +305,9 @@ class FileExecutionTasks:
FileExecutionResult: Result of the file execution
"""
try:
+ logger.info(
+ f"[Execution {workflow_execution.id}] Processing file: '{file_hash.file_name}'"
+ )
# Initialization Phase
execution_context = cls._initialize_execution_context(
file_data, file_hash, workflow_execution
@@ -354,6 +366,26 @@ class FileExecutionTasks:
destination,
execution_result,
)
+ except ExecutionContextInitializationError as error:
+ # This case is not expected under normal conditions — handled here as a safety net.
+ # If it occurs, it likely indicates a logical error (e.g., the same file being processed with a different name under the same execution_id).
+ error_msg = f"Error initializing execution context. {error}"
+ workflow_log = WorkflowLog(
+ execution_id=str(workflow_execution.id),
+ log_stage=LogStage.PROCESSING,
+ organization_id=file_data.organization_id,
+ pipeline_id=str(workflow_execution.pipeline_id),
+ )
+ workflow_log.log_error(logger=logger, message=error_msg)
+ result = FinalOutputResult(output=None, metadata=None, error=error_msg)
+ return cls._build_final_result(
+ workflow_execution=workflow_execution,
+ file_hash=file_hash,
+ result=result,
+ workflow_file_execution=None,
+ error=error_msg,
+ is_api=destination.is_api,
+ )
except Exception as error:
error_msg = f"File execution failed: {error}"
workflow_log.log_error(logger=logger, message=error_msg)
@@ -363,11 +395,11 @@ class FileExecutionTasks:
result = FinalOutputResult(output=None, metadata=None, error=error_msg)
return cls._build_final_result(
workflow_execution=workflow_execution,
- workflow_file_execution=workflow_file_execution,
file_hash=file_hash,
- is_api=destination.is_api,
result=result,
+ workflow_file_execution=workflow_file_execution,
error=error_msg,
+ is_api=destination.is_api,
)
@classmethod
@@ -378,45 +410,53 @@ class FileExecutionTasks:
workflow_execution: WorkflowExecution,
) -> ExecutionContext:
"""Set up all required execution context objects and configurations."""
- workflow = workflow_execution.workflow
- endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
- workflow=workflow,
- endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
- )
+ try:
+ workflow = workflow_execution.workflow
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
- is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
+ is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
- # Create execution record
- file_execution: WorkflowFileExecution = (
- WorkflowFileExecution.objects.get_or_create_file_execution(
- workflow_execution=workflow_execution,
- file_hash=file_hash,
- is_api=is_api,
+ # Create execution record
+ file_execution: WorkflowFileExecution = (
+ WorkflowFileExecution.objects.get_or_create_file_execution(
+ workflow_execution=workflow_execution,
+ file_hash=file_hash,
+ is_api=is_api,
+ )
)
- )
- # Create configurations
- source_config = SourceConfig.from_json(file_data.source_config)
- destination_config = DestinationConfig.from_json(file_data.destination_config)
- source_config.file_execution_id = destination_config.file_execution_id = str(
- file_execution.id
- )
+ # Create configurations
+ source_config = SourceConfig.from_json(file_data.source_config)
+ destination_config = DestinationConfig.from_json(file_data.destination_config)
+ source_config.file_execution_id = destination_config.file_execution_id = str(
+ file_execution.id
+ )
- # Initialize logging
- workflow_log = WorkflowLog(
- execution_id=str(workflow_execution.id),
- log_stage=LogStage.PROCESSING,
- organization_id=file_data.organization_id,
- pipeline_id=str(workflow_execution.pipeline_id),
- file_execution_id=str(file_execution.id),
- )
+ # Initialize logging
+ workflow_log = WorkflowLog(
+ execution_id=str(workflow_execution.id),
+ log_stage=LogStage.PROCESSING,
+ organization_id=file_data.organization_id,
+ pipeline_id=str(workflow_execution.pipeline_id),
+ file_execution_id=str(file_execution.id),
+ )
- return ExecutionContext(
- workflow_log=workflow_log,
- workflow_file_execution=file_execution,
- source_config=source_config,
- destination_config=destination_config,
- )
+ return ExecutionContext(
+ workflow_log=workflow_log,
+ workflow_file_execution=file_execution,
+ source_config=source_config,
+ destination_config=destination_config,
+ )
+ except Exception as e:
+ logger.error(
+ f"Failed to initialize execution context: {str(e)}", exc_info=True
+ )
+ raise ExecutionContextInitializationError(
+ f"Failed to initialize execution context: {str(e)}"
+ ) from e
@classmethod
def _prepare_file_for_processing(
@@ -431,6 +471,7 @@ class FileExecutionTasks:
workflow_file_exec.update_status(ExecutionStatus.EXECUTING)
try:
+ logger.info(f"Preparing file for processing: {file_hash.file_name}")
content_hash = source.add_file_to_volume(
workflow_file_execution=workflow_file_exec,
tags=workflow_execution.tag_names,
@@ -462,13 +503,35 @@ class FileExecutionTasks:
if destination.is_api:
return None
+ # Check for existing file processing history by content hash
file_history = FileHistoryHelper.get_file_history(
workflow=workflow, cache_key=content_hash
)
if not file_history:
return None
- workflow_log.log_info(f"Skipping duplicate '{file_hash.file_name}'")
+ workflow_log.log_info(
+ logger=logger,
+ message=f"Skipping duplicate file: '{file_hash.file_name}' in workflow '{workflow}' (content hash match)",
+ )
+
+ # Check for provider_file_uuid consistency
+ if file_history.provider_file_uuid and file_hash.provider_file_uuid:
+ logger.warning(
+ f"Provider file UUID mismatch for file '{file_hash.file_name}' in workflow '{workflow}'"
+ )
+ elif file_history.provider_file_uuid and not file_hash.provider_file_uuid:
+ logger.warning(
+ f"Provider file UUID is missing for file '{file_hash.file_name}' in workflow '{workflow}'"
+ )
+ elif not file_history.provider_file_uuid and file_hash.provider_file_uuid:
+ logger.info(
+ f"Backfilling provider UUID for '{file_hash.file_name}' in workflow '{workflow}' with value: "
+ f"{file_hash.provider_file_uuid}"
+ )
+ file_history.provider_file_uuid = file_hash.provider_file_uuid
+ file_history.save()
+
return FileExecutionResult(
file=file_hash.file_name,
file_execution_id=str(workflow_file_exec.id),
@@ -488,6 +551,7 @@ class FileExecutionTasks:
total_files: int,
) -> ToolExecutionResult:
"""Execute main workflow processing steps with proper error handling."""
+ logger.info(f"Executing workflow steps for file: '{file_hash.file_name}'")
tool_instances: list[ToolInstance] = (
ToolInstanceHelper.get_tool_instances_by_workflow(
file_data.workflow_id, ToolInstanceKey.STEP
@@ -556,6 +620,7 @@ class FileExecutionTasks:
execution_result: ToolExecutionResult,
) -> FileExecutionResult:
"""Handle final processing steps and result generation."""
+ logger.info(f"Finalizing execution for file: '{file_hash.file_name}'")
result = cls._process_final_output(
destination,
workflow_execution.workflow,
@@ -572,11 +637,11 @@ class FileExecutionTasks:
return cls._build_final_result(
workflow_execution=workflow_execution,
- workflow_file_execution=workflow_file_execution,
file_hash=file_hash,
- is_api=destination.is_api,
result=result,
+ workflow_file_execution=workflow_file_execution,
error=result.error or execution_result.error,
+ is_api=destination.is_api,
)
@classmethod
@@ -614,7 +679,12 @@ class FileExecutionTasks:
if destination.is_api:
execution_metadata = destination.get_metadata(file_history)
- if destination.use_file_history and not file_history:
+ if cls._should_create_file_history(
+ destination=destination,
+ file_history=file_history,
+ output_result=output_result,
+ processing_error=processing_error,
+ ):
FileHistoryHelper.create_file_history(
file_hash=file_hash,
workflow=workflow,
@@ -632,6 +702,41 @@ class FileExecutionTasks:
output=output_result, metadata=execution_metadata, error=None
)
+ @classmethod
+ def _should_create_file_history(
+ cls,
+ destination: DestinationConnector,
+ file_history: FileHistory | None,
+ output_result: str | None,
+ processing_error: str | None,
+ ) -> bool:
+ """Determine whether a new FileHistory record should be created.
+
+ Returns True if:
+ - File history is enabled for the destination,
+ - No existing file history record is present,
+ - There is no processing error (`processing_error` is None),
+ - And one of the following conditions is met:
+ - The destination is not an API, or
+ - The destination is an API and a valid output_result exists.
+
+ Args:
+ destination: Destination connector
+ file_history: File history
+ output_result: Output result
+ processing_error: Processing error
+
+ Returns:
+ bool: True if file history should be created, False otherwise
+ """
+ if not destination.use_file_history or file_history:
+ return False
+ if destination.is_api and not output_result:
+ return False
+ if processing_error:
+ return False
+ return True
+
@classmethod
def _complete_execution(
cls,
@@ -662,11 +767,11 @@ class FileExecutionTasks:
def _build_final_result(
cls,
workflow_execution: WorkflowExecution,
- workflow_file_execution: WorkflowFileExecution,
file_hash: FileHash,
- is_api: bool,
result: FinalOutputResult,
- error: str | None,
+ workflow_file_execution: WorkflowFileExecution | None = None,
+ error: str | None = None,
+ is_api: bool = False,
) -> FileExecutionResult:
"""Construct and cache the final execution result."""
final_result = FileExecutionResult(
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
index c91e234d6..17013b5a9 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
@@ -89,6 +89,24 @@ class AzureCloudStorageFS(UnstractFileSystem):
)
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ inner_metadata = metadata.get("metadata")
+ if not isinstance(inner_metadata, dict):
+ inner_metadata = {}
+
+ is_dir = inner_metadata.get("is_directory") == "true"
+ if not is_dir:
+ is_dir = metadata.get("type") == "directory"
+ return is_dir
+
def test_credentials(self) -> bool:
"""To test credentials for Azure Cloud Storage."""
try:
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
index 910a5acb1..8a6c03347 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
@@ -122,6 +122,17 @@ class BoxFS(UnstractFileSystem):
logger.error(f"[Box] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ raise NotImplementedError
+
def test_credentials(self) -> bool:
"""To test credentials for the Box connector."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
index 3f4fee644..9f983402d 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
@@ -94,6 +94,18 @@ class GoogleCloudStorageFS(UnstractFileSystem):
logger.error(f"[GCS] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ # Note: Here Metadata type seems to be always "file" even for directories
+ return metadata.get("type") == "directory"
+
def test_credentials(self) -> bool:
"""Test Google Cloud Storage credentials by accessing the root path info.
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
index 3955cabbd..a8cb313bb 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
@@ -107,9 +107,24 @@ class GoogleDriveFS(UnstractFileSystem):
Returns:
Optional[str]: The file hash in hexadecimal format or None if not found.
"""
+ # Extracts checksum for GDrive
+ file_hash = metadata.get("checksum")
+ if file_hash:
+ return file_hash.lower()
logger.error(f"[Google Drive] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ return metadata.get("type") == "directory"
+
def test_credentials(self) -> bool:
"""To test credentials for Google Drive."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
index 2908eb147..e19b24e54 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
@@ -81,6 +81,17 @@ class HttpFS(UnstractFileSystem):
logger.error(f"[HTTP] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ raise NotImplementedError
+
def test_credentials(self) -> bool:
"""To test credentials for HTTP(S)."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
index a63d6970a..8a701b25c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
@@ -71,6 +71,17 @@ class LocalStorageFS(UnstractFileSystem):
logger.error(f"[LocalStorage] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ raise NotImplementedError
+
def test_credentials(self, *args, **kwargs) -> bool: # type:ignore
"""To test credentials for LocalStorage."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
index 8531565fd..60a365d16 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -93,6 +93,17 @@ class MinioFS(UnstractFileSystem):
logger.error(f"[MinIO] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ return metadata.get("type") == "directory"
+
def get_fsspec_fs(self) -> S3FileSystem:
return self.s3
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
index 3989eeb95..e6ba61b82 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
@@ -77,6 +77,17 @@ class SftpFS(UnstractFileSystem):
logger.error(f"[SFTP] File hash not found for the metadata: {metadata}")
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ raise NotImplementedError
+
# TODO: Check if this method can be removed, and use it from parent class
# (class UnstractFileSystem)
@staticmethod
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
index 02ac90d0d..d7debda77 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
@@ -91,20 +91,37 @@ class UnstractFileSystem(UnstractConnector, ABC):
"""
pass
+ @abstractmethod
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ pass
+
@staticmethod
def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:
"""Override to get root dir of a connector."""
return f"{input_dir.strip('/')}/"
- def get_file_size(self, file_path: str) -> int | None:
+ def get_file_size(
+ self, file_path: str | None = None, metadata: dict[str, Any] | None = None
+ ) -> int | None:
"""Get the size of a file."""
- fs_fsspec = self.get_fsspec_fs()
- metadata = fs_fsspec.stat(file_path)
+ if not metadata:
+ if not file_path:
+ return None
+ fs_fsspec = self.get_fsspec_fs()
+ metadata = fs_fsspec.stat(file_path)
if not metadata or not isinstance(metadata, dict):
return None
return metadata.get("size")
- def _serialize_metadata_value(self, value: Any, depth: int = 0) -> Any:
+ def serialize_metadata_value(self, value: Any, depth: int = 0) -> Any:
"""Recursively serialize metadata values to ensure JSON compatibility.
Args:
@@ -128,15 +145,15 @@ class UnstractFileSystem(UnstractConnector, ABC):
return base64.b64encode(value).decode("utf-8")
elif isinstance(value, dict):
return {
- k: self._serialize_metadata_value(v, depth + 1) for k, v in value.items()
+ k: self.serialize_metadata_value(v, depth + 1) for k, v in value.items()
}
elif isinstance(value, (list, tuple)):
- return [self._serialize_metadata_value(v, depth + 1) for v in value]
+ return [self.serialize_metadata_value(v, depth + 1) for v in value]
else:
# For custom objects like ContentSettings, convert to dict if possible
if hasattr(value, "__dict__"):
try:
- return self._serialize_metadata_value(value.__dict__, depth + 1)
+ return self.serialize_metadata_value(value.__dict__, depth + 1)
except Exception:
return str(value)
# Last resort: convert to string
@@ -162,24 +179,29 @@ class UnstractFileSystem(UnstractConnector, ABC):
# Recursively serialize all metadata values
serialized_metadata = {}
for key, value in metadata.items():
- serialized_metadata[key] = self._serialize_metadata_value(value)
+ serialized_metadata[key] = self.serialize_metadata_value(value)
return serialized_metadata
except Exception as e:
logger.error(f"Error getting file system metadata for {file_path}: {str(e)}")
return {}
- def get_file_system_uuid(self, file_path: str) -> str | None:
+ def get_file_system_uuid(
+ self, file_path: str, metadata: dict[str, Any]
+ ) -> str | None:
"""Get the UUID of a file.
Args:
file_path (str): Path of the file.
+ metadata (dict[str, Any]): Metadata of the file.
Returns:
Optional[str]: UUID of the file in hex format or None if not found.
"""
- fs_fsspec = self.get_fsspec_fs()
- metadata = fs_fsspec.stat(file_path)
+ if not metadata:
+ fs_fsspec = self.get_fsspec_fs()
+ metadata = fs_fsspec.stat(file_path)
+
file_hash = self.extract_metadata_file_hash(metadata)
if not file_hash:
logger.error(f"File hash not found for {file_path}.")
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
index 576fd3f54..306357185 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
@@ -80,6 +80,17 @@ class DropboxFS(UnstractFileSystem):
)
return None
+ def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
+ """Check if the given path is a directory.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ bool: True if the path is a directory, False otherwise.
+ """
+ return metadata.get("type") == "directory"
+
def test_credentials(self) -> bool:
"""To test credentials for Dropbox."""
try: