LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2901 [FIX] Prevent invalid status updates (EXECUTING/ERROR) from duplicate file processing runs (#1606)
alia219b2dced54
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex 7900ca22e..b12671f1c 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -861,10 +861,10 @@ class WorkflowFileExecutionData:id: str | uuid.UUIDworkflow_execution_id: str | uuid.UUIDfile_name: str- file_path: strfile_size: intfile_hash: strstatus: str = ExecutionStatus.PENDING.value+ file_path: str | None = Noneprovider_file_uuid: str | None = Nonemime_type: str = ""fs_metadata: dict[str, Any] = field(default_factory=dict)@@ -883,8 +883,7 @@ class WorkflowFileExecutionData:# Validate required fieldsif not self.file_name:raise ValueError("file_name is required")- if not self.file_path:- raise ValueError("file_path is required")+ # file_path is optional - None for API workflows or when fetching status only# file_hash can be empty initially - gets populated during file processing with SHA256 hashdef to_dict(self) -> dict[str, Any]:@@ -904,7 +903,7 @@ class WorkflowFileExecutionData:id=data["id"],workflow_execution_id=data["workflow_execution_id"],file_name=data["file_name"],- file_path=data["file_path"],+ file_path=data.get("file_path"),file_size=data.get("file_size", 0),file_hash=data["file_hash"],status=data.get("status", ExecutionStatus.PENDING.value),diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.pyindex c259a1858..f30f34584 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py@@ -160,6 +160,29 @@ class ToolSandboxHelper:).total_seconds()if not_found_duration < POLL_NOT_FOUND_GRACE_PERIOD:+ # RACE CONDITION OPTIMIZATION: Check if another worker completed this file+ # This avoids waiting full grace period when work is already done+ tracker = FileExecutionStatusTracker()+ current_tracker_data = tracker.get_data(+ self.execution_id, file_execution_id+ )+ if (+ current_tracker_data+ and current_tracker_data.stage_status.stage+ == FileExecutionStage.COMPLETED+ ):+ logger.warning(+ f"File execution COMPLETED by another worker during NOT_FOUND grace period - "+ f"duplicate run detected after {not_found_duration:.1f}s for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"+ )+ # Break with ERROR to stop further processing (destination, finalization)+ # This is not a real error - the file was successfully processed by another worker+ response = self._create_run_response(+ status=RunnerContainerRunStatus.ERROR,+ error="File already completed by another worker (duplicate run avoided)",+ )+ break+logger.debug(f"Within polling grace period ({not_found_duration:.1f}s/{POLL_NOT_FOUND_GRACE_PERIOD}s) - "f"continuing poll for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"@@ -300,11 +323,6 @@ class ToolSandboxHelper:file_execution_data=file_execution_data,)self._update_stage_status_for_tool_execution(file_execution_id, response)- self._update_stage_status(- status=FileExecutionStageStatus.IN_PROGRESS,- stage=FileExecutionStage.FINALIZATION,- file_execution_id=file_execution_id,- )elif stage.is_before(FileExecutionStage.TOOL_EXECUTION):self._update_stage_status(status=FileExecutionStageStatus.SUCCESS,@@ -319,11 +337,6 @@ class ToolSandboxHelper:retry_count=retry_count,)self._update_stage_status_for_tool_execution(file_execution_id, response)- self._update_stage_status(- status=FileExecutionStageStatus.IN_PROGRESS,- stage=FileExecutionStage.FINALIZATION,- file_execution_id=file_execution_id,- )else:logger.warning(f"File execution data stage {file_execution_data.stage_status.stage} is after tool execution for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex 3a836d96e..ecf688686 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -128,6 +128,28 @@ class ExecutionFileHandler:metadata_path = self.metadata_fileif not metadata_path:raise FileMetadataJsonNotFound()++ # Check if metadata file already exists - skip to avoid overwriting tool data+ # This prevents Worker 2 from destroying tool_metadata written by Worker 1's tool container+ try:+ self.get_workflow_metadata()+ logger.info(+ f"Metadata file already exists for file_execution_id {file_execution_id}. "+ f"Skipping creation to preserve tool data and avoid race conditions."+ )+ return # Exit early - don't touch existing file+ except (FileMetadataJsonNotFound, FileNotFoundError):+ # Normal case - metadata file doesn't exist yet, proceed with creation+ logger.info(+ f"Creating new metadata file for file_execution_id {file_execution_id}"+ )+ except Exception:+ # Unexpected errors only (e.g., permission issues, S3 connection errors)+ logger.exception(+ f"Error checking metadata existence for file_execution_id {file_execution_id}. "+ f"Proceeding with creation."+ )+filename = os.path.basename(input_file_path)content = {MetaDataKey.SOURCE_NAME: filename,diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.pyindex 9e1f8c8ec..56e75afb3 100644--- a/workers/file_processing/tasks.py+++ b/workers/file_processing/tasks.py@@ -487,7 +487,11 @@ def _refactored_pre_create_file_executions(# This matches the backend's _pre_create_file_executions pattern for ALL workflow types# Includes double safeguard: database check for active filesworkflow_logger = context.metadata.get("workflow_logger")- pre_created_file_executions, skipped_files = _pre_create_file_executions(+ (+ pre_created_file_executions,+ skipped_already_completed,+ skipped_active_duplicate,+ ) = _pre_create_file_executions(file_data=file_data,files=files,workflow_id=workflow_id,@@ -500,12 +504,13 @@ def _refactored_pre_create_file_executions()logger.info(f"Pre-created {len(pre_created_file_executions)} WorkflowFileExecution records for {workflow_type} workflow "- f"(skipped {len(skipped_files)} duplicates)"+ f"(skipped {len(skipped_already_completed)} already completed, {len(skipped_active_duplicate)} active duplicates)")# Add to metadatacontext.pre_created_file_executions = pre_created_file_executions- context.metadata["skipped_files"] = skipped_files+ context.metadata["skipped_already_completed"] = skipped_already_completed+ context.metadata["skipped_active_duplicate"] = skipped_active_duplicatereturn context@@ -527,7 +532,8 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDaapi_client = context.organization_context.api_clientworkflow_execution = context.metadata["workflow_execution"]pre_created_file_executions = context.pre_created_file_executions- skipped_files = context.metadata.get("skipped_files", [])+ skipped_already_completed = context.metadata.get("skipped_already_completed", [])+ skipped_active_duplicate = context.metadata.get("skipped_active_duplicate", [])result = context.metadata["result"]successful_files_for_manual_review = context.metadata["successful_files_for_manual_review"@@ -568,12 +574,20 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDafile_path = file_hash_dict.get("file_path")if provider_uuid and file_path:file_identifier = f"{provider_uuid}:{file_path}"- if file_identifier in skipped_files:- # File was already handled in pre-creation (marked as ERROR in DB, logged to UI)- # Just count it as a failure and move on- logger.debug(- f"File '{file_name}' (identifier: {file_identifier}) was skipped as duplicate - already counted as ERROR"+ # Check both skip lists and handle differently+ if file_identifier in skipped_already_completed:+ # File already COMPLETED in this execution - duplicate prevention worked+ logger.info(+ f"File '{file_name}' already completed in this execution - skipping (not a failure)"+ )+ # Don't increment failure - file is done, duplicate prevention worked+ continue+ elif file_identifier in skipped_active_duplicate:+ # File ACTIVE in different execution - user error (concurrent submission)+ logger.info(+ f"File '{file_name}' active in another execution - marked as ERROR (counted as failure)")+ # Increment failure - this IS a user error (submitting same file to multiple executions)result.increment_failure()continue@@ -710,6 +724,7 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDacontext.metadata.get("is_api_workflow", False), # Pass existing API workflow detection+ skipped_already_completed, # Pass list to track duplicate skips)# Update metadata with results@@ -735,6 +750,7 @@ def _handle_file_processing_result(file_execution_id: str,celery_task_id: str,is_api_workflow: bool,+ skipped_already_completed: list,) -> None:"""Handle the result of individual file processing.@@ -752,6 +768,7 @@ def _handle_file_processing_result(file_execution_id: File execution IDcelery_task_id: Celery task ID for queue detectionis_api_workflow: Whether this is an API workflow (from existing detection)+ skipped_already_completed: List to track files skipped as already completed"""# Handle null execution resultif file_execution_result is None:@@ -769,9 +786,16 @@ def _handle_file_processing_result(f"DUPLICATE SKIP: File '{file_name}' skipped as duplicate - another worker is processing it. "f"execution_id={execution_id}, workflow_id={workflow_id}, "f"file_execution_id={file_execution_id}, celery_task_id={celery_task_id}. "- f"No DB status updates, no counter increments - silent skip to avoid interfering with active worker. "+ f"No DB status updates, but counting in skipped_already_completed for accurate total_files. "f"First worker will handle all status updates and counter increments.")+ # Add to skipped_already_completed so it's counted in total_files+ file_identifier = f"{file_hash.provider_file_uuid}:{file_hash.file_path}"+ if file_identifier not in skipped_already_completed:+ skipped_already_completed.append(file_identifier)+ logger.debug(+ f"Added {file_name} to skipped_already_completed for total_files count"+ )# Exit early without any DB updates - the first worker will handle all updatesreturn@@ -836,7 +860,9 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:"""result = context.metadata["result"]workflow_logger = context.metadata.get("workflow_logger")- skipped_files = context.metadata.get("skipped_files", [])+ skipped_already_completed = context.metadata.get("skipped_already_completed", [])+ skipped_active_duplicate = context.metadata.get("skipped_active_duplicate", [])+ total_skipped = len(skipped_already_completed) + len(skipped_active_duplicate)# Send execution completion summary to UIif workflow_logger:@@ -847,13 +873,14 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:)# Log batch completion summary- # Note: Skipped duplicate files are marked as ERROR in database and counted in failed_files- if skipped_files:+ # Note: Only active duplicates counted in failed_files; already-completed not counted+ if total_skipped > 0:logger.info(f"Function tasks.process_file_batch completed. "f"Successful: {result.successful_files} files, "- f"Failed: {result.failed_files} files "- f"(including {len(skipped_files)} duplicate(s) marked as ERROR). "+ f"Failed: {result.failed_files} files, "+ f"Already completed: {len(skipped_already_completed)}, "+ f"Active duplicates: {len(skipped_active_duplicate)}. "f"Batch execution time: {result.execution_time:.2f}s")else:@@ -871,12 +898,17 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:logger.warning(f"Failed to cleanup StateStore context: {cleanup_error}")# Return the final result matching Django backend format- # Note: Skipped duplicate files are in database as ERROR status, not counted separately+ # Note: Only active duplicates count as failures; already-completed do notreturn {"successful_files": result.successful_files,- "failed_files": result.failed_files, # Includes duplicates marked as ERROR- "total_files": result.successful_files + result.failed_files,- "skipped_duplicates": len(skipped_files), # For informational purposes+ "failed_files": result.failed_files, # Includes active duplicates (user error)+ "total_files": result.successful_files+ + result.failed_files+ + len(skipped_already_completed), # Include all files in batch+ "skipped_already_completed": len(skipped_already_completed), # Not a failure+ "skipped_active_duplicate": len(+ skipped_active_duplicate+ ), # IS a failure (counted above)"execution_time": result.execution_time,"organization_id": context.organization_context.organization_id,}@@ -1039,7 +1071,7 @@ def _pre_create_file_executions(is_api: bool = False,use_file_history: bool = True,workflow_logger: WorkerWorkflowLogger | None = None,-) -> tuple[dict[str, Any], list[str]]:+) -> tuple[dict[str, Any], list[str], list[str]]:"""Pre-create WorkflowFileExecution records with PENDING status to prevent race conditions.This matches the backend's _pre_create_file_executions pattern for ALL workflow types@@ -1059,12 +1091,14 @@ def _pre_create_file_executions(workflow_logger: Workflow logger for UI logging (optional)Returns:- Tuple of (pre_created_data dict, skipped_files list)+ Tuple of (pre_created_data dict, skipped_already_completed list, skipped_active_duplicate list)- pre_created_data: Dict mapping file names to PreCreatedFileData- - skipped_files: List of file identifiers (uuid:path) that were skipped (already active)+ - skipped_already_completed: Files already COMPLETED in this execution (not a failure)+ - skipped_active_duplicate: Files ACTIVE in different execution (IS a failure - user error)"""pre_created_data: dict[str, PreCreatedFileData] = {}- skipped_files: list[str] = [] # List of identifiers (uuid:path), not file names+ skipped_already_completed: list[str] = [] # Already done in this execution+ skipped_active_duplicate: list[str] = [] # Active in different execution (user error)# Use the file history flag passed from execution parameterslogger.info(@@ -1120,6 +1154,21 @@ def _pre_create_file_executions(force_create=not use_file_history, # Force create when file history is disabled)+ # EARLY EXIT: Skip if file already COMPLETED in this execution+ # This catches race conditions where Worker 1 completes before Worker 2's pre-create runs+ # Only checks current execution_id (respects file_history cleanup for different executions)+ if (+ hasattr(workflow_file_execution, "status")+ and workflow_file_execution.status == ExecutionStatus.COMPLETED.value+ ):+ logger.info(+ f"File '{file_name}' already COMPLETED in execution {execution_id} "+ f"(file_execution_id: {workflow_file_execution.id}). Skipping duplicate task creation."+ )+ file_identifier = f"{file_hash.provider_file_uuid}:{file_hash.file_path}"+ skipped_already_completed.append(file_identifier)+ continue # Skip to next file+# Handle duplicate files vs normal files differentlyif is_duplicate:# DUPLICATE DETECTED: Mark as ERROR with explanation for audit trail@@ -1156,7 +1205,7 @@ def _pre_create_file_executions(file_identifier = (f"{file_hash.provider_file_uuid}:{file_hash.file_path}")- skipped_files.append(file_identifier)+ skipped_active_duplicate.append(file_identifier)except Exception as status_error:logger.error(@@ -1204,24 +1253,38 @@ def _pre_create_file_executions(# File history deduplication now handled during individual file processing# Log summary if files were skipped- if skipped_files:+ total_skipped = len(skipped_already_completed) + len(skipped_active_duplicate)+ if total_skipped > 0:logger.warning(- f"Skipped {len(skipped_files)} duplicate file(s) already being processed: {skipped_files}"+ f"Skipped {total_skipped} duplicate file(s): "+ f"{len(skipped_already_completed)} already completed, "+ f"{len(skipped_active_duplicate)} active in other executions")- if workflow_logger:++ # Server-side log for already completed (internal detail, not shown to user)+ if skipped_already_completed:+ logger.info(+ f"{len(skipped_already_completed)} file(s) already completed in this execution "+ f"(internal duplicate prevention - not shown to user)"+ )++ # UI log for active duplicates (user error, SHOULD be shown to user)+ if workflow_logger and skipped_active_duplicate:from shared.infrastructure.logging.helpers import log_file_infolog_file_info(workflow_logger,None,- f"⏭️ Total {len(skipped_files)} duplicate file(s) skipped (already active in other executions)",+ f"⏭️ {len(skipped_active_duplicate)} duplicate file(s) active in other executions (marked as ERROR)",)logger.info(- f"Pre-created {len(pre_created_data)} file execution(s), skipped {len(skipped_files)} duplicate(s)"+ f"Pre-created {len(pre_created_data)} file execution(s), "+ f"skipped {len(skipped_already_completed)} already completed, "+ f"{len(skipped_active_duplicate)} active duplicates")- return pre_created_data, skipped_files+ return pre_created_data, skipped_already_completed, skipped_active_duplicatedef _create_file_hash_from_dict(@@ -1569,6 +1632,32 @@ def _process_single_file_api(logger.info(f"Processing file: {file_name} (execution: {file_execution_id})")+ # RACE CONDITION PROTECTION: Check if already completed before updating to EXECUTING+ try:+ workflow_file_execution = api_client.get_workflow_file_execution(+ file_execution_id+ )+ if workflow_file_execution.status == ExecutionStatus.COMPLETED.value:+ logger.info(+ f"API path: File '{file_name}' already COMPLETED by another worker. "+ f"Skipping processing for execution_id: {execution_id}, "+ f"file_execution_id: {file_execution_id}"+ )+ return {+ "file_execution_id": file_execution_id,+ "file_name": file_name,+ "status": "completed",+ "processing_time": 0.0,+ "result_data": getattr(workflow_file_execution, "result", None),+ "metadata": getattr(workflow_file_execution, "metadata", None) or {},+ "skipped": "already_completed",+ }+ except Exception as e:+ logger.exception(+ f"API path: Failed to validate completion status for {file_execution_id}: {e}. "+ "Proceeding with processing."+ )+# Update file execution status to EXECUTING when processing starts (using common method)api_client.update_file_status_to_executing(file_execution_id, file_name)diff --git a/workers/shared/clients/file_client.py b/workers/shared/clients/file_client.pyindex 92d0f350a..ccb918be4 100644--- a/workers/shared/clients/file_client.py+++ b/workers/shared/clients/file_client.py@@ -16,7 +16,6 @@ import uuidfrom typing import Anyfrom uuid import UUID-from ..constants.api_endpoints import build_internal_endpointfrom ..data.models import (APIResponse,BatchOperationRequest,@@ -49,6 +48,10 @@ except ImportError:def from_dict(cls, data):return cls(**data)+ def has_hash(self):+ # Mock implementation - check if file_hash attribute exists and is non-empty+ return bool(getattr(self, "file_hash", None))+def ensure_hash(self):# Mock implementation - no hash computation needed for testingpass@@ -89,10 +92,10 @@ class FileAPIClient(BaseAPIClient):WorkflowFileExecutionData instance"""# Build URL for file execution detail endpoint- url = build_internal_endpoint(f"file-execution/{file_execution_id}/").lstrip("/")+ url = self._build_url("file_execution", f"{file_execution_id}/")# Get the file execution record- response_data = self.client.get(url, organization_id=organization_id)+ response_data = self.get(url, organization_id=organization_id)# Convert to WorkflowFileExecutionDatareturn WorkflowFileExecutionData.from_dict(response_data)diff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.pyindex d49f3087a..ab9abf4a9 100644--- a/workers/shared/processing/files/processor.py+++ b/workers/shared/processing/files/processor.py@@ -181,8 +181,11 @@ class WorkflowFileExecutionHandler:f"Using pre-created workflow file execution: {context.workflow_file_execution_id}")- # workflow_file_execution_object is guaranteed to be truthy (validated above)- workflow_file_execution = context.workflow_file_execution_object+ # RACE CONDITION FIX: Fetch fresh status from DB instead of using cached object+ # This prevents late-arriving workers from checking stale data and overwriting COMPLETED status+ workflow_file_execution = context.api_client.get_workflow_file_execution(+ context.workflow_file_execution_id+ )# Check if file execution is already completedif workflow_file_execution.status == ExecutionStatus.COMPLETED.value:@@ -501,7 +504,29 @@ class FileProcessor:f"🚀 Starting processing for file '{context.file_name}' ({current_file_idx + 1}/{total_files})",)+ # RACE CONDITION FIX: Validate file execution status BEFORE updating to EXECUTING+ # This prevents late-arriving workers from overwriting COMPLETED status+ log_file_info(+ workflow_logger,+ workflow_file_execution_id,+ f"✅ Validating execution status for '{context.file_name}'",+ )++ completed_result = WorkflowFileExecutionHandler.validate_workflow_file_execution(+ context+ )+ if completed_result:+ logger.info(f"File already completed: {context.file_name}")+ log_file_processing_success(+ workflow_logger, workflow_file_execution_id, context.file_name+ )+ # Return early with duplicate skip flag to signal no further processing needed+ completed_result.is_duplicate_skip = True+ return completed_result+# Update file execution status to EXECUTING when processing starts (using common method)+ # This only happens if validation passed (file not already completed)+ # Note: Redis-based race condition checks happen at TOOL_EXECUTION and DESTINATION_PROCESSING stagescontext.api_client.update_file_status_to_executing(context.workflow_file_execution_id, context.file_name)@@ -553,31 +578,15 @@ class FileProcessor:return cached_result- # Step 2: Validate workflow file execution- log_file_info(- workflow_logger,- workflow_file_execution_id,- f"✅ Validating execution status for '{context.file_name}'",- )-- completed_result = (- WorkflowFileExecutionHandler.validate_workflow_file_execution(context)- )- if completed_result:- logger.info(f"File already completed: {context.file_name}")- log_file_processing_success(- workflow_logger, workflow_file_execution_id, context.file_name- )- return completed_result-- # Step 3: Check file history (if enabled)+ # Step 2: Check file history (if enabled)+ # Note: Validation already done before status update to prevent race conditionslog_file_info(workflow_logger,workflow_file_execution_id,f"📜 Checking processing history for '{context.file_name}'",)- # Step 4: Execute workflow processing (always run tools first)+ # Step 3: Execute workflow processing (always run tools first)log_file_info(workflow_logger,workflow_file_execution_id,diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex c64bf3a9b..1764abca2 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -125,16 +125,16 @@ class WorkerWorkflowExecutionService:error=None,result="Already completed",metadata=WorkflowExecutionMetadata(- total_execution_time=0,- workflow_success=True,- execution_error=None,- tool_instances_data=[],- destination_result=FinalOutputResult(- output="Already completed",- metadata={},- error=None,- processed=True,- ),+ workflow_id=workflow_id,+ execution_id=execution_id,+ execution_time=0.0,+ tool_count=0,+ workflow_executed=True,+ destination_processed=False,+ destination_error=None,+ workflow_type=None,+ cached=False,+ cache_key=None,),execution_time=0,)@@ -201,6 +201,71 @@ class WorkerWorkflowExecutionService:execution_error = str(e)workflow_success = False+ # Step 2.5: Check for duplicate execution BEFORE destination processing+ # This prevents DB overwrites and duplicate processing when multiple workers+ # pick up the same file from the queue (race condition)+ skip_destination_processing = False+ if workflow_file_execution_id:+ try:+ tracker = FileExecutionStatusTracker()+ existing_data = tracker.get_data(execution_id, workflow_file_execution_id)++ if existing_data:+ current_stage = existing_data.stage_status.stage+ current_status = existing_data.stage_status.status++ # Case 1: COMPLETED stage - Full duplicate, skip everything+ if current_stage == FileExecutionStage.COMPLETED:+ logger.info(+ f"DUPLICATE SKIP: File {file_name} already at COMPLETED stage. "+ f"Another worker completed this file. Skipping all processing and DB updates."+ )+ return WorkflowExecutionResult(+ file_name=file_name,+ file_execution_id=workflow_file_execution_id,+ success=True,+ error=None,+ result="Duplicate - already completed by another worker",+ metadata=WorkflowExecutionMetadata(+ workflow_id=workflow_id,+ execution_id=execution_id,+ execution_time=0.0,+ tool_count=0,+ workflow_executed=True,+ destination_processed=False,+ destination_error=None,+ workflow_type=None,+ cached=False,+ cache_key=None,+ ),+ execution_time=0,+ )++ # Case 2: DESTINATION_PROCESSING with SUCCESS - Resume to finalization+ if (+ current_stage == FileExecutionStage.DESTINATION_PROCESSING+ and current_status == FileExecutionStageStatus.SUCCESS+ ):+ logger.info(+ f"File {file_name} destination already processed successfully. "+ f"Skipping destination processing, using existing data."+ )+ # Skip destination processing, use existing result+ destination_result = FinalOutputResult(+ output="Destination already processed",+ metadata={},+ error=existing_data.error, # Preserve any error from first worker+ processed=False, # Not processed by THIS worker+ )+ # Set flag to skip destination call below+ skip_destination_processing = True++ except Exception as tracker_check_error:+ logger.warning(+ f"Failed to check for duplicate execution: {tracker_check_error}"+ )+ # Continue with normal processing if check fails+# Step 3: Process Output - Let destination handle EVERYTHING# This includes:# - Extracting tool results via get_tool_execution_result_from_execution_context@@ -208,70 +273,74 @@ class WorkerWorkflowExecutionService:# - Writing to filesystem/database# - Routing to manual review# - Creating file history- destination_result = None+ if not skip_destination_processing:+ destination_result = Nonedestination_start_time = time.time()- logger.info(- f"TIMING: Destination processing START for {file_name} at {destination_start_time:.6f}"- )- try:- destination_result = self._handle_destination_processing(- file_processing_context=file_processing_context,- workflow=workflow_context,- workflow_id=workflow_id,- execution_id=execution_id,- is_success=workflow_success,- workflow_file_execution_id=workflow_file_execution_id,- organization_id=organization_id,- workflow_logger=workflow_logger,- use_file_history=use_file_history,- is_api=is_api,- execution_error=execution_error,+ if not skip_destination_processing:+ logger.info(+ f"TIMING: Destination processing START for {file_name} at {destination_start_time:.6f}")- logger.info(f"Destination processing completed for {file_name}")-- # Mark destination processing as successful - only if workflow succeeded AND no destination errors AND actually processed (not duplicate)- if (- workflow_file_execution_id- and workflow_success- and destination_result- and not destination_result.error- and getattr(- destination_result, "processed", True- ) # Only update if actually processed (not a duplicate skip)- ):- try:- tracker = FileExecutionStatusTracker()- tracker.update_stage_status(- execution_id=execution_id,- file_execution_id=workflow_file_execution_id,- stage_status=FileExecutionStageData(- stage=FileExecutionStage.DESTINATION_PROCESSING,- status=FileExecutionStageStatus.SUCCESS,- ),- )- logger.info(- f"Marked destination processing as successful for {file_name}"- )- except Exception as tracker_error:- logger.warning(- f"Failed to mark destination processing success: {tracker_error}"- )+ if not skip_destination_processing:+ try:+ destination_result = self._handle_destination_processing(+ file_processing_context=file_processing_context,+ workflow=workflow_context,+ workflow_id=workflow_id,+ execution_id=execution_id,+ is_success=workflow_success,+ workflow_file_execution_id=workflow_file_execution_id,+ organization_id=organization_id,+ workflow_logger=workflow_logger,+ use_file_history=use_file_history,+ is_api=is_api,+ execution_error=execution_error,+ )+ logger.info(f"Destination processing completed for {file_name}")- except Exception as dest_error:- logger.error(- f"Destination processing failed for {file_name}: {dest_error}",- exc_info=True,- )- destination_result = FinalOutputResult(- output=None, metadata=None, error=str(dest_error)- )- finally:- destination_end_time = time.time()- logger.info(- f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)"- )+ # Mark destination processing as successful - only if workflow succeeded AND no destination errors AND actually processed (not duplicate)+ if (+ workflow_file_execution_id+ and workflow_success+ and destination_result+ and not destination_result.error+ and getattr(+ destination_result, "processed", True+ ) # Only update if actually processed (not a duplicate skip)+ ):+ try:+ tracker = FileExecutionStatusTracker()+ tracker.update_stage_status(+ execution_id=execution_id,+ file_execution_id=workflow_file_execution_id,+ stage_status=FileExecutionStageData(+ stage=FileExecutionStage.DESTINATION_PROCESSING,+ status=FileExecutionStageStatus.SUCCESS,+ ),+ )+ logger.info(+ f"Marked destination processing as successful for {file_name}"+ )++ except Exception as tracker_error:+ logger.warning(+ f"Failed to mark destination processing success: {tracker_error}"+ )++ except Exception as dest_error:+ logger.error(+ f"Destination processing failed for {file_name}: {dest_error}",+ exc_info=True,+ )+ destination_result = FinalOutputResult(+ output=None, metadata=None, error=str(dest_error)+ )+ finally:+ destination_end_time = time.time()+ logger.info(+ f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)"+ )# Step 4: Build Final Resultfinal_time = time.time()@@ -699,7 +768,7 @@ class WorkerWorkflowExecutionService:return False# Step 2: Prepare input file and metadata- self._prepare_workflow_input_file(+ computed_hash = self._prepare_workflow_input_file(execution_service=execution_service,file_processing_context=file_processing_context,workflow_id=workflow_id,@@ -707,6 +776,61 @@ class WorkerWorkflowExecutionService:workflow_file_execution_id=workflow_file_execution_id,)+ # Update file_hash object with computed hash if empty+ # This handles the race condition where Worker 2 retrieves the hash from+ # Worker 1's metadata instead of computing it fresh+ if computed_hash and not file_processing_context.file_hash.file_hash:+ logger.info(+ f"Updating file_hash with computed hash: {computed_hash[:16]}... "+ f"for {file_name}"+ )+ file_processing_context.file_hash.file_hash = computed_hash++ # Step 2.5: Check if tool execution already completed (race condition handling)+ # This prevents Worker 2 from re-running the tool when Worker 1's container+ # already completed, which would cause output file conflicts and MIME type errors+ if workflow_file_execution_id:+ try:+ tracker = FileExecutionStatusTracker()+ existing_data = tracker.get_data(+ execution_id, workflow_file_execution_id+ )++ if existing_data:+ current_stage = existing_data.stage_status.stage+ current_status = existing_data.stage_status.status++ # If tool execution already completed successfully, skip re-running+ if (+ current_stage == FileExecutionStage.TOOL_EXECUTION+ and current_status == FileExecutionStageStatus.SUCCESS+ ):+ logger.info(+ f"Tool execution already completed successfully for {file_name}. "+ f"Skipping duplicate tool run (race condition detected). "+ f"Will proceed to destination processing with existing output."+ )+ return True # Workflow considered successful, proceed to destination++ # Also skip if we're past tool execution stage+ if current_stage in [+ FileExecutionStage.DESTINATION_PROCESSING,+ FileExecutionStage.FINALIZATION,+ FileExecutionStage.COMPLETED,+ ]:+ logger.info(+ f"File {file_name} already at stage {current_stage}. "+ f"Skipping tool execution (duplicate worker detected)."+ )+ return True # Proceed to next stage++ except Exception:+ logger.exception(+ f"Failed to check tool execution status for {file_name}. "+ f"Proceeding with tool execution."+ )+ # Continue with normal execution if check fails+# Step 3: Build and execute workflowself._build_and_execute_workflow(execution_service, file_name)@@ -773,27 +897,38 @@ class WorkerWorkflowExecutionService:logger.info(f"Copying source file {file_path} to execution directory")- # Determine connection type and copy file- connection_type = self._determine_connection_type(source_connection_type)-- if connection_type.is_api:- computed_hash = self._copy_api_file(- file_path=file_path,- infile_path=infile_path,- source_file_path=source_file_path,- file_processing_context=file_processing_context,- )+ # Check if files already exist (race condition handling)+ files_exist, existing_hash = self._check_existing_input_files(+ infile_path,+ source_file_path,+ file_handler,+ file_path,+ )+ if files_exist:+ # Files already copied by another worker - use existing hash+ computed_hash = existing_hashelse:- computed_hash = self._copy_filesystem_file(- file_path=file_path,- infile_path=infile_path,- source_file_path=source_file_path,- file_processing_context=file_processing_context,- source_connector_id=source_connector_id,- source_config_connector_settings=source_config_connector_settings,- connector_metadata=connector_metadata,- )+ # Files don't exist - proceed with normal copy+ connection_type = self._determine_connection_type(source_connection_type)++ if connection_type.is_api:+ computed_hash = self._copy_api_file(+ file_path=file_path,+ infile_path=infile_path,+ source_file_path=source_file_path,+ file_processing_context=file_processing_context,+ )+ else:+ computed_hash = self._copy_filesystem_file(+ file_path=file_path,+ infile_path=infile_path,+ source_file_path=source_file_path,+ file_processing_context=file_processing_context,+ source_connector_id=source_connector_id,+ source_config_connector_settings=source_config_connector_settings,+ connector_metadata=connector_metadata,+ )# Create initial METADATA.json file# Extract tag names from workflow execution context@@ -1009,6 +1144,86 @@ class WorkerWorkflowExecutionService:file_processing_context.file_hash.file_hash = computed_hashreturn computed_hash+ def _check_existing_input_files(+ self,+ infile_path: str,+ source_file_path: str,+ file_handler,+ file_path: str,+ ) -> tuple[bool, str | None]:+ """Check if input files already exist and retrieve source hash.++ This handles the race condition where Worker 2 picks up a task after Worker 1+ has already copied files. Since metadata is created BEFORE tool execution,+ if metadata doesn't exist, the tool hasn't run yet and files are safe to re-copy.++ Args:+ infile_path: Path to the infile (tool output location)+ source_file_path: Path to the source file copy+ file_handler: ExecutionFileHandler instance for metadata access+ file_path: Original source file path (for logging)++ Returns:+ tuple: (files_exist: bool, source_hash: str | None)+ - files_exist: True if both infile and source exist with metadata+ - source_hash: Retrieved hash from metadata, or None if should re-copy+ """+ from unstract.filesystem import FileStorageType, FileSystem++ workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ workflow_file_storage = workflow_file_system.get_file_storage()++ # Check if both files exist+ if not (+ workflow_file_storage.exists(infile_path)+ and workflow_file_storage.exists(source_file_path)+ ):+ return False, None++ # Files exist - check if metadata exists too+ logger.info(+ f"Input files already exist for {file_path}. "+ f"Checking metadata for hash (duplicate worker detected)."+ )++ # Try to retrieve source_hash from metadata+ try:+ existing_metadata = file_handler.get_workflow_metadata()+ source_hash = existing_metadata.get("source_hash")++ if not source_hash:+ # This should never happen - source_hash is always set when metadata is created+ raise ValueError("source_hash not found in metadata (unexpected)")++ logger.info(+ f"Retrieved source_hash from existing metadata: {source_hash}. "+ f"Skipping file copy to preserve existing workflow state."+ )+ return True, source_hash++ except FileNotFoundError as error:+ # Metadata doesn't exist - Other Worker was killed before creating it+ # Since metadata is created BEFORE tool execution:+ # - Tool hasn't run yet+ # - infile still contains original source (PDF)+ # - Safe to re-copy+ logger.warning(+ f"Files exist but metadata not found: {error}. "+ f"Other Worker likely killed between file copy and metadata creation. "+ f"Tool hasn't executed yet (metadata created first). "+ f"Safe to re-copy files."+ )+ return False, None++ except Exception as error:+ # Unexpected error (corruption, S3 connection, permissions, etc.)+ # Fail safely - better to crash than risk overwriting tool output+ logger.error(+ f"Unexpected error reading metadata: {error}. "+ f"Cannot determine if tool has executed. Failing safely."+ )+ raise+def _resolve_connector_config(self,source_connector_id: str | None,