Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2901 [FIX] Prevent invalid status updates (EXECUTING/ERROR) from duplicate file processing runs (#1606)

alia219b2dced54
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index 7900ca22e..b12671f1c 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -861,10 +861,10 @@ class WorkflowFileExecutionData:
id: str | uuid.UUID
workflow_execution_id: str | uuid.UUID
file_name: str
- file_path: str
file_size: int
file_hash: str
status: str = ExecutionStatus.PENDING.value
+ file_path: str | None = None
provider_file_uuid: str | None = None
mime_type: str = ""
fs_metadata: dict[str, Any] = field(default_factory=dict)
@@ -883,8 +883,7 @@ class WorkflowFileExecutionData:
# Validate required fields
if not self.file_name:
raise ValueError("file_name is required")
- if not self.file_path:
- raise ValueError("file_path is required")
+ # file_path is optional - None for API workflows or when fetching status only
# file_hash can be empty initially - gets populated during file processing with SHA256 hash
def to_dict(self) -> dict[str, Any]:
@@ -904,7 +903,7 @@ class WorkflowFileExecutionData:
id=data["id"],
workflow_execution_id=data["workflow_execution_id"],
file_name=data["file_name"],
- file_path=data["file_path"],
+ file_path=data.get("file_path"),
file_size=data.get("file_size", 0),
file_hash=data["file_hash"],
status=data.get("status", ExecutionStatus.PENDING.value),
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
index c259a1858..f30f34584 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
@@ -160,6 +160,29 @@ class ToolSandboxHelper:
).total_seconds()
if not_found_duration < POLL_NOT_FOUND_GRACE_PERIOD:
+ # RACE CONDITION OPTIMIZATION: Check if another worker completed this file
+ # This avoids waiting full grace period when work is already done
+ tracker = FileExecutionStatusTracker()
+ current_tracker_data = tracker.get_data(
+ self.execution_id, file_execution_id
+ )
+ if (
+ current_tracker_data
+ and current_tracker_data.stage_status.stage
+ == FileExecutionStage.COMPLETED
+ ):
+ logger.warning(
+ f"File execution COMPLETED by another worker during NOT_FOUND grace period - "
+ f"duplicate run detected after {not_found_duration:.1f}s for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
+ )
+ # Break with ERROR to stop further processing (destination, finalization)
+ # This is not a real error - the file was successfully processed by another worker
+ response = self._create_run_response(
+ status=RunnerContainerRunStatus.ERROR,
+ error="File already completed by another worker (duplicate run avoided)",
+ )
+ break
+
logger.debug(
f"Within polling grace period ({not_found_duration:.1f}s/{POLL_NOT_FOUND_GRACE_PERIOD}s) - "
f"continuing poll for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
@@ -300,11 +323,6 @@ class ToolSandboxHelper:
file_execution_data=file_execution_data,
)
self._update_stage_status_for_tool_execution(file_execution_id, response)
- self._update_stage_status(
- status=FileExecutionStageStatus.IN_PROGRESS,
- stage=FileExecutionStage.FINALIZATION,
- file_execution_id=file_execution_id,
- )
elif stage.is_before(FileExecutionStage.TOOL_EXECUTION):
self._update_stage_status(
status=FileExecutionStageStatus.SUCCESS,
@@ -319,11 +337,6 @@ class ToolSandboxHelper:
retry_count=retry_count,
)
self._update_stage_status_for_tool_execution(file_execution_id, response)
- self._update_stage_status(
- status=FileExecutionStageStatus.IN_PROGRESS,
- stage=FileExecutionStage.FINALIZATION,
- file_execution_id=file_execution_id,
- )
else:
logger.warning(
f"File execution data stage {file_execution_data.stage_status.stage} is after tool execution for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
index 3a836d96e..ecf688686 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
@@ -128,6 +128,28 @@ class ExecutionFileHandler:
metadata_path = self.metadata_file
if not metadata_path:
raise FileMetadataJsonNotFound()
+
+ # Check if metadata file already exists - skip to avoid overwriting tool data
+ # This prevents Worker 2 from destroying tool_metadata written by Worker 1's tool container
+ try:
+ self.get_workflow_metadata()
+ logger.info(
+ f"Metadata file already exists for file_execution_id {file_execution_id}. "
+ f"Skipping creation to preserve tool data and avoid race conditions."
+ )
+ return # Exit early - don't touch existing file
+ except (FileMetadataJsonNotFound, FileNotFoundError):
+ # Normal case - metadata file doesn't exist yet, proceed with creation
+ logger.info(
+ f"Creating new metadata file for file_execution_id {file_execution_id}"
+ )
+ except Exception:
+ # Unexpected errors only (e.g., permission issues, S3 connection errors)
+ logger.exception(
+ f"Error checking metadata existence for file_execution_id {file_execution_id}. "
+ f"Proceeding with creation."
+ )
+
filename = os.path.basename(input_file_path)
content = {
MetaDataKey.SOURCE_NAME: filename,
diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.py
index 9e1f8c8ec..56e75afb3 100644
--- a/workers/file_processing/tasks.py
+++ b/workers/file_processing/tasks.py
@@ -487,7 +487,11 @@ def _refactored_pre_create_file_executions(
# This matches the backend's _pre_create_file_executions pattern for ALL workflow types
# Includes double safeguard: database check for active files
workflow_logger = context.metadata.get("workflow_logger")
- pre_created_file_executions, skipped_files = _pre_create_file_executions(
+ (
+ pre_created_file_executions,
+ skipped_already_completed,
+ skipped_active_duplicate,
+ ) = _pre_create_file_executions(
file_data=file_data,
files=files,
workflow_id=workflow_id,
@@ -500,12 +504,13 @@ def _refactored_pre_create_file_executions(
)
logger.info(
f"Pre-created {len(pre_created_file_executions)} WorkflowFileExecution records for {workflow_type} workflow "
- f"(skipped {len(skipped_files)} duplicates)"
+ f"(skipped {len(skipped_already_completed)} already completed, {len(skipped_active_duplicate)} active duplicates)"
)
# Add to metadata
context.pre_created_file_executions = pre_created_file_executions
- context.metadata["skipped_files"] = skipped_files
+ context.metadata["skipped_already_completed"] = skipped_already_completed
+ context.metadata["skipped_active_duplicate"] = skipped_active_duplicate
return context
@@ -527,7 +532,8 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDa
api_client = context.organization_context.api_client
workflow_execution = context.metadata["workflow_execution"]
pre_created_file_executions = context.pre_created_file_executions
- skipped_files = context.metadata.get("skipped_files", [])
+ skipped_already_completed = context.metadata.get("skipped_already_completed", [])
+ skipped_active_duplicate = context.metadata.get("skipped_active_duplicate", [])
result = context.metadata["result"]
successful_files_for_manual_review = context.metadata[
"successful_files_for_manual_review"
@@ -568,12 +574,20 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDa
file_path = file_hash_dict.get("file_path")
if provider_uuid and file_path:
file_identifier = f"{provider_uuid}:{file_path}"
- if file_identifier in skipped_files:
- # File was already handled in pre-creation (marked as ERROR in DB, logged to UI)
- # Just count it as a failure and move on
- logger.debug(
- f"File '{file_name}' (identifier: {file_identifier}) was skipped as duplicate - already counted as ERROR"
+ # Check both skip lists and handle differently
+ if file_identifier in skipped_already_completed:
+ # File already COMPLETED in this execution - duplicate prevention worked
+ logger.info(
+ f"File '{file_name}' already completed in this execution - skipping (not a failure)"
+ )
+ # Don't increment failure - file is done, duplicate prevention worked
+ continue
+ elif file_identifier in skipped_active_duplicate:
+ # File ACTIVE in different execution - user error (concurrent submission)
+ logger.info(
+ f"File '{file_name}' active in another execution - marked as ERROR (counted as failure)"
)
+ # Increment failure - this IS a user error (submitting same file to multiple executions)
result.increment_failure()
continue
@@ -710,6 +724,7 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDa
context.metadata.get(
"is_api_workflow", False
), # Pass existing API workflow detection
+ skipped_already_completed, # Pass list to track duplicate skips
)
# Update metadata with results
@@ -735,6 +750,7 @@ def _handle_file_processing_result(
file_execution_id: str,
celery_task_id: str,
is_api_workflow: bool,
+ skipped_already_completed: list,
) -> None:
"""Handle the result of individual file processing.
@@ -752,6 +768,7 @@ def _handle_file_processing_result(
file_execution_id: File execution ID
celery_task_id: Celery task ID for queue detection
is_api_workflow: Whether this is an API workflow (from existing detection)
+ skipped_already_completed: List to track files skipped as already completed
"""
# Handle null execution result
if file_execution_result is None:
@@ -769,9 +786,16 @@ def _handle_file_processing_result(
f"DUPLICATE SKIP: File '{file_name}' skipped as duplicate - another worker is processing it. "
f"execution_id={execution_id}, workflow_id={workflow_id}, "
f"file_execution_id={file_execution_id}, celery_task_id={celery_task_id}. "
- f"No DB status updates, no counter increments - silent skip to avoid interfering with active worker. "
+ f"No DB status updates, but counting in skipped_already_completed for accurate total_files. "
f"First worker will handle all status updates and counter increments."
)
+ # Add to skipped_already_completed so it's counted in total_files
+ file_identifier = f"{file_hash.provider_file_uuid}:{file_hash.file_path}"
+ if file_identifier not in skipped_already_completed:
+ skipped_already_completed.append(file_identifier)
+ logger.debug(
+ f"Added {file_name} to skipped_already_completed for total_files count"
+ )
# Exit early without any DB updates - the first worker will handle all updates
return
@@ -836,7 +860,9 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:
"""
result = context.metadata["result"]
workflow_logger = context.metadata.get("workflow_logger")
- skipped_files = context.metadata.get("skipped_files", [])
+ skipped_already_completed = context.metadata.get("skipped_already_completed", [])
+ skipped_active_duplicate = context.metadata.get("skipped_active_duplicate", [])
+ total_skipped = len(skipped_already_completed) + len(skipped_active_duplicate)
# Send execution completion summary to UI
if workflow_logger:
@@ -847,13 +873,14 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:
)
# Log batch completion summary
- # Note: Skipped duplicate files are marked as ERROR in database and counted in failed_files
- if skipped_files:
+ # Note: Only active duplicates counted in failed_files; already-completed not counted
+ if total_skipped > 0:
logger.info(
f"Function tasks.process_file_batch completed. "
f"Successful: {result.successful_files} files, "
- f"Failed: {result.failed_files} files "
- f"(including {len(skipped_files)} duplicate(s) marked as ERROR). "
+ f"Failed: {result.failed_files} files, "
+ f"Already completed: {len(skipped_already_completed)}, "
+ f"Active duplicates: {len(skipped_active_duplicate)}. "
f"Batch execution time: {result.execution_time:.2f}s"
)
else:
@@ -871,12 +898,17 @@ def _compile_batch_result(context: WorkflowContextData) -> dict[str, Any]:
logger.warning(f"Failed to cleanup StateStore context: {cleanup_error}")
# Return the final result matching Django backend format
- # Note: Skipped duplicate files are in database as ERROR status, not counted separately
+ # Note: Only active duplicates count as failures; already-completed do not
return {
"successful_files": result.successful_files,
- "failed_files": result.failed_files, # Includes duplicates marked as ERROR
- "total_files": result.successful_files + result.failed_files,
- "skipped_duplicates": len(skipped_files), # For informational purposes
+ "failed_files": result.failed_files, # Includes active duplicates (user error)
+ "total_files": result.successful_files
+ + result.failed_files
+ + len(skipped_already_completed), # Include all files in batch
+ "skipped_already_completed": len(skipped_already_completed), # Not a failure
+ "skipped_active_duplicate": len(
+ skipped_active_duplicate
+ ), # IS a failure (counted above)
"execution_time": result.execution_time,
"organization_id": context.organization_context.organization_id,
}
@@ -1039,7 +1071,7 @@ def _pre_create_file_executions(
is_api: bool = False,
use_file_history: bool = True,
workflow_logger: WorkerWorkflowLogger | None = None,
-) -> tuple[dict[str, Any], list[str]]:
+) -> tuple[dict[str, Any], list[str], list[str]]:
"""Pre-create WorkflowFileExecution records with PENDING status to prevent race conditions.
This matches the backend's _pre_create_file_executions pattern for ALL workflow types
@@ -1059,12 +1091,14 @@ def _pre_create_file_executions(
workflow_logger: Workflow logger for UI logging (optional)
Returns:
- Tuple of (pre_created_data dict, skipped_files list)
+ Tuple of (pre_created_data dict, skipped_already_completed list, skipped_active_duplicate list)
- pre_created_data: Dict mapping file names to PreCreatedFileData
- - skipped_files: List of file identifiers (uuid:path) that were skipped (already active)
+ - skipped_already_completed: Files already COMPLETED in this execution (not a failure)
+ - skipped_active_duplicate: Files ACTIVE in different execution (IS a failure - user error)
"""
pre_created_data: dict[str, PreCreatedFileData] = {}
- skipped_files: list[str] = [] # List of identifiers (uuid:path), not file names
+ skipped_already_completed: list[str] = [] # Already done in this execution
+ skipped_active_duplicate: list[str] = [] # Active in different execution (user error)
# Use the file history flag passed from execution parameters
logger.info(
@@ -1120,6 +1154,21 @@ def _pre_create_file_executions(
force_create=not use_file_history, # Force create when file history is disabled
)
+ # EARLY EXIT: Skip if file already COMPLETED in this execution
+ # This catches race conditions where Worker 1 completes before Worker 2's pre-create runs
+ # Only checks current execution_id (respects file_history cleanup for different executions)
+ if (
+ hasattr(workflow_file_execution, "status")
+ and workflow_file_execution.status == ExecutionStatus.COMPLETED.value
+ ):
+ logger.info(
+ f"File '{file_name}' already COMPLETED in execution {execution_id} "
+ f"(file_execution_id: {workflow_file_execution.id}). Skipping duplicate task creation."
+ )
+ file_identifier = f"{file_hash.provider_file_uuid}:{file_hash.file_path}"
+ skipped_already_completed.append(file_identifier)
+ continue # Skip to next file
+
# Handle duplicate files vs normal files differently
if is_duplicate:
# DUPLICATE DETECTED: Mark as ERROR with explanation for audit trail
@@ -1156,7 +1205,7 @@ def _pre_create_file_executions(
file_identifier = (
f"{file_hash.provider_file_uuid}:{file_hash.file_path}"
)
- skipped_files.append(file_identifier)
+ skipped_active_duplicate.append(file_identifier)
except Exception as status_error:
logger.error(
@@ -1204,24 +1253,38 @@ def _pre_create_file_executions(
# File history deduplication now handled during individual file processing
# Log summary if files were skipped
- if skipped_files:
+ total_skipped = len(skipped_already_completed) + len(skipped_active_duplicate)
+ if total_skipped > 0:
logger.warning(
- f"Skipped {len(skipped_files)} duplicate file(s) already being processed: {skipped_files}"
+ f"Skipped {total_skipped} duplicate file(s): "
+ f"{len(skipped_already_completed)} already completed, "
+ f"{len(skipped_active_duplicate)} active in other executions"
)
- if workflow_logger:
+
+ # Server-side log for already completed (internal detail, not shown to user)
+ if skipped_already_completed:
+ logger.info(
+ f"{len(skipped_already_completed)} file(s) already completed in this execution "
+ f"(internal duplicate prevention - not shown to user)"
+ )
+
+ # UI log for active duplicates (user error, SHOULD be shown to user)
+ if workflow_logger and skipped_active_duplicate:
from shared.infrastructure.logging.helpers import log_file_info
log_file_info(
workflow_logger,
None,
- f"⏭️ Total {len(skipped_files)} duplicate file(s) skipped (already active in other executions)",
+ f"⏭️ {len(skipped_active_duplicate)} duplicate file(s) active in other executions (marked as ERROR)",
)
logger.info(
- f"Pre-created {len(pre_created_data)} file execution(s), skipped {len(skipped_files)} duplicate(s)"
+ f"Pre-created {len(pre_created_data)} file execution(s), "
+ f"skipped {len(skipped_already_completed)} already completed, "
+ f"{len(skipped_active_duplicate)} active duplicates"
)
- return pre_created_data, skipped_files
+ return pre_created_data, skipped_already_completed, skipped_active_duplicate
def _create_file_hash_from_dict(
@@ -1569,6 +1632,32 @@ def _process_single_file_api(
logger.info(f"Processing file: {file_name} (execution: {file_execution_id})")
+ # RACE CONDITION PROTECTION: Check if already completed before updating to EXECUTING
+ try:
+ workflow_file_execution = api_client.get_workflow_file_execution(
+ file_execution_id
+ )
+ if workflow_file_execution.status == ExecutionStatus.COMPLETED.value:
+ logger.info(
+ f"API path: File '{file_name}' already COMPLETED by another worker. "
+ f"Skipping processing for execution_id: {execution_id}, "
+ f"file_execution_id: {file_execution_id}"
+ )
+ return {
+ "file_execution_id": file_execution_id,
+ "file_name": file_name,
+ "status": "completed",
+ "processing_time": 0.0,
+ "result_data": getattr(workflow_file_execution, "result", None),
+ "metadata": getattr(workflow_file_execution, "metadata", None) or {},
+ "skipped": "already_completed",
+ }
+ except Exception as e:
+ logger.exception(
+ f"API path: Failed to validate completion status for {file_execution_id}: {e}. "
+ "Proceeding with processing."
+ )
+
# Update file execution status to EXECUTING when processing starts (using common method)
api_client.update_file_status_to_executing(file_execution_id, file_name)
diff --git a/workers/shared/clients/file_client.py b/workers/shared/clients/file_client.py
index 92d0f350a..ccb918be4 100644
--- a/workers/shared/clients/file_client.py
+++ b/workers/shared/clients/file_client.py
@@ -16,7 +16,6 @@ import uuid
from typing import Any
from uuid import UUID
-from ..constants.api_endpoints import build_internal_endpoint
from ..data.models import (
APIResponse,
BatchOperationRequest,
@@ -49,6 +48,10 @@ except ImportError:
def from_dict(cls, data):
return cls(**data)
+ def has_hash(self):
+ # Mock implementation - check if file_hash attribute exists and is non-empty
+ return bool(getattr(self, "file_hash", None))
+
def ensure_hash(self):
# Mock implementation - no hash computation needed for testing
pass
@@ -89,10 +92,10 @@ class FileAPIClient(BaseAPIClient):
WorkflowFileExecutionData instance
"""
# Build URL for file execution detail endpoint
- url = build_internal_endpoint(f"file-execution/{file_execution_id}/").lstrip("/")
+ url = self._build_url("file_execution", f"{file_execution_id}/")
# Get the file execution record
- response_data = self.client.get(url, organization_id=organization_id)
+ response_data = self.get(url, organization_id=organization_id)
# Convert to WorkflowFileExecutionData
return WorkflowFileExecutionData.from_dict(response_data)
diff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.py
index d49f3087a..ab9abf4a9 100644
--- a/workers/shared/processing/files/processor.py
+++ b/workers/shared/processing/files/processor.py
@@ -181,8 +181,11 @@ class WorkflowFileExecutionHandler:
f"Using pre-created workflow file execution: {context.workflow_file_execution_id}"
)
- # workflow_file_execution_object is guaranteed to be truthy (validated above)
- workflow_file_execution = context.workflow_file_execution_object
+ # RACE CONDITION FIX: Fetch fresh status from DB instead of using cached object
+ # This prevents late-arriving workers from checking stale data and overwriting COMPLETED status
+ workflow_file_execution = context.api_client.get_workflow_file_execution(
+ context.workflow_file_execution_id
+ )
# Check if file execution is already completed
if workflow_file_execution.status == ExecutionStatus.COMPLETED.value:
@@ -501,7 +504,29 @@ class FileProcessor:
f"🚀 Starting processing for file '{context.file_name}' ({current_file_idx + 1}/{total_files})",
)
+ # RACE CONDITION FIX: Validate file execution status BEFORE updating to EXECUTING
+ # This prevents late-arriving workers from overwriting COMPLETED status
+ log_file_info(
+ workflow_logger,
+ workflow_file_execution_id,
+ f"✅ Validating execution status for '{context.file_name}'",
+ )
+
+ completed_result = WorkflowFileExecutionHandler.validate_workflow_file_execution(
+ context
+ )
+ if completed_result:
+ logger.info(f"File already completed: {context.file_name}")
+ log_file_processing_success(
+ workflow_logger, workflow_file_execution_id, context.file_name
+ )
+ # Return early with duplicate skip flag to signal no further processing needed
+ completed_result.is_duplicate_skip = True
+ return completed_result
+
# Update file execution status to EXECUTING when processing starts (using common method)
+ # This only happens if validation passed (file not already completed)
+ # Note: Redis-based race condition checks happen at TOOL_EXECUTION and DESTINATION_PROCESSING stages
context.api_client.update_file_status_to_executing(
context.workflow_file_execution_id, context.file_name
)
@@ -553,31 +578,15 @@ class FileProcessor:
return cached_result
- # Step 2: Validate workflow file execution
- log_file_info(
- workflow_logger,
- workflow_file_execution_id,
- f"✅ Validating execution status for '{context.file_name}'",
- )
-
- completed_result = (
- WorkflowFileExecutionHandler.validate_workflow_file_execution(context)
- )
- if completed_result:
- logger.info(f"File already completed: {context.file_name}")
- log_file_processing_success(
- workflow_logger, workflow_file_execution_id, context.file_name
- )
- return completed_result
-
- # Step 3: Check file history (if enabled)
+ # Step 2: Check file history (if enabled)
+ # Note: Validation already done before status update to prevent race conditions
log_file_info(
workflow_logger,
workflow_file_execution_id,
f"📜 Checking processing history for '{context.file_name}'",
)
- # Step 4: Execute workflow processing (always run tools first)
+ # Step 3: Execute workflow processing (always run tools first)
log_file_info(
workflow_logger,
workflow_file_execution_id,
diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py
index c64bf3a9b..1764abca2 100644
--- a/workers/shared/workflow/execution/service.py
+++ b/workers/shared/workflow/execution/service.py
@@ -125,16 +125,16 @@ class WorkerWorkflowExecutionService:
error=None,
result="Already completed",
metadata=WorkflowExecutionMetadata(
- total_execution_time=0,
- workflow_success=True,
- execution_error=None,
- tool_instances_data=[],
- destination_result=FinalOutputResult(
- output="Already completed",
- metadata={},
- error=None,
- processed=True,
- ),
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ execution_time=0.0,
+ tool_count=0,
+ workflow_executed=True,
+ destination_processed=False,
+ destination_error=None,
+ workflow_type=None,
+ cached=False,
+ cache_key=None,
),
execution_time=0,
)
@@ -201,6 +201,71 @@ class WorkerWorkflowExecutionService:
execution_error = str(e)
workflow_success = False
+ # Step 2.5: Check for duplicate execution BEFORE destination processing
+ # This prevents DB overwrites and duplicate processing when multiple workers
+ # pick up the same file from the queue (race condition)
+ skip_destination_processing = False
+ if workflow_file_execution_id:
+ try:
+ tracker = FileExecutionStatusTracker()
+ existing_data = tracker.get_data(execution_id, workflow_file_execution_id)
+
+ if existing_data:
+ current_stage = existing_data.stage_status.stage
+ current_status = existing_data.stage_status.status
+
+ # Case 1: COMPLETED stage - Full duplicate, skip everything
+ if current_stage == FileExecutionStage.COMPLETED:
+ logger.info(
+ f"DUPLICATE SKIP: File {file_name} already at COMPLETED stage. "
+ f"Another worker completed this file. Skipping all processing and DB updates."
+ )
+ return WorkflowExecutionResult(
+ file_name=file_name,
+ file_execution_id=workflow_file_execution_id,
+ success=True,
+ error=None,
+ result="Duplicate - already completed by another worker",
+ metadata=WorkflowExecutionMetadata(
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ execution_time=0.0,
+ tool_count=0,
+ workflow_executed=True,
+ destination_processed=False,
+ destination_error=None,
+ workflow_type=None,
+ cached=False,
+ cache_key=None,
+ ),
+ execution_time=0,
+ )
+
+ # Case 2: DESTINATION_PROCESSING with SUCCESS - Resume to finalization
+ if (
+ current_stage == FileExecutionStage.DESTINATION_PROCESSING
+ and current_status == FileExecutionStageStatus.SUCCESS
+ ):
+ logger.info(
+ f"File {file_name} destination already processed successfully. "
+ f"Skipping destination processing, using existing data."
+ )
+ # Skip destination processing, use existing result
+ destination_result = FinalOutputResult(
+ output="Destination already processed",
+ metadata={},
+ error=existing_data.error, # Preserve any error from first worker
+ processed=False, # Not processed by THIS worker
+ )
+ # Set flag to skip destination call below
+ skip_destination_processing = True
+
+ except Exception as tracker_check_error:
+ logger.warning(
+ f"Failed to check for duplicate execution: {tracker_check_error}"
+ )
+ # Continue with normal processing if check fails
+
# Step 3: Process Output - Let destination handle EVERYTHING
# This includes:
# - Extracting tool results via get_tool_execution_result_from_execution_context
@@ -208,70 +273,74 @@ class WorkerWorkflowExecutionService:
# - Writing to filesystem/database
# - Routing to manual review
# - Creating file history
- destination_result = None
+ if not skip_destination_processing:
+ destination_result = None
destination_start_time = time.time()
- logger.info(
- f"TIMING: Destination processing START for {file_name} at {destination_start_time:.6f}"
- )
- try:
- destination_result = self._handle_destination_processing(
- file_processing_context=file_processing_context,
- workflow=workflow_context,
- workflow_id=workflow_id,
- execution_id=execution_id,
- is_success=workflow_success,
- workflow_file_execution_id=workflow_file_execution_id,
- organization_id=organization_id,
- workflow_logger=workflow_logger,
- use_file_history=use_file_history,
- is_api=is_api,
- execution_error=execution_error,
+ if not skip_destination_processing:
+ logger.info(
+ f"TIMING: Destination processing START for {file_name} at {destination_start_time:.6f}"
)
- logger.info(f"Destination processing completed for {file_name}")
-
- # Mark destination processing as successful - only if workflow succeeded AND no destination errors AND actually processed (not duplicate)
- if (
- workflow_file_execution_id
- and workflow_success
- and destination_result
- and not destination_result.error
- and getattr(
- destination_result, "processed", True
- ) # Only update if actually processed (not a duplicate skip)
- ):
- try:
- tracker = FileExecutionStatusTracker()
- tracker.update_stage_status(
- execution_id=execution_id,
- file_execution_id=workflow_file_execution_id,
- stage_status=FileExecutionStageData(
- stage=FileExecutionStage.DESTINATION_PROCESSING,
- status=FileExecutionStageStatus.SUCCESS,
- ),
- )
- logger.info(
- f"Marked destination processing as successful for {file_name}"
- )
- except Exception as tracker_error:
- logger.warning(
- f"Failed to mark destination processing success: {tracker_error}"
- )
+ if not skip_destination_processing:
+ try:
+ destination_result = self._handle_destination_processing(
+ file_processing_context=file_processing_context,
+ workflow=workflow_context,
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ is_success=workflow_success,
+ workflow_file_execution_id=workflow_file_execution_id,
+ organization_id=organization_id,
+ workflow_logger=workflow_logger,
+ use_file_history=use_file_history,
+ is_api=is_api,
+ execution_error=execution_error,
+ )
+ logger.info(f"Destination processing completed for {file_name}")
- except Exception as dest_error:
- logger.error(
- f"Destination processing failed for {file_name}: {dest_error}",
- exc_info=True,
- )
- destination_result = FinalOutputResult(
- output=None, metadata=None, error=str(dest_error)
- )
- finally:
- destination_end_time = time.time()
- logger.info(
- f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)"
- )
+ # Mark destination processing as successful - only if workflow succeeded AND no destination errors AND actually processed (not duplicate)
+ if (
+ workflow_file_execution_id
+ and workflow_success
+ and destination_result
+ and not destination_result.error
+ and getattr(
+ destination_result, "processed", True
+ ) # Only update if actually processed (not a duplicate skip)
+ ):
+ try:
+ tracker = FileExecutionStatusTracker()
+ tracker.update_stage_status(
+ execution_id=execution_id,
+ file_execution_id=workflow_file_execution_id,
+ stage_status=FileExecutionStageData(
+ stage=FileExecutionStage.DESTINATION_PROCESSING,
+ status=FileExecutionStageStatus.SUCCESS,
+ ),
+ )
+ logger.info(
+ f"Marked destination processing as successful for {file_name}"
+ )
+
+ except Exception as tracker_error:
+ logger.warning(
+ f"Failed to mark destination processing success: {tracker_error}"
+ )
+
+ except Exception as dest_error:
+ logger.error(
+ f"Destination processing failed for {file_name}: {dest_error}",
+ exc_info=True,
+ )
+ destination_result = FinalOutputResult(
+ output=None, metadata=None, error=str(dest_error)
+ )
+ finally:
+ destination_end_time = time.time()
+ logger.info(
+ f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)"
+ )
# Step 4: Build Final Result
final_time = time.time()
@@ -699,7 +768,7 @@ class WorkerWorkflowExecutionService:
return False
# Step 2: Prepare input file and metadata
- self._prepare_workflow_input_file(
+ computed_hash = self._prepare_workflow_input_file(
execution_service=execution_service,
file_processing_context=file_processing_context,
workflow_id=workflow_id,
@@ -707,6 +776,61 @@ class WorkerWorkflowExecutionService:
workflow_file_execution_id=workflow_file_execution_id,
)
+ # Update file_hash object with computed hash if empty
+ # This handles the race condition where Worker 2 retrieves the hash from
+ # Worker 1's metadata instead of computing it fresh
+ if computed_hash and not file_processing_context.file_hash.file_hash:
+ logger.info(
+ f"Updating file_hash with computed hash: {computed_hash[:16]}... "
+ f"for {file_name}"
+ )
+ file_processing_context.file_hash.file_hash = computed_hash
+
+ # Step 2.5: Check if tool execution already completed (race condition handling)
+ # This prevents Worker 2 from re-running the tool when Worker 1's container
+ # already completed, which would cause output file conflicts and MIME type errors
+ if workflow_file_execution_id:
+ try:
+ tracker = FileExecutionStatusTracker()
+ existing_data = tracker.get_data(
+ execution_id, workflow_file_execution_id
+ )
+
+ if existing_data:
+ current_stage = existing_data.stage_status.stage
+ current_status = existing_data.stage_status.status
+
+ # If tool execution already completed successfully, skip re-running
+ if (
+ current_stage == FileExecutionStage.TOOL_EXECUTION
+ and current_status == FileExecutionStageStatus.SUCCESS
+ ):
+ logger.info(
+ f"Tool execution already completed successfully for {file_name}. "
+ f"Skipping duplicate tool run (race condition detected). "
+ f"Will proceed to destination processing with existing output."
+ )
+ return True # Workflow considered successful, proceed to destination
+
+ # Also skip if we're past tool execution stage
+ if current_stage in [
+ FileExecutionStage.DESTINATION_PROCESSING,
+ FileExecutionStage.FINALIZATION,
+ FileExecutionStage.COMPLETED,
+ ]:
+ logger.info(
+ f"File {file_name} already at stage {current_stage}. "
+ f"Skipping tool execution (duplicate worker detected)."
+ )
+ return True # Proceed to next stage
+
+ except Exception:
+ logger.exception(
+ f"Failed to check tool execution status for {file_name}. "
+ f"Proceeding with tool execution."
+ )
+ # Continue with normal execution if check fails
+
# Step 3: Build and execute workflow
self._build_and_execute_workflow(execution_service, file_name)
@@ -773,27 +897,38 @@ class WorkerWorkflowExecutionService:
logger.info(f"Copying source file {file_path} to execution directory")
- # Determine connection type and copy file
- connection_type = self._determine_connection_type(source_connection_type)
-
- if connection_type.is_api:
- computed_hash = self._copy_api_file(
- file_path=file_path,
- infile_path=infile_path,
- source_file_path=source_file_path,
- file_processing_context=file_processing_context,
- )
+ # Check if files already exist (race condition handling)
+ files_exist, existing_hash = self._check_existing_input_files(
+ infile_path,
+ source_file_path,
+ file_handler,
+ file_path,
+ )
+ if files_exist:
+ # Files already copied by another worker - use existing hash
+ computed_hash = existing_hash
else:
- computed_hash = self._copy_filesystem_file(
- file_path=file_path,
- infile_path=infile_path,
- source_file_path=source_file_path,
- file_processing_context=file_processing_context,
- source_connector_id=source_connector_id,
- source_config_connector_settings=source_config_connector_settings,
- connector_metadata=connector_metadata,
- )
+ # Files don't exist - proceed with normal copy
+ connection_type = self._determine_connection_type(source_connection_type)
+
+ if connection_type.is_api:
+ computed_hash = self._copy_api_file(
+ file_path=file_path,
+ infile_path=infile_path,
+ source_file_path=source_file_path,
+ file_processing_context=file_processing_context,
+ )
+ else:
+ computed_hash = self._copy_filesystem_file(
+ file_path=file_path,
+ infile_path=infile_path,
+ source_file_path=source_file_path,
+ file_processing_context=file_processing_context,
+ source_connector_id=source_connector_id,
+ source_config_connector_settings=source_config_connector_settings,
+ connector_metadata=connector_metadata,
+ )
# Create initial METADATA.json file
# Extract tag names from workflow execution context
@@ -1009,6 +1144,86 @@ class WorkerWorkflowExecutionService:
file_processing_context.file_hash.file_hash = computed_hash
return computed_hash
+ def _check_existing_input_files(
+ self,
+ infile_path: str,
+ source_file_path: str,
+ file_handler,
+ file_path: str,
+ ) -> tuple[bool, str | None]:
+ """Check if input files already exist and retrieve source hash.
+
+ This handles the race condition where Worker 2 picks up a task after Worker 1
+ has already copied files. Since metadata is created BEFORE tool execution,
+ if metadata doesn't exist, the tool hasn't run yet and files are safe to re-copy.
+
+ Args:
+ infile_path: Path to the infile (tool output location)
+ source_file_path: Path to the source file copy
+ file_handler: ExecutionFileHandler instance for metadata access
+ file_path: Original source file path (for logging)
+
+ Returns:
+ tuple: (files_exist: bool, source_hash: str | None)
+ - files_exist: True if both infile and source exist with metadata
+ - source_hash: Retrieved hash from metadata, or None if should re-copy
+ """
+ from unstract.filesystem import FileStorageType, FileSystem
+
+ workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ workflow_file_storage = workflow_file_system.get_file_storage()
+
+ # Check if both files exist
+ if not (
+ workflow_file_storage.exists(infile_path)
+ and workflow_file_storage.exists(source_file_path)
+ ):
+ return False, None
+
+ # Files exist - check if metadata exists too
+ logger.info(
+ f"Input files already exist for {file_path}. "
+ f"Checking metadata for hash (duplicate worker detected)."
+ )
+
+ # Try to retrieve source_hash from metadata
+ try:
+ existing_metadata = file_handler.get_workflow_metadata()
+ source_hash = existing_metadata.get("source_hash")
+
+ if not source_hash:
+ # This should never happen - source_hash is always set when metadata is created
+ raise ValueError("source_hash not found in metadata (unexpected)")
+
+ logger.info(
+ f"Retrieved source_hash from existing metadata: {source_hash}. "
+ f"Skipping file copy to preserve existing workflow state."
+ )
+ return True, source_hash
+
+ except FileNotFoundError as error:
+ # Metadata doesn't exist - Other Worker was killed before creating it
+ # Since metadata is created BEFORE tool execution:
+ # - Tool hasn't run yet
+ # - infile still contains original source (PDF)
+ # - Safe to re-copy
+ logger.warning(
+ f"Files exist but metadata not found: {error}. "
+ f"Other Worker likely killed between file copy and metadata creation. "
+ f"Tool hasn't executed yet (metadata created first). "
+ f"Safe to re-copy files."
+ )
+ return False, None
+
+ except Exception as error:
+ # Unexpected error (corruption, S3 connection, permissions, etc.)
+ # Fail safely - better to crash than risk overwriting tool output
+ logger.error(
+ f"Unexpected error reading metadata: {error}. "
+ f"Cannot determine if tool has executed. Failing safely."
+ )
+ raise
+
def _resolve_connector_config(
self,
source_connector_id: str | None,