LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
prevent status update from duplicate tasks
alicb37fe3e1d9f
diff --git a/unstract/core/src/unstract/core/worker_models.py b/unstract/core/src/unstract/core/worker_models.pyindex af81cea72..c53254e04 100644--- a/unstract/core/src/unstract/core/worker_models.py+++ b/unstract/core/src/unstract/core/worker_models.py@@ -217,12 +217,14 @@ class FinalOutputResult:output: Any | Nonemetadata: dict[str, Any] | Noneerror: str | None+ processed: bool = True # False if duplicate detected and skippeddef to_dict(self) -> dict[str, Any]:return {"output": self.output,"metadata": self.metadata,"error": self.error,+ "processed": self.processed,}@classmethod@@ -231,6 +233,7 @@ class FinalOutputResult:output=data.get("output"),metadata=data.get("metadata"),error=data.get("error"),+ processed=data.get("processed", True),)diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.pyindex fcbbb5582..83f123806 100644--- a/workers/shared/workflow/destination_connector.py+++ b/workers/shared/workflow/destination_connector.py@@ -674,23 +674,22 @@ class WorkerDestinationConnector:if self.hitl_queue_name:logger.debug(f"HITL queue configured: {self.hitl_queue_name}")- # Extract processing data- result = self._extract_processing_data(exec_ctx, file_ctx)-- # Check if destination already processed and atomically acquire lock+ # Check if destination already processed and atomically acquire lock FIRST# This prevents duplicate insertions during warm shutdown scenarios+ # IMPORTANT: Check lock BEFORE extracting data to avoid unnecessary worklock_acquired = self._check_and_acquire_destination_lock(exec_ctx, file_ctx)if not lock_acquired:- # Duplicate detected or another worker has the lock - skip processing+ # Duplicate detected or another worker has the lock - abort ALL processinglogger.info(- f"Skipping destination processing for file '{file_ctx.file_name}' - "- f"already processed or being processed by another worker"- )- return HandleOutputResult(- output=result.tool_execution_result,- metadata=result.metadata,- connection_type=self.connection_type,+ f"Duplicate detected for file '{file_ctx.file_name}' - "+ f"aborting ALL processing (lock not acquired, already processed or being processed by another worker)")+ # Return None to signal to caller that this is a duplicate skip+ # Caller should not create file history, update stages, or clean up locks+ return None++ # Only extract data if lock was acquired (not a duplicate)+ result = self._extract_processing_data(exec_ctx, file_ctx)# Check and handle HITL if neededresult.has_hitl = self._check_and_handle_hitl(exec_ctx, file_ctx, result)diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex abf267045..19dce7be0 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -230,12 +230,15 @@ class WorkerWorkflowExecutionService:)logger.info(f"Destination processing completed for {file_name}")- # Mark destination processing as successful - only if workflow succeeded AND no destination errors+ # Mark destination processing as successful - only if workflow succeeded AND no destination errors AND actually processed (not duplicate)if (workflow_file_execution_idand workflow_successand destination_resultand not destination_result.error+ and getattr(+ destination_result, "processed", True+ ) # Only update if actually processed (not a duplicate skip)):try:tracker = FileExecutionStatusTracker()@@ -270,8 +273,15 @@ class WorkerWorkflowExecutionService:f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)")- # Always clean up the destination processing lock, whether success or failure- if workflow_file_execution_id:+ # Only clean up lock if we actually processed (not a duplicate skip)+ # CRITICAL: Do not delete lock if we skipped due to duplicate - another worker might hold it!+ if (+ workflow_file_execution_id+ and destination_result+ and getattr(+ destination_result, "processed", True+ ) # Only cleanup if we actually processed+ ):try:tracker = FileExecutionStatusTracker()lock_key = tracker.get_destination_lock_key(@@ -285,6 +295,14 @@ class WorkerWorkflowExecutionService:logger.warning(f"Failed to clean up destination lock: {lock_cleanup_error}")+ elif (+ workflow_file_execution_id+ and destination_result+ and not getattr(destination_result, "processed", True)+ ):+ logger.info(+ f"Skipping lock cleanup for '{file_name}' - duplicate detected, lock held by another worker"+ )# Step 4: Build Final Resultfinal_time = time.time()@@ -330,7 +348,15 @@ class WorkerWorkflowExecutionService:try:tracker = FileExecutionStatusTracker()overall_success = workflow_success and (- destination_result and not destination_result.error+ destination_result+ and not destination_result.error+ and getattr(destination_result, "processed", True)+ )++ # Check if this was a duplicate skip (processed=False)+ # For duplicate skips, we don't update any stages to avoid interfering with active worker+ is_duplicate_skip = destination_result and not getattr(+ destination_result, "processed", True)# Stage flow: DESTINATION_PROCESSING(2) → FINALIZATION(3) → COMPLETED(4)@@ -362,8 +388,8 @@ class WorkerWorkflowExecutionService:ttl_in_second=completed_ttl,)logger.info(f"Tracked successful completion for {file_name}")- else:- # Track failure on finalization stage+ elif not is_duplicate_skip:+ # Track failure on finalization stage (only for actual failures, not duplicate skips)error_msg = execution_error or (destination_result.errorif destination_result@@ -379,12 +405,23 @@ class WorkerWorkflowExecutionService:),)logger.info(f"Tracked failed execution for {file_name}: {error_msg}")+ else:+ # Duplicate skip - don't update any stages to avoid interfering with active worker+ logger.info(+ f"Skipping finalization stage update for '{file_name}' - duplicate detected, stages managed by active worker"+ )- # Clean up tool execution tracker (even on failure)- self._cleanup_tool_execution_tracker(- execution_id=execution_id,- file_execution_id=workflow_file_execution_id,- )+ # Clean up tool execution tracker only if we actually processed (not duplicate)+ # For duplicate skips, the active worker will clean up its own tracker when it finishes+ if not is_duplicate_skip:+ self._cleanup_tool_execution_tracker(+ execution_id=execution_id,+ file_execution_id=workflow_file_execution_id,+ )+ else:+ logger.info(+ f"Skipping tool execution tracker cleanup for '{file_name}' - duplicate detected, tracker managed by active worker"+ )except Exception as tracker_error:logger.warning(f"Failed to track final completion stage: {tracker_error}")@@ -1147,6 +1184,21 @@ class WorkerWorkflowExecutionService:organization_id=organization_id,execution_error=execution_error,)++ # Check if handle_output returned None (duplicate detected)+ if handle_output_result is None:+ logger.info(+ f"Duplicate detected for {file_hash.file_name} - "+ f"skipping file history and all downstream processing"+ )+ # Return special result to indicate duplicate (not an error, not processed)+ return FinalOutputResult(+ output=None,+ metadata=None,+ error=None, # Not an error - just a duplicate skip+ processed=False, # Indicates this was skipped due to duplicate+ )+output_result = handle_output_result.outputmetadata = handle_output_result.metadatasource_connection_type = workflow.source_config.connection_type