LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
move the lock deletion to handle_output just after destination process
ali922d22e06f8b
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.pyindex ec4a58818..a96245a72 100644--- a/workers/shared/workflow/destination_connector.py+++ b/workers/shared/workflow/destination_connector.py@@ -700,6 +700,24 @@ class WorkerDestinationConnector:except Exception as e:self._handle_destination_error(exec_ctx, file_ctx, e)raise+ finally:+ # Release lock after destination processing completes+ # Critical section (stage set + data extraction + destination write) is done+ # File history and stage updates don't need the lock (protected by stage checks)+ try:+ tracker = FileExecutionStatusTracker()+ lock_key = tracker.get_destination_lock_key(+ exec_ctx.execution_id, exec_ctx.file_execution_id+ )+ tracker.redis_client.delete(lock_key)+ logger.info(+ f"Released destination lock for '{file_ctx.file_name}' "+ f"after destination processing (lock_key={lock_key})"+ )+ except Exception as lock_error:+ logger.warning(+ f"Failed to release destination lock for '{file_ctx.file_name}': {lock_error}"+ )# Log successself._log_processing_success(exec_ctx, file_ctx, result.has_hitl)diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex 19dce7be0..9a5096f34 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -273,37 +273,6 @@ class WorkerWorkflowExecutionService:f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)")- # Only clean up lock if we actually processed (not a duplicate skip)- # CRITICAL: Do not delete lock if we skipped due to duplicate - another worker might hold it!- if (- workflow_file_execution_id- and destination_result- and getattr(- destination_result, "processed", True- ) # Only cleanup if we actually processed- ):- try:- tracker = FileExecutionStatusTracker()- lock_key = tracker.get_destination_lock_key(- execution_id, workflow_file_execution_id- )- tracker.redis_client.delete(lock_key)- logger.debug(- f"Cleaned up destination lock for '{file_name}' (lock_key={lock_key})"- )- except Exception as lock_cleanup_error:- logger.warning(- f"Failed to clean up destination lock: {lock_cleanup_error}"- )- elif (- workflow_file_execution_id- and destination_result- and not getattr(destination_result, "processed", True)- ):- logger.info(- f"Skipping lock cleanup for '{file_name}' - duplicate detected, lock held by another worker"- )-# Step 4: Build Final Resultfinal_time = time.time()execution_time = final_time - start_time