Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

move the lock deletion to handle_output just after destination process

ali922d22e06f8b
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py
index ec4a58818..a96245a72 100644
--- a/workers/shared/workflow/destination_connector.py
+++ b/workers/shared/workflow/destination_connector.py
@@ -700,6 +700,24 @@ class WorkerDestinationConnector:
except Exception as e:
self._handle_destination_error(exec_ctx, file_ctx, e)
raise
+ finally:
+ # Release lock after destination processing completes
+ # Critical section (stage set + data extraction + destination write) is done
+ # File history and stage updates don't need the lock (protected by stage checks)
+ try:
+ tracker = FileExecutionStatusTracker()
+ lock_key = tracker.get_destination_lock_key(
+ exec_ctx.execution_id, exec_ctx.file_execution_id
+ )
+ tracker.redis_client.delete(lock_key)
+ logger.info(
+ f"Released destination lock for '{file_ctx.file_name}' "
+ f"after destination processing (lock_key={lock_key})"
+ )
+ except Exception as lock_error:
+ logger.warning(
+ f"Failed to release destination lock for '{file_ctx.file_name}': {lock_error}"
+ )
# Log success
self._log_processing_success(exec_ctx, file_ctx, result.has_hitl)
diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py
index 19dce7be0..9a5096f34 100644
--- a/workers/shared/workflow/execution/service.py
+++ b/workers/shared/workflow/execution/service.py
@@ -273,37 +273,6 @@ class WorkerWorkflowExecutionService:
f"TIMING: Destination processing END for {file_name} at {destination_end_time:.6f} (took {destination_end_time - destination_start_time:.3f}s)"
)
- # Only clean up lock if we actually processed (not a duplicate skip)
- # CRITICAL: Do not delete lock if we skipped due to duplicate - another worker might hold it!
- if (
- workflow_file_execution_id
- and destination_result
- and getattr(
- destination_result, "processed", True
- ) # Only cleanup if we actually processed
- ):
- try:
- tracker = FileExecutionStatusTracker()
- lock_key = tracker.get_destination_lock_key(
- execution_id, workflow_file_execution_id
- )
- tracker.redis_client.delete(lock_key)
- logger.debug(
- f"Cleaned up destination lock for '{file_name}' (lock_key={lock_key})"
- )
- except Exception as lock_cleanup_error:
- logger.warning(
- f"Failed to clean up destination lock: {lock_cleanup_error}"
- )
- elif (
- workflow_file_execution_id
- and destination_result
- and not getattr(destination_result, "processed", True)
- ):
- logger.info(
- f"Skipping lock cleanup for '{file_name}' - duplicate detected, lock held by another worker"
- )
-
# Step 4: Build Final Result
final_time = time.time()
execution_time = final_time - start_time