Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2874 Address CodeRabbit review: Add lock TTL and error checking for destination processing

ali84a9e5d4ef62
diff --git a/workers/sample.env b/workers/sample.env
index b4c00dc61..311b72329 100644
--- a/workers/sample.env
+++ b/workers/sample.env
@@ -257,6 +257,7 @@ MAX_PARALLEL_FILE_BATCHES=1
# File Execution TTL Configuration
FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000
FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=300
+DESTINATION_PROCESSING_LOCK_TTL_IN_SECOND=300
EXECUTION_RESULT_TTL_SECONDS=86400
EXECUTION_CACHE_TTL_SECONDS=86400
INSTANT_WF_POLLING_TIMEOUT=300
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py
index cd120f4dc..8d4980e3e 100644
--- a/workers/shared/workflow/destination_connector.py
+++ b/workers/shared/workflow/destination_connector.py
@@ -352,9 +352,13 @@ class WorkerDestinationConnector:
# Atomically acquire lock by setting DESTINATION_PROCESSING stage
# This prevents race conditions - first worker to set this stage wins
+ # Use shorter TTL for lock to prevent deadlock if worker crashes (10 min default)
+ LOCK_TTL = int(
+ os.environ.get("DESTINATION_PROCESSING_LOCK_TTL_IN_SECOND", 600)
+ )
try:
logger.info(
- f"Acquiring destination processing lock for file '{file_ctx.file_name}'"
+ f"Acquiring destination processing lock for file '{file_ctx.file_name}' with TTL {LOCK_TTL}s"
)
tracker.update_stage_status(
exec_ctx.execution_id,
@@ -363,6 +367,7 @@ class WorkerDestinationConnector:
stage=FileExecutionStage.DESTINATION_PROCESSING,
status=FileExecutionStageStatus.IN_PROGRESS,
),
+ ttl_in_second=LOCK_TTL,
)
logger.info(
f"Successfully acquired destination processing lock for file '{file_ctx.file_name}'"
diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py
index 1dbf59563..c16baf1fe 100644
--- a/workers/shared/workflow/execution/service.py
+++ b/workers/shared/workflow/execution/service.py
@@ -230,8 +230,13 @@ class WorkerWorkflowExecutionService:
)
logger.info(f"Destination processing completed for {file_name}")
- # Mark destination processing as successful
- if workflow_file_execution_id and workflow_success:
+ # Mark destination processing as successful - only if workflow succeeded AND no destination errors
+ if (
+ workflow_file_execution_id
+ and workflow_success
+ and destination_result
+ and not destination_result.error
+ ):
try:
tracker = FileExecutionStatusTracker()
tracker.update_stage_status(