Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2854 [MISC] Set CELERY_TASK_REJECT_ON_WORKER_LOST to false by default to prevent duplicate task processing (#1565)

ali490e394a717d
diff --git a/workers/sample.env b/workers/sample.env
index d5a5e8d76..1951381c7 100644
--- a/workers/sample.env
+++ b/workers/sample.env
@@ -139,7 +139,7 @@ CALLBACK_TASK_SOFT_TIME_LIMIT=3300 # 55 minutes - Callback soft timeou
# Retry Configuration
CELERY_TASK_DEFAULT_RETRY_DELAY=60
CELERY_TASK_MAX_RETRIES=3
-CELERY_TASK_REJECT_ON_WORKER_LOST=true
+CELERY_TASK_REJECT_ON_WORKER_LOST=false
# Advanced Celery Configuration
CELERY_WORKER_POOL_RESTARTS=true
@@ -242,8 +242,8 @@ UNSTRACT_RUNNER_API_BACKOFF_FACTOR=3
# =============================================================================
# File Storage Credentials (MinIO)
-WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}
-API_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}
+WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
+API_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
# File Execution Configuration
WORKFLOW_EXECUTION_DIR_PREFIX=unstract/execution
@@ -269,7 +269,7 @@ ENABLE_WEBHOOK_DELIVERY=true
# Tool Registry
TOOL_REGISTRY_CONFIG_PATH=../unstract/tool-registry/tool_registry_config
-TOOL_REGISTRY_STORAGE_CREDENTIALS={"provider":"local"}
+TOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'
# =============================================================================
# Optional Advanced Settings
diff --git a/workers/shared/models/worker_models.py b/workers/shared/models/worker_models.py
index b894eabdf..cf2cf07f2 100644
--- a/workers/shared/models/worker_models.py
+++ b/workers/shared/models/worker_models.py
@@ -313,7 +313,7 @@ class WorkerCeleryConfig:
prefetch_multiplier: int = 1
max_tasks_per_child: int = 1000
task_acks_late: bool = True
- task_reject_on_worker_lost: bool = True
+ task_reject_on_worker_lost: bool = False
# Timeouts
task_time_limit: int = 7200 # 2 hours
@@ -395,7 +395,10 @@ class WorkerCeleryConfig:
int, # Prevent memory leaks
),
"task_reject_on_worker_lost": get_celery_setting(
- "TASK_REJECT_ON_WORKER_LOST", self.worker_type, True, bool
+ "TASK_REJECT_ON_WORKER_LOST",
+ self.worker_type,
+ self.task_reject_on_worker_lost,
+ bool,
),
"task_acks_on_failure_or_timeout": get_celery_setting(
"TASK_ACKS_ON_FAILURE_OR_TIMEOUT", self.worker_type, True, bool