LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2854 [MISC] Set CELERY_TASK_REJECT_ON_WORKER_LOST to false by default to prevent duplicate task processing (#1565)
ali490e394a717d
diff --git a/workers/sample.env b/workers/sample.envindex d5a5e8d76..1951381c7 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -139,7 +139,7 @@ CALLBACK_TASK_SOFT_TIME_LIMIT=3300 # 55 minutes - Callback soft timeou# Retry ConfigurationCELERY_TASK_DEFAULT_RETRY_DELAY=60CELERY_TASK_MAX_RETRIES=3-CELERY_TASK_REJECT_ON_WORKER_LOST=true+CELERY_TASK_REJECT_ON_WORKER_LOST=false# Advanced Celery ConfigurationCELERY_WORKER_POOL_RESTARTS=true@@ -242,8 +242,8 @@ UNSTRACT_RUNNER_API_BACKOFF_FACTOR=3# =============================================================================# File Storage Credentials (MinIO)-WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}-API_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}+WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+API_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'# File Execution ConfigurationWORKFLOW_EXECUTION_DIR_PREFIX=unstract/execution@@ -269,7 +269,7 @@ ENABLE_WEBHOOK_DELIVERY=true# Tool RegistryTOOL_REGISTRY_CONFIG_PATH=../unstract/tool-registry/tool_registry_config-TOOL_REGISTRY_STORAGE_CREDENTIALS={"provider":"local"}+TOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'# =============================================================================# Optional Advanced Settingsdiff --git a/workers/shared/models/worker_models.py b/workers/shared/models/worker_models.pyindex b894eabdf..cf2cf07f2 100644--- a/workers/shared/models/worker_models.py+++ b/workers/shared/models/worker_models.py@@ -313,7 +313,7 @@ class WorkerCeleryConfig:prefetch_multiplier: int = 1max_tasks_per_child: int = 1000task_acks_late: bool = True- task_reject_on_worker_lost: bool = True+ task_reject_on_worker_lost: bool = False# Timeoutstask_time_limit: int = 7200 # 2 hours@@ -395,7 +395,10 @@ class WorkerCeleryConfig:int, # Prevent memory leaks),"task_reject_on_worker_lost": get_celery_setting(- "TASK_REJECT_ON_WORKER_LOST", self.worker_type, True, bool+ "TASK_REJECT_ON_WORKER_LOST",+ self.worker_type,+ self.task_reject_on_worker_lost,+ bool,),"task_acks_on_failure_or_timeout": get_celery_setting("TASK_ACKS_ON_FAILURE_OR_TIMEOUT", self.worker_type, True, bool