Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Added celery config: visibility_timeout for broker (#1150)

Chandrasekharan Mf32ca9ce2014
diff --git a/backend/backend/celery_config.py b/backend/backend/celery_config.py
new file mode 100644
index 000000000..7fef6e495
--- /dev/null
+++ b/backend/backend/celery_config.py
@@ -0,0 +1,49 @@
+from urllib.parse import quote_plus
+
+from django.conf import settings
+
+from unstract.core.utilities import UnstractUtils
+
+
+class CeleryConfig:
+ """Specifies celery configuration
+
+ Refer https://docs.celeryq.dev/en/stable/userguide/configuration.html
+ """
+
+ # Envs defined for configuration with Unstract
+ CELERY_BROKER_URL = "CELERY_BROKER_URL"
+ CELERY_BROKER_VISIBILITY_TIMEOUT = "CELERY_BROKER_VISIBILITY_TIMEOUT"
+
+ # Result backend configuration
+ result_backend = (
+ f"db+postgresql://{settings.DB_USER}:{quote_plus(settings.DB_PASSWORD)}"
+ f"@{settings.DB_HOST}:{settings.DB_PORT}/{settings.DB_NAME}"
+ )
+
+ # Broker URL configuration
+ broker_url = UnstractUtils.get_env(
+ CELERY_BROKER_URL,
+ f"redis://{settings.REDIS_HOST}:{settings.REDIS_PORT}",
+ raise_err=True,
+ )
+
+ # Task serialization and content settings
+ accept_content = ["json"]
+ task_serializer = "json"
+ result_serializer = "json"
+ result_extended = True
+
+ # Timezone and logger settings
+ timezone = "UTC"
+ worker_hijack_root_logger = False
+
+ beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
+
+ task_acks_late = True
+ # Large timeout avoids worker to pick up same unacknowledged task again
+ broker_transport_options = {
+ "visibility_timeout": float(
+ UnstractUtils.get_env(CELERY_BROKER_VISIBILITY_TIMEOUT, 7200)
+ )
+ }
diff --git a/backend/backend/celery_service.py b/backend/backend/celery_service.py
index 99effcd78..37694cecf 100644
--- a/backend/backend/celery_service.py
+++ b/backend/backend/celery_service.py
@@ -1,12 +1,17 @@
"""This module contains the Celery configuration for the backend project."""
+import logging
+import logging.config
import os
+from pprint import pformat
from celery import Celery
-from django.conf import settings
from utils.constants import ExecutionLogConstants
from backend.celery_task import TaskRegistry
+from backend.settings.base import LOGGING
+
+logger = logging.getLogger(__name__)
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault(
@@ -14,30 +19,26 @@ os.environ.setdefault(
os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
)
+# Configure logging for celery worker using same config as Django
+logging.config.dictConfig(LOGGING)
+
# Create a Celery instance. Default time zone is UTC.
app = Celery("backend")
-# To register the custom tasks.
+# Register custom tasks
TaskRegistry()
-# Use Redis as the message broker.
-app.conf.broker_url = settings.CELERY_BROKER_URL
-app.conf.result_backend = settings.CELERY_RESULT_BACKEND
-
-
# Load task modules from all registered Django app configs.
-app.config_from_object("django.conf:settings", namespace="CELERY")
-
-# Autodiscover tasks in all installed apps.
+app.config_from_object("backend.celery_config.CeleryConfig")
app.autodiscover_tasks()
+logger.debug(
+ f"Celery Configuration:\n" f"{pformat(app.conf.table(with_defaults=True))}"
+)
+
# Define the queues to purge when the Celery broker is restarted.
queues_to_purge = [ExecutionLogConstants.CELERY_QUEUE_NAME]
with app.connection() as connection:
channel = connection.channel()
-
for queue_name in queues_to_purge:
channel.queue_purge(queue_name)
-
-# Use the Django-Celery-Beat scheduler.
-app.conf.beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler"
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 6394f5a23..ea64d63a8 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -12,7 +12,6 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
import os
from pathlib import Path
from typing import Optional
-from urllib.parse import quote_plus
from dotenv import find_dotenv, load_dotenv
from utils.common_utils import CommonUtils
@@ -133,9 +132,6 @@ LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
LOGS_EXPIRATION_TIME_IN_SECOND = int(
get_required_setting("LOGS_EXPIRATION_TIME_IN_SECOND")
)
-CELERY_BROKER_URL = get_required_setting(
- "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"
-)
INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))
@@ -370,20 +366,6 @@ RQ_QUEUES = {
"default": {"USE_REDIS_CACHE": "default"},
}
-
-# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1"
-# Postgres as result backend
-CELERY_RESULT_BACKEND = (
- f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}"
- f"@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-)
-CELERY_ACCEPT_CONTENT = ["json"]
-CELERY_TASK_SERIALIZER = "json"
-CELERY_RESULT_SERIALIZER = "json"
-CELERY_TIMEZONE = "UTC"
-CELERY_TASK_MAX_RETRIES = 3
-CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task
-
# Feature Flag
FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}
diff --git a/backend/sample.env b/backend/sample.env
index 66e780dda..4bca66859 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -128,7 +128,9 @@ LOGS_BATCH_LIMIT=30
LOGS_EXPIRATION_TIME_IN_SECOND=86400
# Celery Configuration
+# Used by celery and to connect to queue to push logs
CELERY_BROKER_URL = "redis://unstract-redis:6379"
+CELERY_BROKER_VISIBILITY_TIMEOUT = "86400" # 1 day
# Indexing flag to prevent re-index
INDEXING_FLAG_TTL=1800
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 5e24e3ae4..772b9db44 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -575,9 +575,8 @@ class WorkflowHelper:
@staticmethod
@shared_task(
name="async_execute_bin",
- acks_late=True,
autoretry_for=(Exception,),
- max_retries=1,
+ max_retries=0,
retry_backoff=True,
retry_backoff_max=500,
retry_jitter=True,