LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Added celery config: visibility_timeout for broker (#1150)
Chandrasekharan Mf32ca9ce2014
diff --git a/backend/backend/celery_config.py b/backend/backend/celery_config.pynew file mode 100644index 000000000..7fef6e495--- /dev/null+++ b/backend/backend/celery_config.py@@ -0,0 +1,49 @@+from urllib.parse import quote_plus++from django.conf import settings++from unstract.core.utilities import UnstractUtils+++class CeleryConfig:+ """Specifies celery configuration++ Refer https://docs.celeryq.dev/en/stable/userguide/configuration.html+ """++ # Envs defined for configuration with Unstract+ CELERY_BROKER_URL = "CELERY_BROKER_URL"+ CELERY_BROKER_VISIBILITY_TIMEOUT = "CELERY_BROKER_VISIBILITY_TIMEOUT"++ # Result backend configuration+ result_backend = (+ f"db+postgresql://{settings.DB_USER}:{quote_plus(settings.DB_PASSWORD)}"+ f"@{settings.DB_HOST}:{settings.DB_PORT}/{settings.DB_NAME}"+ )++ # Broker URL configuration+ broker_url = UnstractUtils.get_env(+ CELERY_BROKER_URL,+ f"redis://{settings.REDIS_HOST}:{settings.REDIS_PORT}",+ raise_err=True,+ )++ # Task serialization and content settings+ accept_content = ["json"]+ task_serializer = "json"+ result_serializer = "json"+ result_extended = True++ # Timezone and logger settings+ timezone = "UTC"+ worker_hijack_root_logger = False++ beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler"++ task_acks_late = True+ # Large timeout avoids worker to pick up same unacknowledged task again+ broker_transport_options = {+ "visibility_timeout": float(+ UnstractUtils.get_env(CELERY_BROKER_VISIBILITY_TIMEOUT, 7200)+ )+ }diff --git a/backend/backend/celery_service.py b/backend/backend/celery_service.pyindex 99effcd78..37694cecf 100644--- a/backend/backend/celery_service.py+++ b/backend/backend/celery_service.py@@ -1,12 +1,17 @@"""This module contains the Celery configuration for the backend project."""+import logging+import logging.configimport os+from pprint import pformatfrom celery import Celery-from django.conf import settingsfrom utils.constants import ExecutionLogConstantsfrom backend.celery_task import TaskRegistry+from backend.settings.base import LOGGING++logger = logging.getLogger(__name__)# Set the default Django settings module for the 'celery' program.os.environ.setdefault(@@ -14,30 +19,26 @@ os.environ.setdefault(os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),)+# Configure logging for celery worker using same config as Django+logging.config.dictConfig(LOGGING)+# Create a Celery instance. Default time zone is UTC.app = Celery("backend")-# To register the custom tasks.+# Register custom tasksTaskRegistry()-# Use Redis as the message broker.-app.conf.broker_url = settings.CELERY_BROKER_URL-app.conf.result_backend = settings.CELERY_RESULT_BACKEND--# Load task modules from all registered Django app configs.-app.config_from_object("django.conf:settings", namespace="CELERY")--# Autodiscover tasks in all installed apps.+app.config_from_object("backend.celery_config.CeleryConfig")app.autodiscover_tasks()+logger.debug(+ f"Celery Configuration:\n" f"{pformat(app.conf.table(with_defaults=True))}"+)+# Define the queues to purge when the Celery broker is restarted.queues_to_purge = [ExecutionLogConstants.CELERY_QUEUE_NAME]with app.connection() as connection:channel = connection.channel()-for queue_name in queues_to_purge:channel.queue_purge(queue_name)--# Use the Django-Celery-Beat scheduler.-app.conf.beat_scheduler = "django_celery_beat.schedulers:DatabaseScheduler"diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 6394f5a23..ea64d63a8 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -12,7 +12,6 @@ https://docs.djangoproject.com/en/4.2/ref/settings/import osfrom pathlib import Pathfrom typing import Optional-from urllib.parse import quote_plusfrom dotenv import find_dotenv, load_dotenvfrom utils.common_utils import CommonUtils@@ -133,9 +132,6 @@ LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))LOGS_EXPIRATION_TIME_IN_SECOND = int(get_required_setting("LOGS_EXPIRATION_TIME_IN_SECOND"))-CELERY_BROKER_URL = get_required_setting(- "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"-)INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))@@ -370,20 +366,6 @@ RQ_QUEUES = {"default": {"USE_REDIS_CACHE": "default"},}--# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1"-# Postgres as result backend-CELERY_RESULT_BACKEND = (- f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}"- f"@{DB_HOST}:{DB_PORT}/{DB_NAME}"-)-CELERY_ACCEPT_CONTENT = ["json"]-CELERY_TASK_SERIALIZER = "json"-CELERY_RESULT_SERIALIZER = "json"-CELERY_TIMEZONE = "UTC"-CELERY_TASK_MAX_RETRIES = 3-CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task-# Feature FlagFEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}diff --git a/backend/sample.env b/backend/sample.envindex 66e780dda..4bca66859 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -128,7 +128,9 @@ LOGS_BATCH_LIMIT=30LOGS_EXPIRATION_TIME_IN_SECOND=86400# Celery Configuration+# Used by celery and to connect to queue to push logsCELERY_BROKER_URL = "redis://unstract-redis:6379"+CELERY_BROKER_VISIBILITY_TIMEOUT = "86400" # 1 day# Indexing flag to prevent re-indexINDEXING_FLAG_TTL=1800diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 5e24e3ae4..772b9db44 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -575,9 +575,8 @@ class WorkflowHelper:@staticmethod@shared_task(name="async_execute_bin",- acks_late=True,autoretry_for=(Exception,),- max_retries=1,+ max_retries=0,retry_backoff=True,retry_backoff_max=500,retry_jitter=True,