Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Celery split for callbacks (#1335)

alia59d8a8ad49d
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index dbe0b0945..a7a09e2b9 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -138,6 +138,10 @@ INSTANT_WF_POLLING_TIMEOUT = int(
) # 5 minutes
MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))
+CELERY_RESULT_CHORD_RETRY_INTERVAL = int(
+ os.environ.get("CELERY_RESULT_CHORD_RETRY_INTERVAL", "3")
+)
+
INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))
ATOMIC_REQUESTS = CommonUtils.str_to_bool(
diff --git a/backend/backend/workers/constants.py b/backend/backend/workers/constants.py
index 3d1eb83df..4feaa351d 100644
--- a/backend/backend/workers/constants.py
+++ b/backend/backend/workers/constants.py
@@ -2,3 +2,4 @@ class CeleryWorkerNames:
"""Names of the celery workers."""
FILE_PROCESSING = "file_processing"
+ FILE_PROCESSING_CALLBACK = "file_processing_callback"
diff --git a/backend/backend/workers/file_processing_callback/__init__.py b/backend/backend/workers/file_processing_callback/__init__.py
new file mode 100644
index 000000000..0f5bf2121
--- /dev/null
+++ b/backend/backend/workers/file_processing_callback/__init__.py
@@ -0,0 +1,4 @@
+from .constants import QueueNames
+from .file_processing_callback import app
+
+__all__ = ["app", "QueueNames"]
diff --git a/backend/backend/workers/file_processing_callback/celery_config.py b/backend/backend/workers/file_processing_callback/celery_config.py
new file mode 100644
index 000000000..ae3f72980
--- /dev/null
+++ b/backend/backend/workers/file_processing_callback/celery_config.py
@@ -0,0 +1,35 @@
+from django.conf import settings
+from kombu import Queue
+
+from backend.celery_config import CeleryConfig as BaseCeleryConfig
+from backend.workers.file_processing_callback.constants import QueueNames
+
+
+class CeleryConfig(BaseCeleryConfig):
+ """Specifies celery configuration for file processing callback
+
+ Refer https://docs.celeryq.dev/en/stable/userguide/configuration.html
+ """
+
+ task_queues = [
+ Queue(
+ QueueNames.FILE_PROCESSING_CALLBACK,
+ routing_key=QueueNames.FILE_PROCESSING_CALLBACK,
+ ),
+ Queue(
+ QueueNames.API_FILE_PROCESSING_CALLBACK,
+ routing_key=QueueNames.API_FILE_PROCESSING_CALLBACK,
+ ),
+ ]
+
+ task_default_queue = QueueNames.FILE_PROCESSING_CALLBACK
+ # Prevents chord loss during scaling events
+ worker_pool_restarts = True
+ broker_connection_retry_on_startup = True
+
+ # Chord-specific settings
+ result_chord_retry_interval = (
+ settings.CELERY_RESULT_CHORD_RETRY_INTERVAL
+ ) # Backend check interval in seconds
+
+ imports = ["workflow_manager.workflow_v2.file_execution_tasks"]
diff --git a/backend/backend/workers/file_processing_callback/constants.py b/backend/backend/workers/file_processing_callback/constants.py
new file mode 100644
index 000000000..e22eb3f77
--- /dev/null
+++ b/backend/backend/workers/file_processing_callback/constants.py
@@ -0,0 +1,3 @@
+class QueueNames:
+ API_FILE_PROCESSING_CALLBACK = "api_file_processing_callback"
+ FILE_PROCESSING_CALLBACK = "file_processing_callback"
diff --git a/backend/backend/workers/file_processing_callback/file_processing_callback.py b/backend/backend/workers/file_processing_callback/file_processing_callback.py
new file mode 100644
index 000000000..ecc20c643
--- /dev/null
+++ b/backend/backend/workers/file_processing_callback/file_processing_callback.py
@@ -0,0 +1,31 @@
+"""This module contains the Celery configuration for the backend project."""
+
+import logging
+import logging.config
+import os
+from pprint import pformat
+
+from celery import Celery
+
+from backend.settings.base import LOGGING
+from backend.workers.constants import CeleryWorkerNames
+from backend.workers.file_processing_callback.celery_config import CeleryConfig
+
+logger = logging.getLogger(__name__)
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault(
+ "DJANGO_SETTINGS_MODULE",
+ os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+)
+
+# Configure logging for celery worker using same config as Django
+logging.config.dictConfig(LOGGING)
+
+# Create a Celery instance. Default time zone is UTC.
+app = Celery(CeleryWorkerNames.FILE_PROCESSING_CALLBACK)
+
+# Load task modules from all registered Django app configs.
+app.config_from_object(CeleryConfig)
+
+logger.debug(f"Celery Configuration:\n {pformat(app.conf.table(with_defaults=True))}")
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index e7f3bd49b..a7106d30a 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -127,6 +127,14 @@ help = "Runs the Unstract file processing worker"
cmd = "celery -A backend.workers.file_processing worker --loglevel=info -Q api_file_processing --autoscale 4,1"
help = "Runs the Unstract file processing api worker"
+[tool.poe.tasks.worker-file-processing-callback]
+cmd = "celery -A backend.workers.file_processing_callback worker --loglevel=info -Q file_processing_callback --autoscale 4,1"
+help = "Runs the Unstract file processing callback worker"
+
+[tool.poe.tasks.worker-api-file-processing-callback]
+cmd = "celery -A backend.workers.file_processing_callback worker --loglevel=info -Q api_file_processing_callback --autoscale 4,1"
+help = "Runs the Unstract api file processing callback worker"
+
[tool.poe.tasks.flower]
cmd = "celery -A backend flower --port=5555"
help = "Runs the Unstract Celery Monitoring Tool"
diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.py
index 9dbe78321..7fb75af88 100644
--- a/backend/workflow_manager/workflow_v2/enums.py
+++ b/backend/workflow_manager/workflow_v2/enums.py
@@ -83,3 +83,8 @@ class AgentName(Enum):
class RuleLogic(Enum):
AND = "AND"
OR = "OR"
+
+
+class TaskType(Enum):
+ FILE_PROCESSING = "FILE_PROCESSING"
+ FILE_PROCESSING_CALLBACK = "FILE_PROCESSING_CALLBACK"
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 10fcb96b3..fe496170d 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -10,8 +10,16 @@ from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
from utils.constants import Account
from utils.local_context import StateStore
-from backend.workers.file_processing.constants import QueueNames
+from backend.workers.file_processing.constants import (
+ QueueNames as FileProcessingQueueNames,
+)
from backend.workers.file_processing.file_processing import app as file_processing_app
+from backend.workers.file_processing_callback.constants import (
+ QueueNames as FileProcessingCallbackQueueNames,
+)
+from backend.workers.file_processing_callback.file_processing_callback import (
+ app as file_processing_callback_app,
+)
from unstract.workflow_execution.enums import LogComponent, LogStage, LogState
from unstract.workflow_execution.exceptions import StopExecution
from workflow_manager.endpoint_v2.destination import DestinationConnector
@@ -36,7 +44,7 @@ from workflow_manager.workflow_v2.dto import (
FinalOutputResult,
ToolExecutionResult,
)
-from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.enums import ExecutionStatus, TaskType
from workflow_manager.workflow_v2.exceptions import (
ExecutionContextInitializationError,
WorkflowDoesNotExistError,
@@ -53,9 +61,20 @@ logger = logging.getLogger(__name__)
class FileExecutionTasks:
@staticmethod
- def get_queue_name(source: SourceConnector) -> str:
+ def get_queue_name(source: SourceConnector, type: TaskType) -> str:
is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
- return QueueNames.API_FILE_PROCESSING if is_api else QueueNames.FILE_PROCESSING
+ if type == TaskType.FILE_PROCESSING:
+ return (
+ FileProcessingQueueNames.API_FILE_PROCESSING
+ if is_api
+ else FileProcessingQueueNames.FILE_PROCESSING
+ )
+ if type == TaskType.FILE_PROCESSING_CALLBACK:
+ return (
+ FileProcessingCallbackQueueNames.API_FILE_PROCESSING_CALLBACK
+ if is_api
+ else FileProcessingCallbackQueueNames.FILE_PROCESSING_CALLBACK
+ )
@staticmethod
def get_workflow_by_id(id: str) -> Workflow:
@@ -176,15 +195,18 @@ class FileExecutionTasks:
file_hash=file_hash,
workflow_execution=workflow_execution,
)
-
if file_execution_result.error:
failed_files += 1
+ logger.info(
+ f"File execution for file {file_name} marked as failed with error: {file_execution_result.error}"
+ )
ExecutionCacheUtils.increment_failed_files(
workflow_id=workflow.id,
execution_id=execution_id,
)
else:
successful_files += 1
+ logger.info(f"File execution for file {file_name} marked as successful")
ExecutionCacheUtils.increment_completed_files(
workflow_id=workflow.id,
execution_id=execution_id,
@@ -194,7 +216,7 @@ class FileExecutionTasks:
failed_files=failed_files,
).to_dict()
- @file_processing_app.task(
+ @file_processing_callback_app.task(
bind=True,
max_retries=0, # Maximum number of retries
retry_backoff=True,
@@ -233,6 +255,7 @@ class FileExecutionTasks:
# Set organization ID in StateStore
StateStore.set(Account.ORGANIZATION_ID, organization_id)
+ logger.info(f"Processing batch callback for execution id: '{execution_id}'")
if not results:
return None
@@ -274,6 +297,13 @@ class FileExecutionTasks:
successful_files=total_successful,
failed_files=total_failed,
)
+ logger.info(
+ f"Execution completed for execution id: '{execution_id}' "
+ f"with status: '{final_status}' "
+ f"and total files: '{total_files}' "
+ f"and successful files: '{total_successful}' "
+ f"and failed files: '{total_failed}'"
+ )
return {
"execution_id": execution_id,
"total_files": total_files,
@@ -569,7 +599,7 @@ class FileExecutionTasks:
use_file_history=file_data.use_file_history,
file_execution_id=str(workflow_file_execution.id),
)
-
+ logger.info(f"Execution service built for file: '{file_hash.file_name}'")
try:
execution_service.initiate_tool_execution(
current_file_idx=current_file_idx,
@@ -577,7 +607,9 @@ class FileExecutionTasks:
file_name=file_hash.file_name,
single_step=file_data.single_step,
)
-
+ logger.info(
+ f"Initiated tool execution for file id: '{workflow_file_execution.id}' file name: '{file_hash.file_name}'"
+ )
if not file_hash.is_executed:
execution_service.execute_input_file(
file_execution_id=str(workflow_file_execution.id),
@@ -585,6 +617,9 @@ class FileExecutionTasks:
single_step=file_data.single_step,
workflow_file_execution=workflow_file_execution,
)
+ logger.info(
+ f"Executed input file for file id: '{workflow_file_execution.id}' file name: '{file_hash.file_name}'"
+ )
# Log execution costs
execution_service.log_total_cost_per_file(
run_id=str(workflow_file_execution.id), file_name=file_hash.file_name
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index c6371d51a..dc98addbd 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -46,7 +46,12 @@ from workflow_manager.workflow_v2.dto import (
FileBatchData,
FileData,
)
-from workflow_manager.workflow_v2.enums import ExecutionStatus, SchemaEntity, SchemaType
+from workflow_manager.workflow_v2.enums import (
+ ExecutionStatus,
+ SchemaEntity,
+ SchemaType,
+ TaskType,
+)
from workflow_manager.workflow_v2.exceptions import (
InvalidRequest,
TaskDoesNotExistError,
@@ -189,7 +194,9 @@ class WorkflowHelper:
batch_data = FileBatchData(files=batch, file_data=file_data)
# Determine the appropriate queue based on execution_mode
- file_processing_queue = FileExecutionTasks.get_queue_name(source)
+ file_processing_queue = FileExecutionTasks.get_queue_name(
+ source, TaskType.FILE_PROCESSING
+ )
# Send each batch to the dedicated file_processing queue
batch_tasks.append(
@@ -198,10 +205,13 @@ class WorkflowHelper:
)
)
try:
+ file_processing_callback_queue = FileExecutionTasks.get_queue_name(
+ source, TaskType.FILE_PROCESSING_CALLBACK
+ )
result = chord(batch_tasks)(
FileExecutionTasks.process_batch_callback.s(
execution_id=str(workflow_execution.id)
- ).set(queue=file_processing_queue)
+ ).set(queue=file_processing_callback_queue)
)
if not result:
exception = f"Failed to queue execution task {workflow_execution.id}"
diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yaml
index f6e92877c..404901888 100644
--- a/docker/sample.compose.override.yaml
+++ b/docker/sample.compose.override.yaml
@@ -6,6 +6,8 @@ services:
command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue,celery_api_deployments --autoscale=${WORKER_AUTOSCALE}"
worker-file-processing:
command: "-A backend.workers.file_processing worker --loglevel=info -Q file_processing,api_file_processing --autoscale=${WORKER_FILE_PROCESSING_AUTOSCALE}"
+ worker-file-processing-callback:
+ command: "-A backend.workers.file_processing_callback worker --loglevel=info -Q file_processing_callback,api_file_processing_callback --autoscale=${WORKER_FILE_PROCESSING_CALLBACK_AUTOSCALE}"
worker-logging:
profiles:
diff --git a/docker/sample.env b/docker/sample.env
index 269bca95f..6ff11c14b 100644
--- a/docker/sample.env
+++ b/docker/sample.env
@@ -12,3 +12,5 @@ WORKER_LOGGING_AUTOSCALE=4,1
WORKER_AUTOSCALE=4,1
WORKER_FILE_PROCESSING_AUTOSCALE=4,1
WORKER_API_FILE_PROCESSING_AUTOSCALE=4,1
+WORKER_FILE_PROCESSING_CALLBACK_AUTOSCALE=4,1
+WORKER_API_FILE_PROCESSING_CALLBACK_AUTOSCALE=4,1
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 15e1a2987..f15fc4a33 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -251,10 +251,13 @@ class StructureTool(BaseTool):
try:
self.stream_log(
- "Dumping prompt studio project's output to workflow's storage"
+ "Writing prompt studio project's output to workflow's storage"
)
output_path = Path(output_dir) / f"{Path(self.source_file_name).stem}.json"
self.workflow_filestorage.json_dump(path=output_path, data=structured_output)
+ self.stream_log(
+ "Prompt studio project's output written successfully to workflow's storage"
+ )
except OSError as e:
self.stream_error_and_exit(f"Error creating output file: {e}")
except json.JSONDecodeError as e: