LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Celery split for callbacks (#1335)
alia59d8a8ad49d
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex dbe0b0945..a7a09e2b9 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -138,6 +138,10 @@ INSTANT_WF_POLLING_TIMEOUT = int() # 5 minutesMAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))+CELERY_RESULT_CHORD_RETRY_INTERVAL = int(+ os.environ.get("CELERY_RESULT_CHORD_RETRY_INTERVAL", "3")+)+INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))ATOMIC_REQUESTS = CommonUtils.str_to_bool(diff --git a/backend/backend/workers/constants.py b/backend/backend/workers/constants.pyindex 3d1eb83df..4feaa351d 100644--- a/backend/backend/workers/constants.py+++ b/backend/backend/workers/constants.py@@ -2,3 +2,4 @@ class CeleryWorkerNames:"""Names of the celery workers."""FILE_PROCESSING = "file_processing"+ FILE_PROCESSING_CALLBACK = "file_processing_callback"diff --git a/backend/backend/workers/file_processing_callback/__init__.py b/backend/backend/workers/file_processing_callback/__init__.pynew file mode 100644index 000000000..0f5bf2121--- /dev/null+++ b/backend/backend/workers/file_processing_callback/__init__.py@@ -0,0 +1,4 @@+from .constants import QueueNames+from .file_processing_callback import app++__all__ = ["app", "QueueNames"]diff --git a/backend/backend/workers/file_processing_callback/celery_config.py b/backend/backend/workers/file_processing_callback/celery_config.pynew file mode 100644index 000000000..ae3f72980--- /dev/null+++ b/backend/backend/workers/file_processing_callback/celery_config.py@@ -0,0 +1,35 @@+from django.conf import settings+from kombu import Queue++from backend.celery_config import CeleryConfig as BaseCeleryConfig+from backend.workers.file_processing_callback.constants import QueueNames+++class CeleryConfig(BaseCeleryConfig):+ """Specifies celery configuration for file processing callback++ Refer https://docs.celeryq.dev/en/stable/userguide/configuration.html+ """++ task_queues = [+ Queue(+ QueueNames.FILE_PROCESSING_CALLBACK,+ routing_key=QueueNames.FILE_PROCESSING_CALLBACK,+ ),+ Queue(+ QueueNames.API_FILE_PROCESSING_CALLBACK,+ routing_key=QueueNames.API_FILE_PROCESSING_CALLBACK,+ ),+ ]++ task_default_queue = QueueNames.FILE_PROCESSING_CALLBACK+ # Prevents chord loss during scaling events+ worker_pool_restarts = True+ broker_connection_retry_on_startup = True++ # Chord-specific settings+ result_chord_retry_interval = (+ settings.CELERY_RESULT_CHORD_RETRY_INTERVAL+ ) # Backend check interval in seconds++ imports = ["workflow_manager.workflow_v2.file_execution_tasks"]diff --git a/backend/backend/workers/file_processing_callback/constants.py b/backend/backend/workers/file_processing_callback/constants.pynew file mode 100644index 000000000..e22eb3f77--- /dev/null+++ b/backend/backend/workers/file_processing_callback/constants.py@@ -0,0 +1,3 @@+class QueueNames:+ API_FILE_PROCESSING_CALLBACK = "api_file_processing_callback"+ FILE_PROCESSING_CALLBACK = "file_processing_callback"diff --git a/backend/backend/workers/file_processing_callback/file_processing_callback.py b/backend/backend/workers/file_processing_callback/file_processing_callback.pynew file mode 100644index 000000000..ecc20c643--- /dev/null+++ b/backend/backend/workers/file_processing_callback/file_processing_callback.py@@ -0,0 +1,31 @@+"""This module contains the Celery configuration for the backend project."""++import logging+import logging.config+import os+from pprint import pformat++from celery import Celery++from backend.settings.base import LOGGING+from backend.workers.constants import CeleryWorkerNames+from backend.workers.file_processing_callback.celery_config import CeleryConfig++logger = logging.getLogger(__name__)++# Set the default Django settings module for the 'celery' program.+os.environ.setdefault(+ "DJANGO_SETTINGS_MODULE",+ os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),+)++# Configure logging for celery worker using same config as Django+logging.config.dictConfig(LOGGING)++# Create a Celery instance. Default time zone is UTC.+app = Celery(CeleryWorkerNames.FILE_PROCESSING_CALLBACK)++# Load task modules from all registered Django app configs.+app.config_from_object(CeleryConfig)++logger.debug(f"Celery Configuration:\n {pformat(app.conf.table(with_defaults=True))}")diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex e7f3bd49b..a7106d30a 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -127,6 +127,14 @@ help = "Runs the Unstract file processing worker"cmd = "celery -A backend.workers.file_processing worker --loglevel=info -Q api_file_processing --autoscale 4,1"help = "Runs the Unstract file processing api worker"+[tool.poe.tasks.worker-file-processing-callback]+cmd = "celery -A backend.workers.file_processing_callback worker --loglevel=info -Q file_processing_callback --autoscale 4,1"+help = "Runs the Unstract file processing callback worker"++[tool.poe.tasks.worker-api-file-processing-callback]+cmd = "celery -A backend.workers.file_processing_callback worker --loglevel=info -Q api_file_processing_callback --autoscale 4,1"+help = "Runs the Unstract api file processing callback worker"+[tool.poe.tasks.flower]cmd = "celery -A backend flower --port=5555"help = "Runs the Unstract Celery Monitoring Tool"diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.pyindex 9dbe78321..7fb75af88 100644--- a/backend/workflow_manager/workflow_v2/enums.py+++ b/backend/workflow_manager/workflow_v2/enums.py@@ -83,3 +83,8 @@ class AgentName(Enum):class RuleLogic(Enum):AND = "AND"OR = "OR"+++class TaskType(Enum):+ FILE_PROCESSING = "FILE_PROCESSING"+ FILE_PROCESSING_CALLBACK = "FILE_PROCESSING_CALLBACK"diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 10fcb96b3..fe496170d 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -10,8 +10,16 @@ from tool_instance_v2.tool_instance_helper import ToolInstanceHelperfrom utils.constants import Accountfrom utils.local_context import StateStore-from backend.workers.file_processing.constants import QueueNames+from backend.workers.file_processing.constants import (+ QueueNames as FileProcessingQueueNames,+)from backend.workers.file_processing.file_processing import app as file_processing_app+from backend.workers.file_processing_callback.constants import (+ QueueNames as FileProcessingCallbackQueueNames,+)+from backend.workers.file_processing_callback.file_processing_callback import (+ app as file_processing_callback_app,+)from unstract.workflow_execution.enums import LogComponent, LogStage, LogStatefrom unstract.workflow_execution.exceptions import StopExecutionfrom workflow_manager.endpoint_v2.destination import DestinationConnector@@ -36,7 +44,7 @@ from workflow_manager.workflow_v2.dto import (FinalOutputResult,ToolExecutionResult,)-from workflow_manager.workflow_v2.enums import ExecutionStatus+from workflow_manager.workflow_v2.enums import ExecutionStatus, TaskTypefrom workflow_manager.workflow_v2.exceptions import (ExecutionContextInitializationError,WorkflowDoesNotExistError,@@ -53,9 +61,20 @@ logger = logging.getLogger(__name__)class FileExecutionTasks:@staticmethod- def get_queue_name(source: SourceConnector) -> str:+ def get_queue_name(source: SourceConnector, type: TaskType) -> str:is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API- return QueueNames.API_FILE_PROCESSING if is_api else QueueNames.FILE_PROCESSING+ if type == TaskType.FILE_PROCESSING:+ return (+ FileProcessingQueueNames.API_FILE_PROCESSING+ if is_api+ else FileProcessingQueueNames.FILE_PROCESSING+ )+ if type == TaskType.FILE_PROCESSING_CALLBACK:+ return (+ FileProcessingCallbackQueueNames.API_FILE_PROCESSING_CALLBACK+ if is_api+ else FileProcessingCallbackQueueNames.FILE_PROCESSING_CALLBACK+ )@staticmethoddef get_workflow_by_id(id: str) -> Workflow:@@ -176,15 +195,18 @@ class FileExecutionTasks:file_hash=file_hash,workflow_execution=workflow_execution,)-if file_execution_result.error:failed_files += 1+ logger.info(+ f"File execution for file {file_name} marked as failed with error: {file_execution_result.error}"+ )ExecutionCacheUtils.increment_failed_files(workflow_id=workflow.id,execution_id=execution_id,)else:successful_files += 1+ logger.info(f"File execution for file {file_name} marked as successful")ExecutionCacheUtils.increment_completed_files(workflow_id=workflow.id,execution_id=execution_id,@@ -194,7 +216,7 @@ class FileExecutionTasks:failed_files=failed_files,).to_dict()- @file_processing_app.task(+ @file_processing_callback_app.task(bind=True,max_retries=0, # Maximum number of retriesretry_backoff=True,@@ -233,6 +255,7 @@ class FileExecutionTasks:# Set organization ID in StateStoreStateStore.set(Account.ORGANIZATION_ID, organization_id)+ logger.info(f"Processing batch callback for execution id: '{execution_id}'")if not results:return None@@ -274,6 +297,13 @@ class FileExecutionTasks:successful_files=total_successful,failed_files=total_failed,)+ logger.info(+ f"Execution completed for execution id: '{execution_id}' "+ f"with status: '{final_status}' "+ f"and total files: '{total_files}' "+ f"and successful files: '{total_successful}' "+ f"and failed files: '{total_failed}'"+ )return {"execution_id": execution_id,"total_files": total_files,@@ -569,7 +599,7 @@ class FileExecutionTasks:use_file_history=file_data.use_file_history,file_execution_id=str(workflow_file_execution.id),)-+ logger.info(f"Execution service built for file: '{file_hash.file_name}'")try:execution_service.initiate_tool_execution(current_file_idx=current_file_idx,@@ -577,7 +607,9 @@ class FileExecutionTasks:file_name=file_hash.file_name,single_step=file_data.single_step,)-+ logger.info(+ f"Initiated tool execution for file id: '{workflow_file_execution.id}' file name: '{file_hash.file_name}'"+ )if not file_hash.is_executed:execution_service.execute_input_file(file_execution_id=str(workflow_file_execution.id),@@ -585,6 +617,9 @@ class FileExecutionTasks:single_step=file_data.single_step,workflow_file_execution=workflow_file_execution,)+ logger.info(+ f"Executed input file for file id: '{workflow_file_execution.id}' file name: '{file_hash.file_name}'"+ )# Log execution costsexecution_service.log_total_cost_per_file(run_id=str(workflow_file_execution.id), file_name=file_hash.file_namediff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex c6371d51a..dc98addbd 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -46,7 +46,12 @@ from workflow_manager.workflow_v2.dto import (FileBatchData,FileData,)-from workflow_manager.workflow_v2.enums import ExecutionStatus, SchemaEntity, SchemaType+from workflow_manager.workflow_v2.enums import (+ ExecutionStatus,+ SchemaEntity,+ SchemaType,+ TaskType,+)from workflow_manager.workflow_v2.exceptions import (InvalidRequest,TaskDoesNotExistError,@@ -189,7 +194,9 @@ class WorkflowHelper:batch_data = FileBatchData(files=batch, file_data=file_data)# Determine the appropriate queue based on execution_mode- file_processing_queue = FileExecutionTasks.get_queue_name(source)+ file_processing_queue = FileExecutionTasks.get_queue_name(+ source, TaskType.FILE_PROCESSING+ )# Send each batch to the dedicated file_processing queuebatch_tasks.append(@@ -198,10 +205,13 @@ class WorkflowHelper:))try:+ file_processing_callback_queue = FileExecutionTasks.get_queue_name(+ source, TaskType.FILE_PROCESSING_CALLBACK+ )result = chord(batch_tasks)(FileExecutionTasks.process_batch_callback.s(execution_id=str(workflow_execution.id)- ).set(queue=file_processing_queue)+ ).set(queue=file_processing_callback_queue))if not result:exception = f"Failed to queue execution task {workflow_execution.id}"diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yamlindex f6e92877c..404901888 100644--- a/docker/sample.compose.override.yaml+++ b/docker/sample.compose.override.yaml@@ -6,6 +6,8 @@ services:command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue,celery_api_deployments --autoscale=${WORKER_AUTOSCALE}"worker-file-processing:command: "-A backend.workers.file_processing worker --loglevel=info -Q file_processing,api_file_processing --autoscale=${WORKER_FILE_PROCESSING_AUTOSCALE}"+ worker-file-processing-callback:+ command: "-A backend.workers.file_processing_callback worker --loglevel=info -Q file_processing_callback,api_file_processing_callback --autoscale=${WORKER_FILE_PROCESSING_CALLBACK_AUTOSCALE}"worker-logging:profiles:diff --git a/docker/sample.env b/docker/sample.envindex 269bca95f..6ff11c14b 100644--- a/docker/sample.env+++ b/docker/sample.env@@ -12,3 +12,5 @@ WORKER_LOGGING_AUTOSCALE=4,1WORKER_AUTOSCALE=4,1WORKER_FILE_PROCESSING_AUTOSCALE=4,1WORKER_API_FILE_PROCESSING_AUTOSCALE=4,1+WORKER_FILE_PROCESSING_CALLBACK_AUTOSCALE=4,1+WORKER_API_FILE_PROCESSING_CALLBACK_AUTOSCALE=4,1diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 15e1a2987..f15fc4a33 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -251,10 +251,13 @@ class StructureTool(BaseTool):try:self.stream_log(- "Dumping prompt studio project's output to workflow's storage"+ "Writing prompt studio project's output to workflow's storage")output_path = Path(output_dir) / f"{Path(self.source_file_name).stem}.json"self.workflow_filestorage.json_dump(path=output_path, data=structured_output)+ self.stream_log(+ "Prompt studio project's output written successfully to workflow's storage"+ )except OSError as e:self.stream_error_and_exit(f"Error creating output file: {e}")except json.JSONDecodeError as e: