LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
parallel execution (#1295)
ali005f7d23f231
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 8f3b04693..c59f246e1 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -128,6 +128,13 @@ LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))LOGS_EXPIRATION_TIME_IN_SECOND = int(get_required_setting("LOGS_EXPIRATION_TIME_IN_SECOND", "86400"))+EXECUTION_RESULT_TTL_SECONDS = int(+ os.environ.get("EXECUTION_RESULT_TTL_SECONDS", 10800)+) # 3 hours+EXECUTION_CACHE_TTL_SECONDS = int(+ os.environ.get("EXECUTION_CACHE_TTL_SECONDS", 10800)+) # 3 hours+MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))diff --git a/backend/backend/workers/__init__.py b/backend/backend/workers/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/backend/workers/constants.py b/backend/backend/workers/constants.pynew file mode 100644index 000000000..3d1eb83df--- /dev/null+++ b/backend/backend/workers/constants.py@@ -0,0 +1,4 @@+class CeleryWorkerNames:+ """Names of the celery workers."""++ FILE_PROCESSING = "file_processing"diff --git a/backend/backend/workers/file_processing/__init__.py b/backend/backend/workers/file_processing/__init__.pynew file mode 100644index 000000000..7f1a02c21--- /dev/null+++ b/backend/backend/workers/file_processing/__init__.py@@ -0,0 +1,4 @@+from .constants import QueueNames+from .file_processing import app++__all__ = ["app", "QueueNames"]diff --git a/backend/backend/workers/file_processing/celery_config.py b/backend/backend/workers/file_processing/celery_config.pynew file mode 100644index 000000000..629c314fa--- /dev/null+++ b/backend/backend/workers/file_processing/celery_config.py@@ -0,0 +1,20 @@+from kombu import Queue++from backend.celery_config import CeleryConfig as BaseCeleryConfig+from backend.workers.file_processing.constants import QueueNames+++class CeleryConfig(BaseCeleryConfig):+ """Specifies celery configuration for file processing++ Refer https://docs.celeryq.dev/en/stable/userguide/configuration.html+ """++ task_queues = [+ Queue(QueueNames.FILE_PROCESSING, routing_key=QueueNames.FILE_PROCESSING),+ Queue(QueueNames.API_FILE_PROCESSING, routing_key=QueueNames.API_FILE_PROCESSING),+ ]++ task_default_queue = QueueNames.FILE_PROCESSING++ imports = ["workflow_manager.workflow_v2.file_execution_tasks"]diff --git a/backend/backend/workers/file_processing/constants.py b/backend/backend/workers/file_processing/constants.pynew file mode 100644index 000000000..a0a9fa68a--- /dev/null+++ b/backend/backend/workers/file_processing/constants.py@@ -0,0 +1,3 @@+class QueueNames:+ API_FILE_PROCESSING = "api_file_processing"+ FILE_PROCESSING = "file_processing"diff --git a/backend/backend/workers/file_processing/file_processing.py b/backend/backend/workers/file_processing/file_processing.pynew file mode 100644index 000000000..b6c1f975e--- /dev/null+++ b/backend/backend/workers/file_processing/file_processing.py@@ -0,0 +1,31 @@+"""This module contains the Celery configuration for the backend project."""++import logging+import logging.config+import os+from pprint import pformat++from celery import Celery++from backend.settings.base import LOGGING+from backend.workers.constants import CeleryWorkerNames+from backend.workers.file_processing.celery_config import CeleryConfig++logger = logging.getLogger(__name__)++# Set the default Django settings module for the 'celery' program.+os.environ.setdefault(+ "DJANGO_SETTINGS_MODULE",+ os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),+)++# Configure logging for celery worker using same config as Django+logging.config.dictConfig(LOGGING)++# Create a Celery instance. Default time zone is UTC.+app = Celery(CeleryWorkerNames.FILE_PROCESSING)++# Load task modules from all registered Django app configs.+app.config_from_object(CeleryConfig)++logger.debug(f"Celery Configuration:\n {pformat(app.conf.table(with_defaults=True))}")diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 5971e97f7..a3fe83dd2 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -119,6 +119,14 @@ help = "Runs the Unstract log management worker"cmd = "celery -A backend worker --loglevel=info -Q celery_api_deployments --autoscale 4,1"help = "Runs the Unstract API deployment worker"+[tool.poe.tasks.worker-file-processing]+cmd = "celery -A backend.workers.file_processing worker --loglevel=info -Q file_processing --autoscale 4,1"+help = "Runs the Unstract file processing worker"++[tool.poe.tasks.worker-api-file-processing]+cmd = "celery -A backend.workers.file_processing worker --loglevel=info -Q api_file_processing --autoscale 4,1"+help = "Runs the Unstract file processing api worker"+[tool.poe.tasks.flower]cmd = "celery -A backend flower --port=5555"help = "Runs the Unstract Celery Monitoring Tool"diff --git a/backend/sample.env b/backend/sample.envindex e6dd7250c..e7463eb2f 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -168,3 +168,12 @@ TOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'# Highlight data to be available in api deploymentENABLE_HIGHLIGHT_API_DEPLOYMENT=False++# Execution result and cache expire time+# For API results cached per workflow execution+EXECUTION_RESULT_TTL_SECONDS=86400 # 24 hours+# For execution metadata cached per workflow execution+EXECUTION_CACHE_TTL_SECONDS=86400 # 24 hours++# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution+MAX_PARALLEL_FILE_BATCHES=1 # 1 file at a timediff --git a/backend/utils/cache_service.py b/backend/utils/cache_service.pyindex 80f67ee80..8633de58c 100644--- a/backend/utils/cache_service.py+++ b/backend/utils/cache_service.py@@ -1,3 +1,4 @@+import jsonfrom typing import Anyfrom django.conf import settings@@ -77,6 +78,95 @@ class CacheService:def lrange(key, start_index, end_index) -> list[Any]:return redis_cache.lrange(key, start_index, end_index)+ @staticmethod+ def hset(+ key: str,+ field: str = None,+ value: Any = None,+ mapping: dict[str, Any] = None,+ expire_time: int = int(settings.CACHE_TTL_SEC),+ ) -> None:+ """Set Redis hash fields. Supports both single field-value and full mapping.++ Args:+ key (str): The key of the Redis hash.+ field (str, optional): The field to set. Defaults to None.+ value (Any, optional): The value to set. Defaults to None.+ mapping (dict[str, Any], optional): The mapping of fields to values.+ Defaults to None.+ expire_time (int, optional): The expiration time for hash in seconds.+ Defaults to int(settings.CACHE_TTL_SEC).+ """+ if mapping is not None:+ redis_cache.hset(key, mapping=mapping)+ elif field is not None and value is not None:+ redis_cache.hset(key, field, value)+ else:+ raise ValueError("Provide either 'mapping' or both 'field' and 'value'")++ redis_cache.expire(key, expire_time)++ @staticmethod+ def hget(key: str, field: str) -> Any:+ """Get a value from a Redis hash."""+ return redis_cache.hget(key, field)++ @staticmethod+ def hgetall(key: str) -> Any:+ """Get all values from a Redis hash."""+ return redis_cache.hgetall(key)++ @staticmethod+ def hincrby(key: str, field: str, increment: int) -> None:+ """Increment a value in a Redis hash."""+ redis_cache.hincrby(key, field, increment)++ @staticmethod+ def exists(key: str) -> bool:+ """Check if a key exists in Redis."""+ return redis_cache.exists(key)++ @staticmethod+ def rpush_with_expire(+ key: str, value: Any, expire: int = int(settings.CACHE_TTL_SEC)+ ) -> None:+ """Push a value to a Redis list and reset the list's TTL in a single atomic operation.++ Args:+ key (str): The key of the Redis list.+ value (Any): The value to push to the list.+ expire (int, optional): The expiration time for list in seconds.+ Defaults to int(settings.CACHE_TTL_SEC).++ Returns:+ None+ """+ pipe = redis_cache.pipeline()+ pipe.rpush(key, json.dumps(value))+ pipe.expire(key, expire)+ pipe.execute()++ @staticmethod+ def lrange_json(key: str, start_index: int = 0, end_index: int = -1) -> list[Any]:+ """Get all items from list and parse JSON.++ Args:+ key (str): The key of the Redis list.+ start_index (int, optional): The starting index for the range.+ Defaults to 0.+ end_index (int, optional): The ending index for the range.+ Defaults to -1.++ Returns:+ list[Any]: A list of parsed JSON values from the list.+ """+ results = redis_cache.lrange(key, start_index, end_index)+ if not results:+ return []+ return [+ json.loads(r.decode("utf-8") if isinstance(r, bytes) else r) for r in results+ ]+@staticmethoddef remove_all_session_keys(user_id: str | None = None,diff --git a/backend/utils/date/enums.py b/backend/utils/date/enums.pyindex 9a1b3a874..d7d6a4858 100644--- a/backend/utils/date/enums.py+++ b/backend/utils/date/enums.py@@ -1,6 +1,7 @@+from __future__ import annotations+from datetime import datetime, timedeltafrom enum import Enum-from typing import Optionalfrom django.utils import timezone@@ -44,7 +45,7 @@ class DateRangePresets(Enum):return self.get_start_date(), self.get_end_date()@classmethod- def from_value(cls, value: str) -> Optional["DateRangePresets"]:+ def from_value(cls, value: str) -> DateRangePresets | None:try:return next(preset for preset in cls if preset.key == value)except StopIteration as e:diff --git a/backend/utils/dto.py b/backend/utils/dto.pyindex a2be3237e..af10f475f 100644--- a/backend/utils/dto.py+++ b/backend/utils/dto.py@@ -1,7 +1,9 @@+from __future__ import annotations+import jsonimport loggingfrom datetime import UTC, datetime-from typing import Any, Optional+from typing import Anyfrom django.utils import timezone as dj_timezone@@ -43,7 +45,7 @@ class LogDataDTO:self.data: dict[str, Any] = data@classmethod- def from_json(cls, json_data: str) -> Optional["LogDataDTO"]:+ def from_json(cls, json_data: str) -> LogDataDTO | None:try:json_data = json.loads(json_data)execution_id = json_data.get(LogFieldName.EXECUTION_ID)diff --git a/backend/uv.lock b/backend/uv.lockindex ba36766f0..3bbe12f6e 100644--- a/backend/uv.lock+++ b/backend/uv.lock@@ -1699,15 +1699,16 @@ wheels = [[[package]]name = "kombu"-version = "5.3.7"+version = "5.5.3"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "amqp" },+ { name = "tzdata" },{ name = "vine" },]-sdist = { url = "https://files.pythonhosted.org/packages/99/3a/2fb09708fef243e35c286414f2bf78543dc311ae7d3de5d343bd8437e38d/kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf", size = 439344 }+sdist = { url = "https://files.pythonhosted.org/packages/60/0a/128b65651ed8120460fc5af754241ad595eac74993115ec0de4f2d7bc459/kombu-5.5.3.tar.gz", hash = "sha256:021a0e11fcfcd9b0260ef1fb64088c0e92beb976eb59c1dfca7ddd4ad4562ea2", size = 461784 }wheels = [- { url = "https://files.pythonhosted.org/packages/b4/9a/1951f2261271d6994f0df5a55b3e9cdad42ed1fc3020a0dc7f6de80a4566/kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9", size = 200190 },+ { url = "https://files.pythonhosted.org/packages/5d/35/1407fb0b2f5b07b50cbaf97fce09ad87d3bfefbf64f7171a8651cd8d2f68/kombu-5.5.3-py3-none-any.whl", hash = "sha256:5b0dbceb4edee50aa464f59469d34b97864be09111338cfb224a10b6a163909b", size = 209921 },][[package]]@@ -4020,7 +4021,7 @@ dependencies = [[package.metadata]requires-dist = [{ name = "flask", marker = "extra == 'flask'", specifier = "~=3.1.0" },- { name = "kombu", specifier = "==5.3.7" },+ { name = "kombu", specifier = "~=5.5.3" },{ name = "redis", specifier = "~=5.2.1" },{ name = "requests", specifier = "==2.31.0" },]diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.pyindex 199a5d31b..66fec358f 100644--- a/backend/workflow_manager/endpoint_v2/base_connector.py+++ b/backend/workflow_manager/endpoint_v2/base_connector.py@@ -13,13 +13,19 @@ from unstract.workflow_execution.execution_file_handler import ExecutionFileHandclass BaseConnector(ExecutionFileHandler):"""Base class for connectors providing common methods and utilities."""- def __init__(self, workflow_id: str, execution_id: str, organization_id: str) -> None:+ def __init__(+ self,+ workflow_id: str,+ execution_id: str,+ organization_id: str,+ file_execution_id: str | None = None,+ ) -> None:"""Initialize the BaseConnector class.This class serves as a base for connectors and provides commonutilities."""- super().__init__(workflow_id, execution_id, organization_id)+ super().__init__(workflow_id, execution_id, organization_id, file_execution_id)def get_fsspec(self, settings: dict[str, Any], connector_id: str@@ -95,3 +101,20 @@ class BaseConnector(ExecutionFileHandler):workflow_id, execution_id, organization_id)return api_storage_dir++ @classmethod+ def get_execution_dir_path(cls, workflow_id: str, execution_id: str) -> str:+ """Get the directory path for storing execution-related files.++ Parameters:+ - workflow_id (str): Identifier for the workflow.+ - execution_id (str): Identifier for the execution.++ Returns:+ str: The directory path for the execution.+ """+ organization_id = UserContext.get_organization_identifier()+ execution_dir: str = cls.get_execution_dir(+ workflow_id, execution_id, organization_id+ )+ return execution_dirdiff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 7f704b55f..a1795945f 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -14,10 +14,9 @@ from workflow_manager.endpoint_v2.constants import (ApiDeploymentResultStatus,DestinationKey,QueueResultStatus,- WorkflowFileType,)from workflow_manager.endpoint_v2.database_utils import DatabaseUtils-from workflow_manager.endpoint_v2.dto import FileHash+from workflow_manager.endpoint_v2.dto import DestinationConfig, FileHashfrom workflow_manager.endpoint_v2.exceptions import (DestinationConnectorNotConfigured,InvalidDestinationConnectionType,@@ -27,8 +26,8 @@ from workflow_manager.endpoint_v2.exceptions import ()from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtils+from workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.enums import ExecutionStatus-from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -59,7 +58,9 @@ class DestinationConnector(BaseConnector):self,workflow: Workflow,execution_id: str,- execution_service: WorkflowExecutionServiceHelper | None = None,+ workflow_log: WorkflowLog,+ use_file_history: bool,+ file_execution_id: str | None = None,) -> None:"""Initialize a DestinationConnector object.@@ -67,16 +68,17 @@ class DestinationConnector(BaseConnector):workflow (Workflow): _description_"""organization_id = UserContext.get_organization_identifier()- super().__init__(workflow.id, execution_id, organization_id)+ super().__init__(workflow.id, execution_id, organization_id, file_execution_id)self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)self.execution_id = execution_id- self.api_results: list[dict[str, Any]] = []self.queue_results: list[dict[str, Any]] = []- self.execution_service = execution_serviceself.is_api: bool = (self.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API)+ self.workflow_log = workflow_log+ self.use_file_history = use_file_history+ self.workflow = workflowdef _get_endpoint_for_workflow(self,@@ -162,7 +164,7 @@ class DestinationConnector(BaseConnector):file_execution_id: str,) -> bool:"""Handles HITL processing, returning True if data was pushed to the queue."""- execution_result = self.get_result()+ execution_result = self.get_tool_execution_result()if WorkflowUtil.validate_db_rule(execution_result, workflow, file_hash.file_destination):@@ -182,7 +184,7 @@ class DestinationConnector(BaseConnector):input_file_path: str,file_execution_id: str,) -> None:- result = self.get_result()+ result = self.get_tool_execution_result()meta_data = self.get_metadata()self._push_to_queue(file_name=file_name,@@ -199,21 +201,15 @@ class DestinationConnector(BaseConnector):file_hash: FileHash,workflow: Workflow,input_file_path: str,- error: str | None = None,- use_file_history: bool = True,file_execution_id: str = None,- ) -> None:+ ) -> str | None:"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_type- result: str | None = None+ tool_execution_result: str | None = Nonemetadata: str | None = None- if error:- if connection_type == WorkflowEndpoint.ConnectionType.API:- self._handle_api_result(file_name=file_name, error=error, result=result)- returnfile_history = None- if use_file_history:+ if self.use_file_history:file_history = FileHistoryHelper.get_file_history(workflow=workflow, cache_key=file_hash.file_hash)@@ -229,11 +225,7 @@ class DestinationConnector(BaseConnector):):self.insert_into_db(input_file_path=input_file_path)elif connection_type == WorkflowEndpoint.ConnectionType.API:- result = self.get_result(file_history)- exec_metadata = self.get_metadata(file_history)- self._handle_api_result(- file_name=file_name, error=error, result=result, metadata=exec_metadata- )+ tool_execution_result = self.get_tool_execution_result(file_history)elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:self._push_data_to_queue(file_name,@@ -241,20 +233,20 @@ class DestinationConnector(BaseConnector):input_file_path,file_execution_id,)- if self.execution_service:- self.execution_service.publish_log(- message=f"File '{file_name}' processed successfully"- )+ self.workflow_log.publish_log(+ message=f"File '{file_name}' processed successfully"+ )- if use_file_history and not file_history:+ if self.use_file_history and not file_history:FileHistoryHelper.create_file_history(file_hash=file_hash,workflow=workflow,status=ExecutionStatus.COMPLETED,- result=result,+ result=tool_execution_result,metadata=metadata,file_name=file_name,)+ return tool_execution_resultdef copy_output_to_output_directory(self) -> None:"""Copy output to the destination directory."""@@ -273,7 +265,9 @@ class DestinationConnector(BaseConnector):input_dir=output_directory, root_path=root_path)logger.debug(f"destination output directory {output_directory}")- destination_volume_path = os.path.join(self.execution_dir, ToolExecKey.OUTPUT_DIR)+ destination_volume_path = os.path.join(+ self.file_execution_dir, ToolExecKey.OUTPUT_DIR+ )try:destination_fs.create_dir_if_not_exists(input_dir=output_directory)@@ -328,7 +322,7 @@ class DestinationConnector(BaseConnector):execution_id_name = str(destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id"))- data = self.get_result()+ data = self.get_tool_execution_result()# If data is None, don't execute CREATE or INSERT queryif not data:return@@ -418,7 +412,10 @@ class DestinationConnector(BaseConnector):Returns:None"""- api_result: dict[str, Any] = {"file": file_name}+ api_result: dict[str, Any] = {+ "file": file_name,+ "file_execution_id": self.file_execution_id,+ }if error:api_result.update({"status": ApiDeploymentResultStatus.FAILED, "error": error}@@ -436,7 +433,7 @@ class DestinationConnector(BaseConnector):api_result.update({"status": ApiDeploymentResultStatus.SUCCESS, "result": ""})- self.api_results.append(api_result)+ self.update_api_results(api_result)def parse_string(self, original_string: str) -> Any:"""Parse the given string, attempting to evaluate it as a Python@@ -472,15 +469,17 @@ class DestinationConnector(BaseConnector):# assume it's a plain stringreturn original_string- def get_result(self, file_history: FileHistory | None = None) -> Any | None:+ def get_tool_execution_result(+ self, file_history: FileHistory | None = None+ ) -> Any | None:"""Get result data from the output file.Returns:Union[dict[str, Any], str]: Result data."""- return self.get_result_with_file_storage(file_history=file_history)+ return self.get_tool_execution_result_from_metadata(file_history=file_history)- def get_result_with_file_storage(+ def get_tool_execution_result_from_metadata(self, file_history: FileHistory | None = None) -> Any | None:"""Get result data from the output file.@@ -490,7 +489,7 @@ class DestinationConnector(BaseConnector):"""if file_history and file_history.result:return self.parse_string(file_history.result)- output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)+ output_file = self.infilemetadata: dict[str, Any] = self.get_workflow_metadata()output_type = self.get_output_type(metadata)result: dict[str, Any] | str = ""@@ -536,25 +535,28 @@ class DestinationConnector(BaseConnector):return metadata- def delete_execution_directory(self) -> None:- """Delete the execution directory.+ def delete_file_execution_directory(self) -> None:+ """Delete the file execution directory.Returns:None"""file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)file_storage = file_system.get_file_storage()- if file_storage.exists(self.execution_dir):- file_storage.rm(self.execution_dir, recursive=True)+ if self.file_execution_dir and file_storage.exists(self.file_execution_dir):+ file_storage.rm(self.file_execution_dir, recursive=True)- def delete_execution_and_api_storage_dir(self) -> None:+ @classmethod+ def delete_execution_and_api_storage_dir(+ cls, workflow_id: str, execution_id: str+ ) -> None:"""Delete the execution and api storage directories.Returns:None"""- self.delete_execution_directory()- self.delete_api_storage_dir(self.workflow_id, self.execution_id)+ cls.delete_execution_directory(workflow_id, execution_id)+ cls.delete_api_storage_dir(workflow_id, execution_id)@classmethoddef delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:@@ -570,6 +572,23 @@ class DestinationConnector(BaseConnector):file_storage = file_system.get_file_storage()if file_storage.exists(api_storage_dir):file_storage.rm(api_storage_dir, recursive=True)+ logger.info(f"API storage directory deleted: {api_storage_dir}")++ @classmethod+ def delete_execution_directory(cls, workflow_id: str, execution_id: str) -> None:+ """Delete the execution directory.++ Returns:+ None+ """+ execution_dir = cls.get_execution_dir_path(+ workflow_id=workflow_id, execution_id=execution_id+ )+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ if file_storage.exists(execution_dir):+ file_storage.rm(execution_dir, recursive=True)+ logger.info(f"Execution directory deleted: {execution_dir}")@classmethoddef create_endpoint_for_workflow(@@ -621,6 +640,44 @@ class DestinationConnector(BaseConnector):)return cls.get_json_schema(file_path=schema_path)+ def get_config(self) -> DestinationConfig:+ """Get serializable configuration for the destination connector.++ Returns:+ DestinationConfig: Configuration containing all necessary data to reconstruct the connector+ """+ return DestinationConfig(+ workflow_id=self.workflow.id,+ execution_id=self.execution_id,+ use_file_history=self.use_file_history,+ )++ @classmethod+ def from_config(+ cls, workflow_log: WorkflowLog, config: DestinationConfig+ ) -> "DestinationConnector":+ """Create a DestinationConnector instance from configuration.++ Args:+ config (DestinationConfig): Configuration containing all necessary data to reconstruct the connector++ Returns:+ DestinationConnector: New instance+ """+ # Reconstruct workflow+ workflow = Workflow.objects.get(id=config.workflow_id)++ # Create destination connector instance+ destination = cls(+ workflow=workflow,+ execution_id=config.execution_id,+ workflow_log=workflow_log,+ use_file_history=config.use_file_history,+ file_execution_id=config.file_execution_id,+ )++ return destination+def _push_to_queue(self,file_name: str,diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.pyindex 3f21f8ef5..ead8e5c02 100644--- a/backend/workflow_manager/endpoint_v2/dto.py+++ b/backend/workflow_manager/endpoint_v2/dto.py@@ -2,6 +2,10 @@ import jsonfrom dataclasses import dataclassfrom typing import Any+from workflow_manager.endpoint_v2.constants import (+ ApiDeploymentResultStatus,+)+@dataclassclass FileHash:@@ -42,3 +46,99 @@ class FileHash:# Otherwise, assume it's a JSON stringdata = json.loads(json_str_or_dict)return FileHash(**data)+++@dataclass+class SourceConfig:+ workflow_id: str+ execution_id: str+ organization_id: str+ use_file_history: bool+ file_execution_id: str | None = None++ def to_json(self) -> dict[str, Any]:+ """Serialize the SourceConfig instance to a JSON string."""+ file_execution_id = (+ str(self.file_execution_id) if self.file_execution_id else None+ )+ return {+ "workflow_id": str(self.workflow_id),+ "execution_id": str(self.execution_id),+ "organization_id": str(self.organization_id),+ "use_file_history": self.use_file_history,+ "file_execution_id": file_execution_id,+ }++ @staticmethod+ def from_json(json_str_or_dict: Any) -> "SourceConfig":+ """Deserialize a JSON string or dictionary to a SourceConfig instance."""+ if isinstance(json_str_or_dict, dict):+ data = json_str_or_dict+ else:+ data = json.loads(json_str_or_dict)+ return SourceConfig(**data)+++@dataclass+class DestinationConfig:+ workflow_id: str+ execution_id: str+ use_file_history: bool+ file_execution_id: str | None = None++ def to_json(self) -> dict[str, Any]:+ """Serialize the DestinationConfig instance to a JSON string."""+ file_execution_id = (+ str(self.file_execution_id) if self.file_execution_id else None+ )+ return {+ "workflow_id": str(self.workflow_id),+ "execution_id": str(self.execution_id),+ "use_file_history": self.use_file_history,+ "file_execution_id": file_execution_id,+ }++ @staticmethod+ def from_json(json_str_or_dict: Any) -> "DestinationConfig":+ """Deserialize a JSON string or dictionary to a DestinationConfig instance."""+ if isinstance(json_str_or_dict, dict):+ data = json_str_or_dict+ else:+ data = json.loads(json_str_or_dict)+ return DestinationConfig(**data)+++@dataclass+class FileExecutionResult:+ file: str+ file_execution_id: str+ status: str | None = None+ result: str | None = None+ error: str | None = None+ metadata: dict[str, Any] | None = None++ def __post_init__(self) -> None:+ if self.error:+ self.status = ApiDeploymentResultStatus.FAILED+ else:+ self.status = ApiDeploymentResultStatus.SUCCESS++ def to_json(self) -> dict[str, Any]:+ """Serialize the FileExecutionResult instance to a JSON string."""+ return {+ "file": self.file,+ "file_execution_id": self.file_execution_id,+ "status": self.status,+ "result": self.result,+ "error": self.error,+ "metadata": self.metadata,+ }++ @staticmethod+ def from_json(json_str_or_dict: Any) -> "FileExecutionResult":+ """Deserialize a JSON string or dictionary to a FileExecutionResult instance."""+ if isinstance(json_str_or_dict, dict):+ data = json_str_or_dict+ else:+ data = json.loads(json_str_or_dict)+ return FileExecutionResult(**data)diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.pyindex 4eef455b9..e994627c8 100644--- a/backend/workflow_manager/endpoint_v2/exceptions.py+++ b/backend/workflow_manager/endpoint_v2/exceptions.py@@ -107,3 +107,8 @@ class UnstractQueueException(APIException):def __init__(self, detail: str = default_detail) -> None:status_code = 500super().__init__(detail=detail, code=status_code)+++class SourceFileOrInfilePathNotFound(APIException):+ status_code = 500+ default_detail = "Unable to obtain file (Source or Infile) for execution."diff --git a/backend/workflow_manager/endpoint_v2/result_cache_utils.py b/backend/workflow_manager/endpoint_v2/result_cache_utils.pynew file mode 100644index 000000000..6fe94a48c--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/result_cache_utils.py@@ -0,0 +1,40 @@+from typing import Any++from django.conf import settings+from utils.cache_service import CacheService+from workflow_manager.endpoint_v2.dto import FileExecutionResult+++class ResultCacheUtils:+ expire_time = int(settings.EXECUTION_RESULT_TTL_SECONDS)++ @staticmethod+ def _get_api_results_cache_key(workflow_id: str, execution_id: str) -> str:+ """Get Redis cache key for api_results."""+ return f"api_results:{workflow_id}:{execution_id}"++ @classmethod+ def get_api_results(cls, workflow_id: str, execution_id: str) -> list[dict[str, Any]]:+ """Get api_results from Redis cache."""+ cache_key = cls._get_api_results_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ return CacheService.lrange_json(cache_key)++ @classmethod+ def update_api_results(+ cls, workflow_id: str, execution_id: str, api_result: FileExecutionResult+ ) -> None:+ """Update api_results in Redis cache."""+ cache_key = cls._get_api_results_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ CacheService.rpush_with_expire(cache_key, api_result.to_json(), cls.expire_time)++ @classmethod+ def delete_api_results(cls, workflow_id: str, execution_id: str) -> None:+ """Delete api_results from Redis cache."""+ cache_key = cls._get_api_results_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ CacheService.delete_a_key(cache_key)diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 02835e564..46ae39553 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -19,19 +19,19 @@ from workflow_manager.endpoint_v2.constants import (FileType,SourceConstant,SourceKey,- WorkflowFileType,)-from workflow_manager.endpoint_v2.dto import FileHash+from workflow_manager.endpoint_v2.dto import FileHash, SourceConfigfrom workflow_manager.endpoint_v2.exceptions import (InvalidInputDirectory,InvalidSourceConnectionType,MissingSourceConnectionType,OrganizationIdNotFound,SourceConnectorNotConfigured,+ SourceFileOrInfilePathNotFound,)from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.file_execution.models import WorkflowFileExecution-from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper+from workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -65,8 +65,10 @@ class SourceConnector(BaseConnector):self,workflow: Workflow,execution_id: str,+ workflow_log: WorkflowLog,+ use_file_history: bool,organization_id: str | None = None,- execution_service: WorkflowExecutionServiceHelper | None = None,+ file_execution_id: str | None = None,) -> None:"""Create a SourceConnector.@@ -84,13 +86,14 @@ class SourceConnector(BaseConnector):organization_id = organization_id or UserContext.get_organization_identifier()if not organization_id:raise OrganizationIdNotFound()- super().__init__(workflow.id, execution_id, organization_id)+ super().__init__(workflow.id, execution_id, organization_id, file_execution_id)self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)self.workflow = workflowself.execution_id = execution_idself.organization_id = organization_idself.hash_value_of_file_content: str | None = None- self.execution_service = execution_service+ self.workflow_log = workflow_log+ self.use_file_history = use_file_historydef _get_endpoint_for_workflow(self,@@ -226,36 +229,30 @@ class SourceConnector(BaseConnector):msg (str): Message to log"""logger.info(msg)- if self.execution_service:- self.execution_service.publish_log(msg)+ self.workflow_log.publish_log(message=msg)def publish_input_output_list_file_logs(self, folders: list[str], matched_files: dict[str, FileHash], count: int) -> None:- if not self.execution_service:- return None-folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)input_log = f"## Folders to process:\n\n{folders_list}\n\n"- self.execution_service.publish_update_log(+ self.workflow_log.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log)output_log = self._matched_files_component_log(matched_files, count)- self.execution_service.publish_update_log(+ self.workflow_log.publish_update_log(state=LogState.OUTPUT_UPDATE, message=output_log)def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:- if not self.execution_service:- return Noneoutput_log_message = f"## Input text:\n\n```text\n{input_text}\n```\n\n"input_log_message = (f"## Input file:\n\n```text\n{os.path.basename(input_file_path)}\n```\n\n")- self.execution_service.publish_update_log(+ self.workflow_log.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log_message)- self.execution_service.publish_update_log(+ self.workflow_log.publish_update_log(state=LogState.OUTPUT_UPDATE, message=output_log_message)@@ -357,7 +354,7 @@ class SourceConnector(BaseConnector):if matches_blocked:message = f"Skipping '{file_name}' as it has an unsupported file format"logger.debug(message)- self.execution_service.publish_log(message)+ self.workflow_log.publish_log(message)return Falsereturn True@@ -369,12 +366,8 @@ class SourceConnector(BaseConnector):file_history = self._get_file_history(workflow, source_fs, file_path)# In case of ETL pipelines, its necessary to skip files which have# already been processed- if (- self.execution_service.use_file_history- and file_history- and file_history.is_completed()- ):- self.execution_service.publish_log(+ if self.use_file_history and file_history and file_history.is_completed():+ self.workflow_log.publish_log(f"Skipping file {file_path} as it has already been processed. ""Clear the file markers to process it again.")@@ -505,6 +498,8 @@ class SourceConnector(BaseConnector):input_file_path = file_hash.file_pathsource_file_path = self.source_fileinfile_path = self.infile+ if not source_file_path or not infile_path:+ raise SourceFileOrInfilePathNotFound()connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatasource_fs = self.get_fsspec(@@ -544,8 +539,10 @@ class SourceConnector(BaseConnector):def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:"""Add input file to execution directory from api storage."""- infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)- source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)+ source_file_path = self.source_file+ infile_path = self.infile+ if not source_file_path or not infile_path:+ raise SourceFileOrInfilePathNotFound()api_file_system = FileSystem(FileStorageType.API_EXECUTION)api_file_storage = api_file_system.get_file_storage()@@ -555,7 +552,7 @@ class SourceConnector(BaseConnector):source_storage=api_file_storage,destination_storage=workflow_file_storage,source_path=input_file_path,- destination_paths=[infile_path, source_path],+ destination_paths=[infile_path, source_file_path],)logger.info(f"File {input_file_path} added to execution directory")return file_content_hash@@ -770,3 +767,45 @@ class SourceConnector(BaseConnector):os.path.dirname(__file__), "static", "src", "file.json")return cls.get_json_schema(file_path=schema_path)++ def get_config(self) -> SourceConfig:+ """Get serializable configuration for the source connector.++ Returns:+ SourceConfig: Configuration containing all necessary data to reconstruct the connector+ """+ source_config = SourceConfig(+ workflow_id=self.workflow.id,+ execution_id=self.execution_id,+ organization_id=self.organization_id,+ use_file_history=self.use_file_history,+ )+ return source_config++ @classmethod+ def from_config(+ cls, workflow_log: WorkflowLog, config: SourceConfig+ ) -> "SourceConnector":+ """Create a SourceConnector instance from configuration.++ Args:+ workflow_log (WorkflowLog): Workflow log instance+ config (SourceConfig): Configuration containing all necessary data to reconstruct the connector++ Returns:+ SourceConnector: New instance+ """+ # Reconstruct workflow+ workflow = Workflow.objects.get(id=config.workflow_id)++ # Create source connector instance+ source = cls(+ workflow=workflow,+ execution_id=config.execution_id,+ workflow_log=workflow_log,+ use_file_history=config.use_file_history,+ organization_id=config.organization_id,+ file_execution_id=config.file_execution_id,+ )++ return sourcediff --git a/backend/workflow_manager/execution/dto.py b/backend/workflow_manager/execution/dto.pynew file mode 100644index 000000000..ee3915f66--- /dev/null+++ b/backend/workflow_manager/execution/dto.py@@ -0,0 +1,38 @@+from dataclasses import dataclass+from typing import Any++from workflow_manager.workflow_v2.enums import ExecutionStatus+++class ExecutionCacheFields:+ WORKFLOW_ID = "workflow_id"+ EXECUTION_ID = "execution_id"+ STATUS = "status"+ TOTAL_FILES = "total_files"+ COMPLETED_FILES = "completed_files"+ FAILED_FILES = "failed_files"+++@dataclass+class ExecutionCache:+ workflow_id: str+ execution_id: str+ status: ExecutionStatus+ total_files: int = 0+ completed_files: int = 0+ failed_files: int = 0++ def __post_init__(self):+ self.workflow_id = str(self.workflow_id)+ self.execution_id = str(self.execution_id)+ self.status = ExecutionStatus(self.status)++ def to_json(self) -> dict[str, Any]:+ return {+ ExecutionCacheFields.WORKFLOW_ID: self.workflow_id,+ ExecutionCacheFields.EXECUTION_ID: self.execution_id,+ ExecutionCacheFields.STATUS: self.status.value,+ ExecutionCacheFields.TOTAL_FILES: self.total_files,+ ExecutionCacheFields.COMPLETED_FILES: self.completed_files,+ ExecutionCacheFields.FAILED_FILES: self.failed_files,+ }diff --git a/backend/workflow_manager/execution/execution_cache_utils.py b/backend/workflow_manager/execution/execution_cache_utils.pynew file mode 100644index 000000000..a20d1e1a2--- /dev/null+++ b/backend/workflow_manager/execution/execution_cache_utils.py@@ -0,0 +1,95 @@+from django.conf import settings+from utils.cache_service import CacheService++from workflow_manager.execution.dto import ExecutionCache, ExecutionCacheFields+from workflow_manager.workflow_v2.enums import ExecutionStatus+++class ExecutionCacheUtils:+ """Utility class for accessing and managing workflow execution status and+ related information from cache to reduce database load.+ """++ expire_time = int(settings.EXECUTION_CACHE_TTL_SECONDS)++ @staticmethod+ def _get_execution_cache_key(workflow_id: str, execution_id: str) -> str:+ """Get Redis cache key for execution."""+ return f"execution:{workflow_id}:{execution_id}"++ @classmethod+ def get_execution(cls, workflow_id: str, execution_id: str) -> ExecutionCache | None:+ """Get or create execution."""+ cache_key = cls._get_execution_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ execution = CacheService.hgetall(cache_key)+ if execution:+ # Decode keys and values from bytes to strings+ decoded = {k.decode(): v.decode() for k, v in execution.items()}+ return ExecutionCache(**decoded)+ else:+ return None++ @classmethod+ def is_execution_exists(cls, workflow_id: str, execution_id: str) -> bool:+ """Check if execution exists."""+ cache_key = cls._get_execution_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ return CacheService.check_a_key_exist(cache_key)++ @classmethod+ def create_execution(cls, execution: ExecutionCache) -> None:+ """Create execution."""+ cache_key = cls._get_execution_cache_key(+ workflow_id=execution.workflow_id, execution_id=execution.execution_id+ )+ CacheService.hset(+ cache_key, mapping=execution.to_json(), expire_time=cls.expire_time+ )++ @classmethod+ def update_status(+ cls, workflow_id: str, execution_id: str, status: ExecutionStatus+ ) -> None:+ """Update execution status."""+ try:+ status_enum = ExecutionStatus(status)+ except ValueError:+ raise ValueError(+ f"Invalid status: {status}. Must be a valid ExecutionStatus."+ )+ cache_key = cls._get_execution_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ CacheService.hset(+ cache_key,+ field=ExecutionCacheFields.STATUS,+ value=status_enum.value,+ expire_time=cls.expire_time,+ )++ @classmethod+ def increment_completed_files(cls, workflow_id: str, execution_id: str) -> None:+ """Increment completed files."""+ cache_key = cls._get_execution_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ CacheService.hincrby(cache_key, ExecutionCacheFields.COMPLETED_FILES, 1)++ @classmethod+ def increment_failed_files(cls, workflow_id: str, execution_id: str) -> None:+ """Increment failed files."""+ cache_key = cls._get_execution_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ CacheService.hincrby(cache_key, ExecutionCacheFields.FAILED_FILES, 1)++ @classmethod+ def delete_execution(cls, workflow_id: str, execution_id: str) -> None:+ """Delete execution."""+ cache_key = cls._get_execution_cache_key(+ workflow_id=workflow_id, execution_id=execution_id+ )+ CacheService.delete_a_key(cache_key)diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.pyindex d96f1f0f4..ce8434b77 100644--- a/backend/workflow_manager/file_execution/models.py+++ b/backend/workflow_manager/file_execution/models.py@@ -7,7 +7,6 @@ from utils.common_utils import CommonUtilsfrom utils.models.base_model import BaseModelfrom workflow_manager.endpoint_v2.dto import FileHash-from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.workflow_v2.enums import ExecutionStatusFILE_NAME_LENGTH = 255@@ -21,7 +20,7 @@ class WorkflowFileExecutionManager(models.Manager):self,workflow_execution: Any,file_hash: FileHash,- connection_type: WorkflowEndpoint.ConnectionType,+ is_api: bool = False,) -> "WorkflowFileExecution":"""Retrieves or creates a new input file record for a workflow execution.@@ -34,7 +33,6 @@ class WorkflowFileExecutionManager(models.Manager):Returns:The `WorkflowFileExecution` object."""- is_api = connection_type == WorkflowEndpoint.ConnectionType.API# Determine file path based on connection typeexecution_file_path = file_hash.file_path if not is_api else Nonediff --git a/backend/workflow_manager/utils/__init__.py b/backend/workflow_manager/utils/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.pynew file mode 100644index 000000000..1769f527e--- /dev/null+++ b/backend/workflow_manager/utils/workflow_log.py@@ -0,0 +1,91 @@+import logging++from account_v2.constants import Common+from utils.local_context import StateStore++from unstract.core.pubsub_helper import LogPublisher+from unstract.workflow_execution.enums import (+ LogComponent,+ LogLevel,+ LogStage,+ LogState,+)+++class WorkflowLog:+ def __init__(+ self,+ execution_id: str,+ log_stage: LogStage,+ file_execution_id: str | None = None,+ organization_id: str | None = None,+ pipeline_id: str | None = None,+ ):+ log_events_id: str | None = StateStore.get(Common.LOG_EVENTS_ID)+ self.messaging_channel = log_events_id if log_events_id else pipeline_id+ self.execution_id = str(execution_id)+ self.file_execution_id = str(file_execution_id) if file_execution_id else None+ self.organization_id = str(organization_id) if organization_id else None+ self.log_stage = log_stage++ def publish_log(+ self,+ message: str,+ level: LogLevel = LogLevel.INFO,+ step: int | None = None,+ ) -> None:+ log_details = LogPublisher.log_workflow(+ stage=self.log_stage.value,+ message=message,+ level=level.value,+ step=step,+ execution_id=self.execution_id,+ file_execution_id=self.file_execution_id,+ organization_id=self.organization_id,+ )+ LogPublisher.publish(self.messaging_channel, log_details)++ def log_error(self, logger: logging.Logger, message: str) -> None:+ self.publish_log(message, level=LogLevel.ERROR)+ logger.error(message, exc_info=True, stack_info=True)++ def log_info(self, message: str) -> None:+ self.publish_log(message, level=LogLevel.INFO)++ def publish_update_log(+ self,+ state: LogState,+ message: str,+ component: str | LogComponent | None = None,+ ) -> None:+ """Publishes update logs for monitoring the execution of a workflow.++ Args:+ state (LogState): The state of the log, such as "RUN" or "COMPILE".+ message (str): The log message to be published.+ component (LogComponent, optional): The component associated+ with the log. Defaults to None.++ Returns:+ None+ """+ if isinstance(component, LogComponent):+ component = component.value++ log_details = LogPublisher.log_workflow_update(state.value, message, component)+ LogPublisher.publish(self.messaging_channel, log_details)++ def publish_initial_workflow_logs(self, total_files: int) -> None:+ """Publishes the initial logs for the workflow.++ Args:+ total_files (int): The total number of matched files.++ Returns:+ None+ """+ self.publish_log(f"Total matched files: {total_files}")+ self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)+ self.publish_update_log(+ LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW+ )diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.pyindex 7d2859166..fba0f3460 100644--- a/backend/workflow_manager/workflow_v2/dto.py+++ b/backend/workflow_manager/workflow_v2/dto.py@@ -1,8 +1,13 @@-from dataclasses import dataclass+from __future__ import annotations++from dataclasses import asdict, dataclass, fieldsfrom typing import Anyfrom celery.result import AsyncResult+from workflow_manager.endpoint_v2.dto import DestinationConfig, SourceConfig+from workflow_manager.file_execution.models import WorkflowFileExecution+from workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.constants import WorkflowKey@@ -135,3 +140,73 @@ class AsyncResultData:"status": self.status,"result": self.result,}+++@dataclass+class FileData:+ workflow_id: str+ source_config: dict[str, Any]+ destination_config: dict[str, Any]+ execution_id: str+ single_step: bool+ organization_id: str+ pipeline_id: str+ scheduled: bool+ execution_mode: str+ use_file_history: bool++ @classmethod+ def from_dict(cls, data: dict[str, Any]) -> FileData:+ field_names = {f.name for f in fields(cls)}+ filtered_data = {k: v for k, v in data.items() if k in field_names}+ return cls(**filtered_data)++ def __str__(self) -> str:+ return f"FileData({self.__dict__})"++ def to_dict(self) -> dict[str, Any]:+ return asdict(self)+++@dataclass+class FileBatchData:+ files: list+ file_data: FileData++ @classmethod+ def from_dict(cls, data: dict[str, Any]) -> FileBatchData:+ file_data = FileData.from_dict(data["file_data"])+ return cls(files=data["files"], file_data=file_data)++ def __str__(self) -> str:+ return f"FileBatchData(files={self.files}, file_data={self.file_data})"++ def to_dict(self) -> dict[str, Any]:+ return asdict(self)+++@dataclass+class ToolExecutionResult:+ error: str | None+ result: Any | None++ def to_dict(self) -> dict[str, Any]:+ return {+ "error": self.error,+ "result": self.result,+ }+++@dataclass+class FinalOutputResult:+ output: Any | None+ metadata: dict[str, Any] | None+ error: str | None+++@dataclass+class ExecutionContext:+ workflow_log: WorkflowLog+ workflow_file_execution: WorkflowFileExecution+ source_config: SourceConfig+ destination_config: DestinationConfigdiff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.pyindex 8cb77e9ae..9dbe78321 100644--- a/backend/workflow_manager/workflow_v2/enums.py+++ b/backend/workflow_manager/workflow_v2/enums.py@@ -1,3 +1,5 @@+from __future__ import annotations+from enum import Enumfrom django.db.models import TextChoices@@ -31,6 +33,17 @@ class ExecutionStatus(TextChoices):STOPPED = "STOPPED"ERROR = "ERROR"+ @classmethod+ def is_completed(cls, status: str | ExecutionStatus) -> bool:+ """Check if the execution status is completed."""+ try:+ status_enum = cls(status)+ except ValueError:+ raise ValueError(+ f"Invalid status: {status}. Must be a valid ExecutionStatus."+ )+ return status_enum in [cls.COMPLETED, cls.STOPPED, cls.ERROR]+class SchemaType(Enum):"""Possible types for workflow module's JSON schema.diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 469b39497..720108612 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -45,7 +45,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,workflow_execution: WorkflowExecution | None = None,use_file_history: bool = True,+ file_execution_id: str | None = None,) -> None:+ self.file_execution_id = file_execution_idtool_instances_as_dto = []for tool_instance in tool_instances:tool_instances_as_dto.append(@@ -66,6 +68,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):tool_instances=tool_instances_as_dto,platform_service_api_key=str(platform_key.key),ignore_processed_entities=False,+ file_execution_id=file_execution_id,)if not workflow_execution:# Use pipline_id for pipelines / API deployment@@ -146,7 +149,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):else WorkflowExecution.Type.COMPLETE)execution_log_id = log_events_id if log_events_id else pipeline_id-# Create the workflow executionworkflow_execution = WorkflowExecution.objects.create(id=execution_id if execution_id else uuid.uuid4(),@@ -253,7 +255,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):start_time = time.time()try:self.execute_workflow(- file_execution_id=file_execution_id,execution_type=execution_type,)end_time = time.time()diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pynew file mode 100644index 000000000..49fc52d9b--- /dev/null+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -0,0 +1,638 @@+import logging+from typing import Any+from uuid import UUID++from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil+from tool_instance_v2.constants import ToolInstanceKey+from tool_instance_v2.models import ToolInstance+from tool_instance_v2.tool_instance_helper import ToolInstanceHelper+from utils.constants import Account+from utils.local_context import StateStore++from backend.workers.file_processing.constants import QueueNames+from backend.workers.file_processing.file_processing import app as file_processing_app+from unstract.workflow_execution.enums import LogComponent, LogStage, LogState+from unstract.workflow_execution.exceptions import StopExecution+from workflow_manager.endpoint_v2.destination import DestinationConnector+from workflow_manager.endpoint_v2.dto import (+ DestinationConfig,+ FileExecutionResult,+ FileHash,+ SourceConfig,+)+from workflow_manager.endpoint_v2.models import WorkflowEndpoint+from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils+from workflow_manager.endpoint_v2.source import SourceConnector+from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils+from workflow_manager.file_execution.models import WorkflowFileExecution+from workflow_manager.utils.workflow_log import WorkflowLog+from workflow_manager.workflow_v2.dto import (+ ExecutionContext,+ FileBatchData,+ FileData,+ FinalOutputResult,+ ToolExecutionResult,+)+from workflow_manager.workflow_v2.enums import ExecutionStatus+from workflow_manager.workflow_v2.exceptions import (+ WorkflowDoesNotExistError,+ WorkflowExecutionError,+)+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper+from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper+from workflow_manager.workflow_v2.models.execution import WorkflowExecution+from workflow_manager.workflow_v2.models.workflow import Workflow++logger = logging.getLogger(__name__)+++class FileExecutionTasks:+ @staticmethod+ def get_queue_name(source: SourceConnector) -> str:+ is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API+ return QueueNames.API_FILE_PROCESSING if is_api else QueueNames.FILE_PROCESSING++ @staticmethod+ def get_workflow_by_id(id: str) -> Workflow:+ try:+ workflow: Workflow = Workflow.objects.get(pk=id)+ return workflow+ except Workflow.DoesNotExist:+ logger.error(f"Error getting workflow: {id}")+ raise WorkflowDoesNotExistError()++ @staticmethod+ def _build_workflow_execution_service(+ organization_id: str | None,+ workflow: Workflow,+ tool_instances: list[ToolInstance],+ pipeline_id: str | None,+ single_step: bool,+ scheduled: bool,+ execution_mode: tuple[str, str],+ workflow_execution: WorkflowExecution,+ use_file_history: bool = True, # Will be False for API deployment alone+ file_execution_id: str | None = None,+ ) -> WorkflowExecutionServiceHelper:+ workflow_execution_service = WorkflowExecutionServiceHelper(+ organization_id=organization_id,+ workflow=workflow,+ tool_instances=tool_instances,+ pipeline_id=pipeline_id,+ single_step=single_step,+ scheduled=scheduled,+ mode=execution_mode,+ workflow_execution=workflow_execution,+ use_file_history=use_file_history,+ file_execution_id=file_execution_id,+ )+ workflow_execution_service.build()+ return workflow_execution_service++ @file_processing_app.task(+ bind=True,+ max_retries=0, # Maximum number of retries+ ignore_result=False, # result is passed to the callback task+ retry_backoff=True,+ retry_backoff_max=500, # Max delay between retries (500 seconds)+ retry_jitter=True, # Add random jitter to prevent thundering herd+ default_retry_delay=5, # Initial retry delay (5 seconds)+ )+ def process_file_batch(self, file_batch_data) -> dict[str, Any]:+ """Process a batch of files in parallel using Celery.++ Args:+ file_batch_data (dict): Contains all necessary data to process files in the batch+ - files: List of (file_name, file_hash) tuples+ - workflow_id: ID of the workflow+ - source_config: Source connector configuration+ - destination_config: Destination connector configuration+ - execution_id: ID of execution service+ - single_step: Whether to process in single step mode+ """+ file_batch_data = FileBatchData.from_dict(file_batch_data)+ file_data = file_batch_data.file_data+ logger.info(f"File processing context {file_data}")++ organization_id = file_data.organization_id+ StateStore.set(Account.ORGANIZATION_ID, organization_id)++ # Use proper logger instead of print+ logger.info(+ f"Starting to process file batch for execution {file_data.execution_id}"+ )+ logger.debug(f"Batch data received: {file_batch_data}")++ successful_files = 0+ failed_files = 0+ execution_id = file_data.execution_id+ workflow_id = file_data.workflow_id+ # Reconstruct necessary objects+ workflow = FileExecutionTasks.get_workflow_by_id(str(workflow_id))+ workflow_execution = WorkflowExecution.objects.get(id=UUID(execution_id))++ total_files = len(file_batch_data.files)+ q_file_no_list = (+ WorkflowUtil.get_q_no_list(workflow, total_files) if total_files > 0 else []+ )++ logger.info(+ f"Processing {total_files} files of execution {execution_id} in a batch"+ )++ for file_number, (file_name, file_hash_dict) in enumerate(+ file_batch_data.files, 1+ ):+ file_hash = FileHash(+ file_path=file_hash_dict.get("file_path"),+ file_name=file_hash_dict.get("file_name"),+ source_connection_type=file_hash_dict.get("source_connection_type"),+ file_hash=file_hash_dict.get("file_hash"),+ file_size=file_hash_dict.get("file_size"),+ provider_file_uuid=file_hash_dict.get("provider_file_uuid"),+ mime_type=file_hash_dict.get("mime_type"),+ fs_metadata=file_hash_dict.get("fs_metadata"),+ file_destination=file_hash_dict.get("file_destination"),+ is_executed=file_hash_dict.get("is_executed"),+ )+ file_hash = WorkflowUtil.add_file_destination_filehash(+ file_number,+ q_file_no_list,+ file_hash,+ )+ file_execution_result = FileExecutionTasks._process_file(+ current_file_idx=file_number,+ total_files=total_files,+ file_data=file_data,+ file_hash=file_hash,+ workflow_execution=workflow_execution,+ )+ if file_execution_result.error:+ failed_files += 1+ ExecutionCacheUtils.increment_failed_files(+ workflow_id=workflow.id,+ execution_id=execution_id,+ )+ else:+ successful_files += 1+ ExecutionCacheUtils.increment_completed_files(+ workflow_id=workflow.id,+ execution_id=execution_id,+ )+ return {+ "successful_files": successful_files,+ "failed_files": failed_files,+ }++ @file_processing_app.task(+ bind=True,+ max_retries=0, # Maximum number of retries+ retry_backoff=True,+ retry_backoff_max=500, # Max delay between retries (500 seconds)+ retry_jitter=True, # Add random jitter to prevent thundering herd+ default_retry_delay=5, # Initial retry delay (5 seconds)+ )+ def process_batch_callback(self, results, **kwargs):+ """Callback task to handle batch processing results.++ Args:+ results (list): List of results from each batch+ Each result is a dictionary containing:+ - successful_files: Number of successfully processed files+ - failed_files: Number of failed files+ - error_messages: List of error messages+ - execution_id: ID of the execution+ """+ execution_id = kwargs.get("execution_id")+ workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(+ id=execution_id+ )+ workflow = workflow_execution.workflow+ organization = workflow.organization+ organization_id = organization.organization_id++ # Set organization ID in StateStore+ StateStore.set(Account.ORGANIZATION_ID, organization_id)++ if not results:+ return None++ # Aggregate results+ total_successful = sum(result["successful_files"] for result in results)+ total_failed = sum(result["failed_files"] for result in results)++ total_files = total_successful + total_failed++ # Update final status+ final_status = (+ ExecutionStatus.COMPLETED if total_successful else ExecutionStatus.ERROR+ )+ error_message = None+ # Update execution service with final results+ workflow_execution.update_execution(+ status=final_status,+ error=error_message,+ )+ # clean up execution and api storage directories+ DestinationConnector.delete_execution_and_api_storage_dir(+ workflow_id=workflow.id, execution_id=execution_id+ )+ return {+ "execution_id": execution_id,+ "total_files": total_files,+ "successful_files": total_successful,+ "failed_files": total_failed,+ "error_message": error_message,+ "status": final_status,+ }++ @classmethod+ def _process_file(+ cls,+ current_file_idx: int,+ total_files: int,+ file_data: FileData,+ file_hash: FileHash,+ workflow_execution: WorkflowExecution,+ ) -> FileExecutionResult:+ """Process a single file in the workflow.++ Args:+ current_file_idx (int): Index of the current file+ total_files (int): Total number of files+ file_data (FileData): File data+ file_hash (FileHash): File hash+ workflow_execution (WorkflowExecution): Workflow execution instance++ Returns:+ FileExecutionResult: Result of the file execution+ """+ try:+ # Initialization Phase+ execution_context = cls._initialize_execution_context(+ file_data, file_hash, workflow_execution+ )+ workflow_log = execution_context.workflow_log+ workflow_file_execution = execution_context.workflow_file_execution+ source_config = execution_context.source_config+ destination_config = execution_context.destination_config+ source = SourceConnector.from_config(workflow_log, source_config)+ destination = DestinationConnector.from_config(+ workflow_log, destination_config+ )+ destination.delete_file_execution_directory()++ # File Preparation Phase+ content_hash = cls._prepare_file_for_processing(+ source,+ workflow_log,+ workflow_file_execution,+ file_hash,+ workflow_execution,+ )++ # History Check Phase+ if early_result := cls._check_processing_history(+ destination,+ workflow_execution.workflow,+ content_hash,+ file_hash,+ workflow_log,+ workflow_file_execution,+ ):+ return early_result++ # Core Execution Phase+ execution_result = cls._execute_workflow_steps(+ file_data,+ workflow_execution,+ workflow_file_execution,+ file_hash,+ current_file_idx,+ total_files,+ )++ # Finalization Phase+ return cls._finalize_execution(+ workflow_execution,+ workflow_file_execution,+ file_hash,+ workflow_log,+ destination,+ execution_result,+ )+ except Exception as error:+ error_msg = f"File execution failed: {error}"+ workflow_log.log_error(logger=logger, message=error_msg)+ workflow_file_execution.update_status(+ status=ExecutionStatus.ERROR, execution_error=error_msg[:500]+ )+ result = FinalOutputResult(output=None, metadata=None, error=error_msg)+ return cls._build_final_result(+ workflow_execution=workflow_execution,+ workflow_file_execution=workflow_file_execution,+ file_hash=file_hash,+ error=error_msg,+ result=result,+ )++ @classmethod+ def _initialize_execution_context(+ cls,+ file_data: FileData,+ file_hash: FileHash,+ workflow_execution: WorkflowExecution,+ ) -> ExecutionContext:+ """Set up all required execution context objects and configurations."""+ workflow = workflow_execution.workflow+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ )++ is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API++ # Create execution record+ file_execution: WorkflowFileExecution = (+ WorkflowFileExecution.objects.get_or_create_file_execution(+ workflow_execution=workflow_execution,+ file_hash=file_hash,+ is_api=is_api,+ )+ )++ # Create configurations+ source_config = SourceConfig.from_json(file_data.source_config)+ destination_config = DestinationConfig.from_json(file_data.destination_config)+ source_config.file_execution_id = destination_config.file_execution_id = str(+ file_execution.id+ )++ # Initialize logging+ workflow_log = WorkflowLog(+ execution_id=str(workflow_execution.id),+ log_stage=LogStage.PROCESSING,+ organization_id=file_data.organization_id,+ pipeline_id=str(workflow_execution.pipeline_id),+ file_execution_id=str(file_execution.id),+ )++ return ExecutionContext(+ workflow_log=workflow_log,+ workflow_file_execution=file_execution,+ source_config=source_config,+ destination_config=destination_config,+ )++ @classmethod+ def _prepare_file_for_processing(+ cls,+ source: SourceConnector,+ workflow_log: WorkflowLog,+ workflow_file_exec: WorkflowFileExecution,+ file_hash: FileHash,+ workflow_execution: WorkflowExecution,+ ) -> str:+ """Handle file preparation and volume storage."""+ workflow_file_exec.update_status(ExecutionStatus.EXECUTING)++ try:+ content_hash = source.add_file_to_volume(+ workflow_file_execution=workflow_file_exec,+ tags=workflow_execution.tag_names,+ file_hash=file_hash,+ )+ file_hash.file_hash = content_hash+ workflow_file_exec.update(file_hash=content_hash)+ return content_hash+ except FileNotFoundError as error:+ error_msg = f"File not Found in execution dir: {error}"+ workflow_log.log_error(logger=logger, message=error_msg)+ raise WorkflowExecutionError(error_msg) from error+ except Exception as error:+ error_msg = f"File preparation failed: {error}"+ workflow_log.log_error(logger=logger, message=error_msg)+ raise WorkflowExecutionError(error_msg) from error++ @classmethod+ def _check_processing_history(+ cls,+ destination: DestinationConnector,+ workflow: Workflow,+ content_hash: str,+ file_hash: FileHash,+ workflow_log: WorkflowLog,+ workflow_file_exec: WorkflowFileExecution,+ ) -> FileExecutionResult | None:+ """Check for existing file processing history and return early result if found."""+ if destination.is_api:+ return None++ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=content_hash+ )+ if not file_history:+ return None++ workflow_log.log_info(f"Skipping duplicate '{file_hash.file_name}'")+ return FileExecutionResult(+ file=file_hash.file_name,+ file_execution_id=str(workflow_file_exec.id),+ error=None,+ result=file_history.result,+ metadata=file_history.metadata,+ )++ @classmethod+ def _execute_workflow_steps(+ cls,+ file_data: FileData,+ workflow_execution: WorkflowExecution,+ workflow_file_execution: WorkflowFileExecution,+ file_hash: FileHash,+ current_file_idx: int,+ total_files: int,+ ) -> ToolExecutionResult:+ """Execute main workflow processing steps with proper error handling."""+ tool_instances: list[ToolInstance] = (+ ToolInstanceHelper.get_tool_instances_by_workflow(+ file_data.workflow_id, ToolInstanceKey.STEP+ )+ )+ execution_service = cls._build_workflow_execution_service(+ organization_id=file_data.organization_id,+ workflow=workflow_execution.workflow,+ tool_instances=tool_instances,+ pipeline_id=file_data.pipeline_id,+ single_step=file_data.single_step,+ scheduled=file_data.scheduled,+ execution_mode=file_data.execution_mode,+ workflow_execution=workflow_execution,+ use_file_history=file_data.use_file_history,+ file_execution_id=str(workflow_file_execution.id),+ )++ try:+ execution_service.initiate_tool_execution(+ current_file_idx=current_file_idx,+ total_files=total_files,+ file_name=file_hash.file_name,+ single_step=file_data.single_step,+ )++ if not file_hash.is_executed:+ execution_service.execute_input_file(+ file_execution_id=str(workflow_file_execution.id),+ file_name=file_hash.file_name,+ single_step=file_data.single_step,+ workflow_file_execution=workflow_file_execution,+ )+ # Log execution costs+ execution_service.log_total_cost_per_file(+ run_id=str(workflow_file_execution.id), file_name=file_hash.file_name+ )+ tool_execution_result = ToolExecutionResult(error=None, result=None)+ return tool_execution_result+ except StopExecution:+ workflow_file_execution.update_status(status=ExecutionStatus.STOPPED)+ raise+ except WorkflowExecutionError as error:+ tool_execution_result = ToolExecutionResult(+ error=str(error),+ result=None,+ )+ logger.error(f"Workflow execution failed: {str(error)}", exc_info=True)+ return tool_execution_result+ except Exception as error:+ tool_execution_result = ToolExecutionResult(+ error=f"Unexpected error: {error}",+ result=None,+ )+ logger.error(f"Workflow execution failed: {str(error)}", exc_info=True)+ return tool_execution_result++ @classmethod+ def _finalize_execution(+ cls,+ workflow_execution: WorkflowExecution,+ workflow_file_execution: WorkflowFileExecution,+ file_hash: FileHash,+ workflow_log: WorkflowLog,+ destination: DestinationConnector,+ execution_result: ToolExecutionResult,+ ) -> FileExecutionResult:+ """Handle final processing steps and result generation."""+ result = cls._process_final_output(+ destination,+ workflow_execution.workflow,+ file_hash,+ str(workflow_file_execution.id),+ execution_result.error,+ )++ cls._complete_execution(+ workflow_file_execution=workflow_file_execution,+ workflow_log=workflow_log,+ error=execution_result.error,+ )++ return cls._build_final_result(+ workflow_execution=workflow_execution,+ workflow_file_execution=workflow_file_execution,+ file_hash=file_hash,+ error=execution_result.error,+ result=result,+ )++ @classmethod+ def _process_final_output(+ cls,+ destination: DestinationConnector,+ workflow: Workflow,+ file_hash: FileHash,+ file_execution_id: str,+ processing_error: str | None,+ ) -> FinalOutputResult:+ """Handle final output processing and metadata collection."""+ output_result = None+ execution_metadata = None++ try:+ if not processing_error:+ # Process final output through destination+ output_result = destination.handle_output(+ file_name=file_hash.file_name,+ file_hash=file_hash,+ workflow=workflow,+ input_file_path=file_hash.file_path,+ file_execution_id=file_execution_id,+ )++ # Collect metadata from file history if available+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash.file_hash+ )+ if file_history and destination.is_api:+ execution_metadata = destination.get_metadata(file_history)++ except Exception as e:+ error_msg = f"Final output processing failed: {str(e)}"+ logger.error(error_msg, exc_info=True)+ return FinalOutputResult(output=None, metadata=None, error=error_msg)++ return FinalOutputResult(+ output=output_result, metadata=execution_metadata, error=None+ )++ @classmethod+ def _complete_execution(+ cls,+ workflow_file_execution: WorkflowFileExecution,+ workflow_log: WorkflowLog,+ error: str | None,+ ) -> None:+ """Final cleanup and status updates for the execution."""+ try:+ # Update execution status+ final_status = ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED+ workflow_file_execution.update_status(+ status=final_status, execution_error=error+ )++ # Publish final log status+ log_state = LogState.ERROR if error else LogState.SUCCESS+ workflow_log.publish_update_log(+ state=log_state,+ message=f"Execution {'failed' if error else 'completed'} for {workflow_file_execution.file_name}",+ component=LogComponent.DESTINATION,+ )++ except Exception as e:+ logger.error(f"Completion cleanup failed: {str(e)}", exc_info=True)++ @classmethod+ def _build_final_result(+ cls,+ workflow_execution: WorkflowExecution,+ workflow_file_execution: WorkflowFileExecution,+ file_hash: FileHash,+ error: str | None,+ result: FinalOutputResult,+ ) -> FileExecutionResult:+ """Construct and cache the final execution result."""+ final_result = FileExecutionResult(+ file=file_hash.file_name,+ file_execution_id=str(workflow_file_execution.id),+ error=error,+ result=result.output,+ metadata=result.metadata,+ )++ # Update cache with final result+ ResultCacheUtils.update_api_results(+ workflow_id=workflow_execution.workflow.id,+ execution_id=str(workflow_execution.id),+ api_result=final_result,+ )++ return final_resultdiff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex 19d04d9d6..31c8988fe 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -6,6 +6,7 @@ from api_v2.models import APIDeploymentfrom django.core.exceptions import ObjectDoesNotExistfrom django.db import modelsfrom django.db.models import QuerySet, Sum+from django.utils import timezonefrom pipeline_v2.models import Pipelinefrom tags.models import Tagfrom usage_v2.constants import UsageKeys@@ -13,6 +14,8 @@ from usage_v2.models import Usagefrom utils.common_utils import CommonUtilsfrom utils.models.base_model import BaseModel+from workflow_manager.execution.dto import ExecutionCache+from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtilsfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models import Workflow@@ -220,6 +223,10 @@ class WorkflowExecution(BaseModel):return total_cost+ @property+ def is_completed(self) -> bool:+ return ExecutionStatus.is_completed(self.status)+def __str__(self) -> str:return (f"Workflow execution: {self.id} ("@@ -229,3 +236,67 @@ class WorkflowExecution(BaseModel):f"files: {self.total_files}, "f"error message: {self.error_message})")++ def update_execution(+ self,+ status: ExecutionStatus | None = None,+ error: str | None = None,+ increment_attempt: bool = False,+ ) -> None:+ """Update the execution status and related fields.++ Args:+ status (Optional[ExecutionStatus], optional): New execution status. Defaults to None.+ error (Optional[str], optional): Error message if any. Defaults to None.+ increment_attempt (bool, optional): Whether to increment attempt counter. Defaults to False.+ """+ if status is not None:+ self.status = status.value+ if (+ status+ in [+ ExecutionStatus.COMPLETED,+ ExecutionStatus.ERROR,+ ExecutionStatus.STOPPED,+ ]+ and not self.execution_time+ ):+ self.execution_time = round(+ (timezone.now() - self.created_at).total_seconds(), 3+ )+ if error:+ self.error_message = error[:EXECUTION_ERROR_LENGTH]+ if increment_attempt:+ self.attempts += 1++ self.save()++ def update_execution_err(self, err_msg: str = "") -> None:+ """Update execution status to ERROR with an error message.++ Args:+ err_msg (str, optional): Error message to store. Defaults to "".+ """+ self.update_execution(status=ExecutionStatus.ERROR, error=err_msg)++ def _handle_execution_cache(self):+ if not ExecutionCacheUtils.is_execution_exists(+ workflow_id=self.workflow.id, execution_id=self.id+ ):+ execution_cache = ExecutionCache(+ workflow_id=self.workflow.id,+ execution_id=self.id,+ total_files=self.total_files,+ status=self.status,+ )+ ExecutionCacheUtils.create_execution(+ execution=execution_cache,+ )+ else:+ ExecutionCacheUtils.update_status(+ workflow_id=self.workflow.id, execution_id=self.id, status=self.status+ )++ def save(self, *args, **kwargs):+ super().save(*args, **kwargs)+ self._handle_execution_cache()diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex f1a57fac8..97a49682f 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -1,19 +1,21 @@import jsonimport logging+import mathimport os+import timeimport tracebackfrom typing import Anyfrom account_v2.constants import Commonfrom api_v2.models import APIDeploymentfrom api_v2.utils import APIDeploymentUtils-from celery import current_task, shared_task+from celery import chord, current_taskfrom celery import exceptions as celery_exceptionsfrom celery.result import AsyncResult+from django.conf import settingsfrom django.db import IntegrityErrorfrom pipeline_v2.models import Pipelinefrom pipeline_v2.pipeline_processor import PipelineProcessor-from plugins.workflow_manager.workflow_v2.utils import WorkflowUtilfrom rest_framework import serializersfrom tool_instance_v2.constants import ToolInstanceKeyfrom tool_instance_v2.models import ToolInstance@@ -24,19 +26,26 @@ from utils.local_context import StateStorefrom utils.user_context import UserContextfrom backend.celery_service import app as celery_app-from unstract.workflow_execution.enums import LogComponent, LogLevel, LogState-from unstract.workflow_execution.exceptions import StopExecution+from unstract.workflow_execution.enums import LogStagefrom workflow_manager.endpoint_v2.destination import DestinationConnector-from workflow_manager.endpoint_v2.dto import FileHash+from workflow_manager.endpoint_v2.dto import (+ FileHash,+)+from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtilsfrom workflow_manager.endpoint_v2.source import SourceConnector-from workflow_manager.file_execution.models import WorkflowFileExecution+from workflow_manager.execution.dto import ExecutionCache+from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils+from workflow_manager.utils.workflow_log import WorkflowLogfrom workflow_manager.workflow_v2.constants import (- CeleryConfigurations,WorkflowErrors,WorkflowExecutionKey,WorkflowMessages,)-from workflow_manager.workflow_v2.dto import AsyncResultData, ExecutionResponse+from workflow_manager.workflow_v2.dto import (+ ExecutionResponse,+ FileBatchData,+ FileData,+)from workflow_manager.workflow_v2.enums import ExecutionStatus, SchemaEntity, SchemaTypefrom workflow_manager.workflow_v2.exceptions import (InvalidRequest,@@ -46,6 +55,7 @@ from workflow_manager.workflow_v2.exceptions import (WorkflowExecutionNotExist,)from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper+from workflow_manager.workflow_v2.file_execution_tasks import FileExecutionTasksfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.execution import WorkflowExecutionfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -58,8 +68,6 @@ class WorkflowHelper:def get_workflow_by_id(id: str) -> Workflow:try:workflow: Workflow = Workflow.objects.get(pk=id)- if not workflow or workflow is None:- raise WorkflowDoesNotExistError()return workflowexcept Workflow.DoesNotExist:logger.error(f"Error getting workflow: {id}")@@ -84,221 +92,123 @@ class WorkflowHelper:workflow.save()return workflow- @staticmethod- def build_workflow_execution_service(- organization_id: str | None,- workflow: Workflow,- tool_instances: list[ToolInstance],- pipeline_id: str | None,- single_step: bool,- scheduled: bool,- execution_mode: tuple[str, str],- workflow_execution: WorkflowExecution,- use_file_history: bool = True, # Will be False for API deployment alone- ) -> WorkflowExecutionServiceHelper:- workflow_execution_service = WorkflowExecutionServiceHelper(- organization_id=organization_id,- workflow=workflow,- tool_instances=tool_instances,- pipeline_id=pipeline_id,- single_step=single_step,- scheduled=scheduled,- mode=execution_mode,- workflow_execution=workflow_execution,- use_file_history=use_file_history,- )- workflow_execution_service.build()- return workflow_execution_service-@classmethod- def process_input_files(- cls,- workflow: Workflow,- source: SourceConnector,- destination: DestinationConnector,- execution_service: WorkflowExecutionServiceHelper,- single_step: bool,- input_files: dict[str, FileHash],- ) -> WorkflowExecution:- total_files = len(input_files)- error_message = None- successful_files = 0- failed_files = 0- execution_service.publish_initial_workflow_logs(total_files)- execution_service.update_execution(- ExecutionStatus.EXECUTING, increment_attempt=True- )- q_file_no_list = (- WorkflowUtil.get_q_no_list(workflow, total_files) if total_files > 0 else []- )+ def get_file_batches(+ cls, input_files: dict[str, FileHash]+ ) -> list[list[tuple[str, FileHash]]]:+ """_summary_- for index, (file_name, file_hash) in enumerate(input_files.items()):- # Get workflow execution file- file_number = index + 1- file_hash = WorkflowUtil.add_file_destination_filehash(- file_number,- q_file_no_list,- file_hash,- )- try:- error = cls._process_file(- current_file_idx=file_number,- total_files=total_files,- input_file=file_hash.file_path,- workflow=workflow,- source=source,- destination=destination,- execution_service=execution_service,- single_step=single_step,- file_hash=file_hash,- )- if error:- failed_files += 1- else:- successful_files += 1- except StopExecution as e:- execution_service.update_execution(ExecutionStatus.STOPPED, error=str(e))- break- except Exception as e:- failed_files += 1- error_message = f"Error processing file '{file_name}'. {e}"- logger.error(error_message, stack_info=True, exc_info=True)- execution_service.publish_log(message=error_message, level=LogLevel.ERROR)- # TODO: Store only generic WF errors here (concerning all failed files)- # TODO: Review if we need partial success- final_status = (- ExecutionStatus.ERROR- if total_files > 0 and failed_files >= total_files- else ExecutionStatus.COMPLETED- )- execution_service.update_execution(final_status, error=error_message)+ Args:+ input_files (dict[str, FileHash]): input files- workflow_execution = execution_service.get_execution_instance()+ Returns:+ batches: batches of input files+ """+ json_serializable_files = {+ file_name: file_hash.to_json() for file_name, file_hash in input_files.items()+ }- execution_service.publish_final_workflow_logs(- total_files=total_files,- successful_files=successful_files,- failed_files=failed_files,- )+ # Prepare batches of files for parallel processing+ BATCH_SIZE = settings.MAX_PARALLEL_FILE_BATCHES # Max number of batches+ file_items = list(json_serializable_files.items())++ # Calculate how many items per batch+ num_files = len(file_items)+ num_batches = min(BATCH_SIZE, num_files)+ items_per_batch = math.ceil(num_files / num_batches)- return workflow_execution+ # Split into batches+ batches = []+ for start_index in range(0, len(file_items), items_per_batch):+ end_index = start_index + items_per_batch+ batch = file_items[start_index:end_index]+ batches.append(batch)++ return batches@classmethod- def _process_file(+ def process_input_files(cls,- current_file_idx: int,- total_files: int,- input_file: str,workflow: Workflow,source: SourceConnector,destination: DestinationConnector,- execution_service: WorkflowExecutionServiceHelper,+ workflow_log: WorkflowLog,+ workflow_execution: WorkflowExecution,single_step: bool,- file_hash: FileHash,+ input_files: dict[str, FileHash],+ organization_id: str,+ pipeline_id: str,+ scheduled: bool,+ execution_mode: tuple[str, str],+ use_file_history: bool,) -> str | None:- # Delete previous execution directory to avoid conflicts- destination.delete_execution_directory()-- # Multiple run_ids are linked to an execution_id- # Each run_id corresponds to workflow runs for a single file- # It should e uuid of workflow_file_execution- workflow_file_execution: WorkflowFileExecution = (- WorkflowFileExecution.objects.get_or_create_file_execution(- workflow_execution=execution_service.workflow_execution,- file_hash=file_hash,- connection_type=source.endpoint.connection_type,- )- )- workflow_file_execution.update_status(status=ExecutionStatus.EXECUTING)- file_execution_id = str(workflow_file_execution.id)- file_name = file_hash.file_name-- # This will add the file to the volume- file_content_hash = source.add_file_to_volume(- workflow_file_execution=workflow_file_execution,- tags=execution_service.tags,- file_hash=file_hash,- )+ total_files = len(input_files)+ workflow_log.publish_initial_workflow_logs(total_files=total_files)- # Update file_hash after adding to volume- file_hash.file_hash = file_content_hash- workflow_file_execution.update(file_hash=file_content_hash)+ workflow_execution.update_execution(+ status=ExecutionStatus.EXECUTING, increment_attempt=True+ )- # Ensure no duplicate file processing- if (- FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_content_hash- )- and not destination.is_api- ):- logger.info(f"Skipping '{file_name}', already processed.")- execution_service.publish_log(- f"Skipping '{file_name}', already processed.", LogLevel.INFO+ if not input_files:+ logger.info(f"Execution {workflow_execution.id} no files to process")+ workflow_execution.update_execution(+ status=ExecutionStatus.COMPLETED,)- return None+ return- try:- execution_service.file_execution_id = file_execution_id- execution_service.initiate_tool_execution(- current_file_idx, total_files, file_name, single_step+ batches = cls.get_file_batches(input_files=input_files)+ batch_tasks = []+ mode = (+ execution_mode[1]+ if isinstance(execution_mode, tuple)+ else str(execution_mode)+ )+ result = None+ logger.info(+ f"Execution {workflow_execution.id} processing {total_files} files in {len(batches)} batches"+ )+ for batch in batches:+ # Convert all UUIDs to strings in batch_data+ file_data = FileData(+ workflow_id=str(workflow.id),+ source_config=source.get_config().to_json(),+ destination_config=destination.get_config().to_json(),+ execution_id=str(workflow_execution.id),+ single_step=single_step,+ organization_id=str(organization_id),+ pipeline_id=str(pipeline_id),+ scheduled=scheduled,+ execution_mode=mode,+ use_file_history=use_file_history,)- workflow_file_execution.update_status(status=ExecutionStatus.EXECUTING)- if not file_hash.is_executed:- execution_service.execute_input_file(- file_execution_id=file_execution_id,- file_name=file_name,- single_step=single_step,- workflow_file_execution=workflow_file_execution,+ batch_data = FileBatchData(files=batch, file_data=file_data)++ # Determine the appropriate queue based on execution_mode+ file_processing_queue = FileExecutionTasks.get_queue_name(source)++ # Send each batch to the dedicated file_processing queue+ batch_tasks.append(+ FileExecutionTasks.process_file_batch.s(batch_data.to_dict()).set(+ queue=file_processing_queue)- except StopExecution:- workflow_file_execution.update_status(status=ExecutionStatus.STOPPED)- raise- except WorkflowExecutionError as e:- error_message = f"Error processing file '{file_name}': {e}"- execution_service.publish_log(error_message, level=LogLevel.ERROR)- error = error_message- except Exception as e:- error_message = f"Unexpected error processing file '{file_name}': {str(e)}"- logger.error(error_message, stack_info=True, exc_info=True)- execution_service.publish_log(error_message, level=LogLevel.ERROR)- error = error_message- # Handling error based on destination and continuing for other files- else:- error = None- execution_service.publish_update_log(- LogState.RUNNING,- f"Processing output for {file_name}",- LogComponent.DESTINATION,- )- try:- destination.handle_output(- file_name=file_name,- file_hash=file_hash,- workflow=workflow,- input_file_path=input_file,- error=error,- use_file_history=execution_service.use_file_history,- file_execution_id=file_execution_id,)+ try:+ result = chord(batch_tasks)(+ FileExecutionTasks.process_batch_callback.s(+ execution_id=str(workflow_execution.id)+ ).set(queue=file_processing_queue)+ )+ if not result.id:+ exception = f"Failed to queue execution task {workflow_execution.id}"+ logger.error(exception)+ raise WorkflowExecutionError(exception)+ logger.info(f"Execution {workflow_execution.id} task queued successfully")except Exception as e:- error_message = f"Error during output processing for '{file_name}': {str(e)}"- logger.error(error_message, stack_info=True, exc_info=True)- execution_service.publish_log(error_message, level=LogLevel.ERROR)- error = error_message- finally:- execution_service.log_total_cost_per_file(- run_id=file_execution_id, file_name=file_name- )- execution_service.publish_update_log(- LogState.SUCCESS,- f"{file_name}'s output is processed successfully",- LogComponent.DESTINATION,- )- workflow_file_execution.update_status(- status=ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED,- execution_error=error if error else None,+ workflow_execution.update_execution(+ status=ExecutionStatus.ERROR,+ error=f"Error while processing files: {str(e)}",)- return error+ return result.id@staticmethoddef validate_tool_instances_meta(@@ -328,31 +238,27 @@ class WorkflowHelper:workflow.id, ToolInstanceKey.STEP))-WorkflowHelper.validate_tool_instances_meta(tool_instances=tool_instances)execution_mode = execution_mode or WorkflowExecution.Mode.INSTANT- execution_service = WorkflowHelper.build_workflow_execution_service(+ execution_id = str(workflow_execution.id)+ workflow_log = WorkflowLog(+ execution_id=workflow_execution.id,organization_id=organization_id,- workflow=workflow,- tool_instances=tool_instances,+ log_stage=LogStage.INITIALIZE,pipeline_id=pipeline_id,- single_step=single_step,- scheduled=scheduled,- execution_mode=execution_mode,- workflow_execution=workflow_execution,- use_file_history=use_file_history,)- execution_id = execution_service.execution_idsource = SourceConnector(- organization_id=organization_id,workflow=workflow,execution_id=execution_id,- execution_service=execution_service,+ workflow_log=workflow_log,+ use_file_history=use_file_history,+ organization_id=organization_id,)destination = DestinationConnector(workflow=workflow,execution_id=execution_id,- execution_service=execution_service,+ workflow_log=workflow_log,+ use_file_history=use_file_history,)try:# Validating endpoints@@ -362,40 +268,44 @@ class WorkflowHelper:input_files, total_files = source.list_files_from_source(hash_values_of_files)workflow_execution.total_files = total_filesworkflow_execution.save()- workflow_execution = WorkflowHelper.process_input_files(- workflow,- source,- destination,- execution_service,+ WorkflowHelper.process_input_files(+ workflow=workflow,+ source=source,+ destination=destination,+ workflow_log=workflow_log,+ workflow_execution=workflow_execution,single_step=single_step,input_files=input_files,+ organization_id=organization_id,+ pipeline_id=pipeline_id,+ scheduled=scheduled,+ use_file_history=use_file_history,+ execution_mode=execution_mode,)WorkflowHelper._update_pipeline_status(pipeline_id=pipeline_id, workflow_execution=workflow_execution)+ api_results = []return ExecutionResponse(str(workflow.id),str(workflow_execution.id),workflow_execution.status,- log_id=str(execution_service.execution_log_id),+ log_id=workflow_log.messaging_channel,error=workflow_execution.error_message,mode=workflow_execution.execution_mode,- result=destination.api_results,+ result=api_results,)except Exception as e:logger.error(f"Error executing workflow {workflow}: {e}")logger.error(f"Error {traceback.format_exc()}")- workflow_execution = WorkflowExecutionServiceHelper.update_execution_err(- execution_id, str(e)+ workflow_execution.update_execution(+ status=ExecutionStatus.ERROR,+ error=str(e),)WorkflowHelper._update_pipeline_status(pipeline_id=pipeline_id, workflow_execution=workflow_execution)raise- finally:- # TODO: Handle error gracefully during delete- # Mark status as an ERROR correctly- destination.delete_execution_and_api_storage_dir()@staticmethoddef _update_pipeline_status(@@ -449,26 +359,26 @@ class WorkflowHelper:Returns:ExecutionResponse: _description_"""- execution = WorkflowExecution.objects.get(id=execution_id)+ execution: WorkflowExecution = WorkflowExecution.objects.get(id=execution_id)if not execution.task_id:raise TaskDoesNotExistError(f"No task ID found for execution: {execution_id}")- result = celery_app.AsyncResult(str(execution.task_id))- task = AsyncResultData(async_result=result)-+ task_result = None+ result_acknowledged = execution.result_acknowledged# Prepare the initial response with the task's current status and result.+ if execution.is_completed:+ task_result = ResultCacheUtils.get_api_results(+ workflow_id=str(execution.workflow.id), execution_id=execution_id+ )+ cls._set_result_acknowledge(execution)+result_response = ExecutionResponse(- execution.workflow_id,- execution_id,- execution.status,- result=task.result,- result_acknowledged=execution.result_acknowledged,+ workflow_id=str(execution.workflow.id),+ execution_id=execution_id,+ execution_status=execution.status,+ result=task_result,+ result_acknowledged=result_acknowledged,)-- # If task is complete, handle acknowledgment and forgetting the- if result.ready():- result.forget() # Remove the result from the result backend.- cls._set_result_acknowledge(execution)return result_response@staticmethod@@ -487,6 +397,37 @@ class WorkflowHelper:logger.info(f"ExecutionID [{execution.id}] - Task {execution.task_id} acknowledged")+ # Delete api results from cache+ ResultCacheUtils.delete_api_results(+ workflow_id=str(execution.workflow.id), execution_id=str(execution.id)+ )+ ExecutionCacheUtils.delete_execution(+ workflow_id=str(execution.workflow.id), execution_id=str(execution.id)+ )++ @classmethod+ def _get_execution_status(+ cls, workflow_id: str, execution_id: str+ ) -> ExecutionStatus:+ execution_cache = ExecutionCacheUtils.get_execution(+ workflow_id=workflow_id, execution_id=execution_id+ )+ if not execution_cache:+ execution_model: WorkflowExecution = WorkflowExecution.objects.get(+ id=execution_id+ )+ execution_cache = ExecutionCache(+ workflow_id=workflow_id,+ execution_id=execution_id,+ status=ExecutionStatus(execution_model.status),+ total_files=execution_model.total_files,+ completed_files=execution_model.completed_files,+ failed_files=execution_model.failed_files,+ )+ ExecutionCacheUtils.create_execution(+ execution=execution_cache,+ )+ return execution_cache.status@classmethoddef execute_workflow_async(@@ -540,29 +481,37 @@ class WorkflowHelper:f"[{org_schema}] Job '{async_execution}' has been enqueued for "f"execution_id '{execution_id}', '{len(hash_values_of_files)}' files")- workflow_execution = WorkflowExecution.objects.get(id=execution_id)+ workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(+ id=execution_id+ )workflow_execution.task_id = async_execution.idworkflow_execution.save()+ execution_status = workflow_execution.statusif timeout > -1:- async_execution.wait(- timeout=timeout,- interval=CeleryConfigurations.INTERVAL,+ while not ExecutionStatus.is_completed(execution_status) and timeout > 0:+ time.sleep(2)+ timeout -= 2++ execution_status = cls._get_execution_status(+ workflow_id=workflow_id, execution_id=execution_id+ )+ if ExecutionStatus.is_completed(execution_status):+ # Fetch the object agian to get the latest status.+ workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(+ id=execution_id+ )+ task_result = ResultCacheUtils.get_api_results(+ workflow_id=workflow_id, execution_id=execution_id)- task = AsyncResultData(async_result=async_execution)- celery_result = task.to_dict()- task_result = celery_result.get("result")- # Fetch the object agian to get the latest status.- workflow_execution = WorkflowExecution.objects.get(id=execution_id)+ cls._set_result_acknowledge(workflow_execution)+ else:+ task_result = Noneexecution_response = ExecutionResponse(workflow_id,execution_id,- workflow_execution.status,+ execution_status,result=task_result,)- # If task is complete, handle acknowledgment and forgetting the- if async_execution.ready():- async_execution.forget() # Remove the result from the result backend.- cls._set_result_acknowledge(workflow_execution)return execution_responseexcept celery_exceptions.TimeoutError:return ExecutionResponse(@@ -587,7 +536,7 @@ class WorkflowHelper:)@staticmethod- @shared_task(+ @celery_app.task(name="async_execute_bin",autoretry_for=(Exception,),max_retries=0,@@ -630,7 +579,7 @@ class WorkflowHelper:task_id = current_task.request.id# Set organization in state store for executionStateStore.set(Account.ORGANIZATION_ID, schema_name)- return WorkflowHelper.execute_workflow(+ WorkflowHelper.execute_workflow(organization_id=schema_name,task_id=task_id,workflow_id=workflow_id,@@ -775,6 +724,9 @@ class WorkflowHelper:use_file_history=use_file_history,log_events_id=log_events_id,)+ ExecutionCacheUtils.delete_execution(+ workflow_id=str(workflow.id), execution_id=str(execution_id)+ )updated_execution = WorkflowExecution.objects.get(id=execution_id)execution_response = ExecutionResponse(workflow.id,diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex 6fd958231..e23afc9fa 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -93,6 +93,47 @@ services:- ./workflow_data:/data- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config+ # Celery worker for handling file processing tasks+ worker-file-processing:+ image: unstract/backend:${VERSION}+ container_name: unstract-worker-file-processing+ restart: unless-stopped+ entrypoint: .venv/bin/celery+ command: "-A backend.workers.file_processing worker --loglevel=info -Q file_processing --autoscale=${WORKER_FILE_PROCESSING_AUTOSCALE}"+ env_file:+ - ../backend/.env+ depends_on:+ - redis+ environment:+ - ENVIRONMENT=development+ - APPLICATION_NAME=unstract-worker-file-processing+ labels:+ - traefik.enable=false+ volumes:+ - ./workflow_data:/data+ - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config+++ # Celery worker for handling API file processing tasks+ worker-api-file-processing:+ image: unstract/backend:${VERSION}+ container_name: unstract-worker-api-file-processing+ restart: unless-stopped+ entrypoint: .venv/bin/celery+ command: "-A backend.workers.file_processing worker --loglevel=info -Q api_file_processing --autoscale=${WORKER_API_FILE_PROCESSING_AUTOSCALE}"+ env_file:+ - ../backend/.env+ depends_on:+ - redis+ environment:+ - ENVIRONMENT=development+ - APPLICATION_NAME=unstract-worker-api-file-processing+ labels:+ - traefik.enable=false+ volumes:+ - ./workflow_data:/data+ - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config+# Celery Flowercelery-flower:image: unstract/backend:${VERSION}diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yamlindex f4ac352b2..f6e92877c 100644--- a/docker/sample.compose.override.yaml+++ b/docker/sample.compose.override.yaml@@ -4,6 +4,8 @@services:worker:command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue,celery_api_deployments --autoscale=${WORKER_AUTOSCALE}"+ worker-file-processing:+ command: "-A backend.workers.file_processing worker --loglevel=info -Q file_processing,api_file_processing --autoscale=${WORKER_FILE_PROCESSING_AUTOSCALE}"worker-logging:profiles:diff --git a/docker/sample.env b/docker/sample.envindex 9af5d82fd..269bca95f 100644--- a/docker/sample.env+++ b/docker/sample.env@@ -10,3 +10,5 @@ TOOL_REGISTRY_CONFIG_SRC_PATH="${PWD}/../unstract/tool-registry/tool_registry_coWORKER_API_DEPLOYMENTS_AUTOSCALE=4,1WORKER_LOGGING_AUTOSCALE=4,1WORKER_AUTOSCALE=4,1+WORKER_FILE_PROCESSING_AUTOSCALE=4,1+WORKER_API_FILE_PROCESSING_AUTOSCALE=4,1diff --git a/runner/src/unstract/runner/clients/interface.py b/runner/src/unstract/runner/clients/interface.pyindex b1cd9b42c..09b6fbbee 100644--- a/runner/src/unstract/runner/clients/interface.py+++ b/runner/src/unstract/runner/clients/interface.py@@ -1,7 +1,9 @@+from __future__ import annotations+import loggingfrom abc import ABC, abstractmethodfrom collections.abc import Iterator-from typing import Any, Optional+from typing import Anyclass ContainerInterface(ABC):@@ -24,7 +26,7 @@ class ContainerInterface(ABC):pass@abstractmethod- def cleanup(self, client: Optional["ContainerClientInterface"] = None) -> None:+ def cleanup(self, client: ContainerClientInterface | None = None) -> None:"""Stops and removes the running container."""passdiff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.pyindex 870b2b069..0d253096a 100644--- a/runner/src/unstract/runner/runner.py+++ b/runner/src/unstract/runner/runner.py@@ -335,6 +335,7 @@ class UnstractRunner:organization_id,workflow_id,execution_id,+ file_execution_id,)envs[Env.WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS] = os.getenv(Env.WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS, "{}"diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py b/unstract/workflow-execution/src/unstract/workflow_execution/enums.pyindex b41818dea..dc141bb61 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py@@ -33,6 +33,10 @@ class LogStage(Enum):BUILD = "BUILD"TOOL = "TOOL"RUN = "RUN"+ SOURCE = "SOURCE"+ DESTINATION = "DESTINATION"+ INITIALIZE = "INITIALIZE"+ PROCESSING = "PROCESSING"class LogState(Enum):diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py b/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.pyindex 7ece4b5bc..4706cda34 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py@@ -54,3 +54,24 @@ class ToolMetadataNotFound(Exception):self, message: str = "Internal server error: Tool metadata not found.") -> None:self.message = message+++class ExecutionDirectoryNotFound(Exception):+ def __init__(+ self, message: str = "Internal server error: Execution directory not set."+ ) -> None:+ self.message = message+++class FileExecutionNotFound(Exception):+ def __init__(+ self, message: str = "Internal server error: File execution not found."+ ) -> None:+ self.message = message+++class FileMetadataJsonNotFound(Exception):+ def __init__(+ self, message: str = "Internal server error: File metadata JSON not found."+ ) -> None:+ self.message = messagediff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex 30a5301e7..9a8ec919e 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -12,25 +12,36 @@ from unstract.workflow_execution.constants import (ToolRuntimeVariable,WorkflowFileType,)-from unstract.workflow_execution.exceptions import ToolMetadataNotFound+from unstract.workflow_execution.exceptions import (+ ExecutionDirectoryNotFound,+ FileExecutionNotFound,+ FileMetadataJsonNotFound,+ ToolMetadataNotFound,+)from unstract.workflow_execution.tools_utils import ToolsUtilslogger = logging.getLogger(__name__)class ExecutionFileHandler:- def __init__(self, workflow_id: str, execution_id: str, organization_id: str) -> None:+ def __init__(+ self,+ workflow_id: str,+ execution_id: str,+ organization_id: str,+ file_execution_id: str | None = None,+ ) -> None:self.organization_id = organization_idself.workflow_id = workflow_idself.execution_id = execution_id+ self.file_execution_id = file_execution_idself.execution_dir = self.get_execution_dir(workflow_id, execution_id, organization_id)- self.source_file = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)- self.infile = os.path.join(self.execution_dir, WorkflowFileType.INFILE)- self.metadata_file = os.path.join(- self.execution_dir, WorkflowFileType.METADATA_JSON- )+ self.file_execution_dir = self._get_file_execution_dir()+ self.source_file = self._get_source_file_path()+ self.infile = self._get_infile_path()+ self.metadata_file = self._get_metadata_file_path()def get_workflow_metadata(self) -> dict[str, Any]:"""Get metadata for the workflow.@@ -38,6 +49,8 @@ class ExecutionFileHandler:Returns:dict[str, Any]: Workflow metadata."""+ if not self.metadata_file:+ raise FileMetadataJsonNotFound()file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)file_storage = file_system.get_file_storage()metadata_content = file_storage.read(path=self.metadata_file, mode="r")@@ -106,7 +119,11 @@ class ExecutionFileHandler:Raises:None"""- metadata_path = os.path.join(self.execution_dir, WorkflowFileType.METADATA_JSON)+ if not self.file_execution_dir:+ raise FileExecutionNotFound()+ metadata_path = self.metadata_file+ if not metadata_path:+ raise FileMetadataJsonNotFound()filename = os.path.basename(input_file_path)content = {MetaDataKey.SOURCE_NAME: filename,@@ -125,6 +142,18 @@ class ExecutionFileHandler:f"metadata for {input_file_path} is " "added in to execution directory")+ def _get_file_execution_dir(self) -> str | None:+ """Get the directory path for a specific file execution.++ Returns:+ str: (Optional) The directory path for the file execution.+ """+ if not self.execution_dir:+ raise ExecutionDirectoryNotFound()+ if not self.file_execution_id:+ return None+ return os.path.join(self.execution_dir, self.file_execution_id)+@classmethoddef get_execution_dir(cls, workflow_id: str, execution_id: str, organization_id: str@@ -134,7 +163,7 @@ class ExecutionFileHandler:Parameters:- workflow_id (str): Identifier for the workflow.- execution_id (str): Identifier for the execution.- - organization_id (Optional[str]):+ - organization_id (str | None):Identifier for the organization (default: None).Returns:@@ -158,7 +187,7 @@ class ExecutionFileHandler:Parameters:- workflow_id (str): Identifier for the workflow.- execution_id (str): Identifier for the execution.- - organization_id (Optional[str]):+ - organization_id (str | None):Identifier for the organization (default: None).Returns:@@ -169,3 +198,31 @@ class ExecutionFileHandler:Path(path_prefix) / organization_id / str(workflow_id) / str(execution_id))return str(execution_dir)++ def _get_source_file_path(self) -> str | None:+ """Returns:+ str: (Optional) The path to the source file.+ """+ if not self.file_execution_dir:+ return None+ else:+ return os.path.join(self.file_execution_dir, WorkflowFileType.SOURCE)++ def _get_infile_path(self) -> str | None:+ """Returns:+ str: (Optional) The path to the infile.+ """+ if not self.file_execution_dir:+ return None+ return os.path.join(self.file_execution_dir, WorkflowFileType.INFILE)++ def _get_metadata_file_path(self) -> str | None:+ """Get the path to the metadata file.++ Args:+ None+ Returns: str: (Optional) The path to the metadata file.+ """+ if not self.file_execution_dir:+ return None+ return os.path.join(self.file_execution_dir, WorkflowFileType.METADATA_JSON)diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 7c117b02f..acd7a64fe 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -44,6 +44,7 @@ class WorkflowExecutionService:tool_instances: list[ToolInstance],platform_service_api_key: str,ignore_processed_entities: bool = False,+ file_execution_id: str | None = None,) -> None:self.organization_id = organization_idself.workflow_id = workflow_id@@ -60,7 +61,7 @@ class WorkflowExecutionService:self.ignore_processed_entities = ignore_processed_entitiesself.override_single_step = Falseself.execution_id: str = ""- self.file_execution_id: str | None = None+ self.file_execution_id: str | None = file_execution_idself.messaging_channel: str | None = Noneself.input_files: list[str] = []self.log_stage: LogStage = LogStage.COMPILE@@ -89,7 +90,10 @@ class WorkflowExecutionService:try:self.execution_id = str(execution_id)self.file_handler = ExecutionFileHandler(- self.workflow_id, self.execution_id, self.organization_id+ self.workflow_id,+ self.execution_id,+ self.organization_id,+ file_execution_id=self.file_execution_id,)logger.info(f"Execution {execution_id}: compilation completed")@@ -136,9 +140,7 @@ class WorkflowExecutionService:logger.info(f"Execution {self.execution_id}: Build completed")- def execute_workflow(- self, file_execution_id: str, execution_type: ExecutionType- ) -> None:+ def execute_workflow(self, execution_type: ExecutionType) -> None:"""Executes the complete workflow by running each tools one by one.Returns the result from final tool in a dictionary.@@ -156,10 +158,10 @@ class WorkflowExecutionService:Eg:- {"result": "RESULT_FROM_FINAL_TOOL"}"""self.log_stage = LogStage.RUN+self._initialize_execution()total_steps = len(self.tool_sandboxes)self.total_steps = total_steps- self.file_execution_id = file_execution_id# Currently each tool is run serially for files and workflows contain 1 tool# only. While supporting more tools in a workflow, correct the tool container# name to avoid conflicts.@@ -315,6 +317,8 @@ class WorkflowExecutionService:"""if not self.execution_id:raise BadRequestException("Execution Id not found")+ if not self.file_execution_id:+ raise BadRequestException("File Execution Id not found")def _finalize_execution(self, execution_type: ExecutionType) -> None:"""Finalize the execution process.@@ -339,11 +343,14 @@ class WorkflowExecutionService:self._handling_step_execution()def validate_execution_result(self, step: int) -> bool:- workflow_metadata = self.file_handler.get_workflow_metadata()- metadata_list = self.file_handler.get_list_of_tool_metadata(workflow_metadata)- if len(metadata_list) == step:- return True- return False+ try:+ workflow_metadata = self.file_handler.get_workflow_metadata()+ metadata_list = self.file_handler.get_list_of_tool_metadata(workflow_metadata)+ if len(metadata_list) == step:+ return True+ return False+ except Exception:+ return Falsedef publish_log(self,