Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

parallel execution (#1295)

ali005f7d23f231
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 8f3b04693..c59f246e1 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -128,6 +128,13 @@ LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
LOGS_EXPIRATION_TIME_IN_SECOND = int(
get_required_setting("LOGS_EXPIRATION_TIME_IN_SECOND", "86400")
)
+EXECUTION_RESULT_TTL_SECONDS = int(
+ os.environ.get("EXECUTION_RESULT_TTL_SECONDS", 10800)
+) # 3 hours
+EXECUTION_CACHE_TTL_SECONDS = int(
+ os.environ.get("EXECUTION_CACHE_TTL_SECONDS", 10800)
+) # 3 hours
+MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))
INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))
diff --git a/backend/backend/workers/__init__.py b/backend/backend/workers/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/backend/workers/constants.py b/backend/backend/workers/constants.py
new file mode 100644
index 000000000..3d1eb83df
--- /dev/null
+++ b/backend/backend/workers/constants.py
@@ -0,0 +1,4 @@
+class CeleryWorkerNames:
+ """Names of the celery workers."""
+
+ FILE_PROCESSING = "file_processing"
diff --git a/backend/backend/workers/file_processing/__init__.py b/backend/backend/workers/file_processing/__init__.py
new file mode 100644
index 000000000..7f1a02c21
--- /dev/null
+++ b/backend/backend/workers/file_processing/__init__.py
@@ -0,0 +1,4 @@
+from .constants import QueueNames
+from .file_processing import app
+
+__all__ = ["app", "QueueNames"]
diff --git a/backend/backend/workers/file_processing/celery_config.py b/backend/backend/workers/file_processing/celery_config.py
new file mode 100644
index 000000000..629c314fa
--- /dev/null
+++ b/backend/backend/workers/file_processing/celery_config.py
@@ -0,0 +1,20 @@
+from kombu import Queue
+
+from backend.celery_config import CeleryConfig as BaseCeleryConfig
+from backend.workers.file_processing.constants import QueueNames
+
+
+class CeleryConfig(BaseCeleryConfig):
+ """Specifies celery configuration for file processing
+
+ Refer https://docs.celeryq.dev/en/stable/userguide/configuration.html
+ """
+
+ task_queues = [
+ Queue(QueueNames.FILE_PROCESSING, routing_key=QueueNames.FILE_PROCESSING),
+ Queue(QueueNames.API_FILE_PROCESSING, routing_key=QueueNames.API_FILE_PROCESSING),
+ ]
+
+ task_default_queue = QueueNames.FILE_PROCESSING
+
+ imports = ["workflow_manager.workflow_v2.file_execution_tasks"]
diff --git a/backend/backend/workers/file_processing/constants.py b/backend/backend/workers/file_processing/constants.py
new file mode 100644
index 000000000..a0a9fa68a
--- /dev/null
+++ b/backend/backend/workers/file_processing/constants.py
@@ -0,0 +1,3 @@
+class QueueNames:
+ API_FILE_PROCESSING = "api_file_processing"
+ FILE_PROCESSING = "file_processing"
diff --git a/backend/backend/workers/file_processing/file_processing.py b/backend/backend/workers/file_processing/file_processing.py
new file mode 100644
index 000000000..b6c1f975e
--- /dev/null
+++ b/backend/backend/workers/file_processing/file_processing.py
@@ -0,0 +1,31 @@
+"""This module contains the Celery configuration for the backend project."""
+
+import logging
+import logging.config
+import os
+from pprint import pformat
+
+from celery import Celery
+
+from backend.settings.base import LOGGING
+from backend.workers.constants import CeleryWorkerNames
+from backend.workers.file_processing.celery_config import CeleryConfig
+
+logger = logging.getLogger(__name__)
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault(
+ "DJANGO_SETTINGS_MODULE",
+ os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+)
+
+# Configure logging for celery worker using same config as Django
+logging.config.dictConfig(LOGGING)
+
+# Create a Celery instance. Default time zone is UTC.
+app = Celery(CeleryWorkerNames.FILE_PROCESSING)
+
+# Load task modules from all registered Django app configs.
+app.config_from_object(CeleryConfig)
+
+logger.debug(f"Celery Configuration:\n {pformat(app.conf.table(with_defaults=True))}")
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 5971e97f7..a3fe83dd2 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -119,6 +119,14 @@ help = "Runs the Unstract log management worker"
cmd = "celery -A backend worker --loglevel=info -Q celery_api_deployments --autoscale 4,1"
help = "Runs the Unstract API deployment worker"
+[tool.poe.tasks.worker-file-processing]
+cmd = "celery -A backend.workers.file_processing worker --loglevel=info -Q file_processing --autoscale 4,1"
+help = "Runs the Unstract file processing worker"
+
+[tool.poe.tasks.worker-api-file-processing]
+cmd = "celery -A backend.workers.file_processing worker --loglevel=info -Q api_file_processing --autoscale 4,1"
+help = "Runs the Unstract file processing api worker"
+
[tool.poe.tasks.flower]
cmd = "celery -A backend flower --port=5555"
help = "Runs the Unstract Celery Monitoring Tool"
diff --git a/backend/sample.env b/backend/sample.env
index e6dd7250c..e7463eb2f 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -168,3 +168,12 @@ TOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'
# Highlight data to be available in api deployment
ENABLE_HIGHLIGHT_API_DEPLOYMENT=False
+
+# Execution result and cache expire time
+# For API results cached per workflow execution
+EXECUTION_RESULT_TTL_SECONDS=86400 # 24 hours
+# For execution metadata cached per workflow execution
+EXECUTION_CACHE_TTL_SECONDS=86400 # 24 hours
+
+# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution
+MAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time
diff --git a/backend/utils/cache_service.py b/backend/utils/cache_service.py
index 80f67ee80..8633de58c 100644
--- a/backend/utils/cache_service.py
+++ b/backend/utils/cache_service.py
@@ -1,3 +1,4 @@
+import json
from typing import Any
from django.conf import settings
@@ -77,6 +78,95 @@ class CacheService:
def lrange(key, start_index, end_index) -> list[Any]:
return redis_cache.lrange(key, start_index, end_index)
+ @staticmethod
+ def hset(
+ key: str,
+ field: str = None,
+ value: Any = None,
+ mapping: dict[str, Any] = None,
+ expire_time: int = int(settings.CACHE_TTL_SEC),
+ ) -> None:
+ """Set Redis hash fields. Supports both single field-value and full mapping.
+
+ Args:
+ key (str): The key of the Redis hash.
+ field (str, optional): The field to set. Defaults to None.
+ value (Any, optional): The value to set. Defaults to None.
+ mapping (dict[str, Any], optional): The mapping of fields to values.
+ Defaults to None.
+ expire_time (int, optional): The expiration time for hash in seconds.
+ Defaults to int(settings.CACHE_TTL_SEC).
+ """
+ if mapping is not None:
+ redis_cache.hset(key, mapping=mapping)
+ elif field is not None and value is not None:
+ redis_cache.hset(key, field, value)
+ else:
+ raise ValueError("Provide either 'mapping' or both 'field' and 'value'")
+
+ redis_cache.expire(key, expire_time)
+
+ @staticmethod
+ def hget(key: str, field: str) -> Any:
+ """Get a value from a Redis hash."""
+ return redis_cache.hget(key, field)
+
+ @staticmethod
+ def hgetall(key: str) -> Any:
+ """Get all values from a Redis hash."""
+ return redis_cache.hgetall(key)
+
+ @staticmethod
+ def hincrby(key: str, field: str, increment: int) -> None:
+ """Increment a value in a Redis hash."""
+ redis_cache.hincrby(key, field, increment)
+
+ @staticmethod
+ def exists(key: str) -> bool:
+ """Check if a key exists in Redis."""
+ return redis_cache.exists(key)
+
+ @staticmethod
+ def rpush_with_expire(
+ key: str, value: Any, expire: int = int(settings.CACHE_TTL_SEC)
+ ) -> None:
+ """Push a value to a Redis list and reset the list's TTL in a single atomic operation.
+
+ Args:
+ key (str): The key of the Redis list.
+ value (Any): The value to push to the list.
+ expire (int, optional): The expiration time for list in seconds.
+ Defaults to int(settings.CACHE_TTL_SEC).
+
+ Returns:
+ None
+ """
+ pipe = redis_cache.pipeline()
+ pipe.rpush(key, json.dumps(value))
+ pipe.expire(key, expire)
+ pipe.execute()
+
+ @staticmethod
+ def lrange_json(key: str, start_index: int = 0, end_index: int = -1) -> list[Any]:
+ """Get all items from list and parse JSON.
+
+ Args:
+ key (str): The key of the Redis list.
+ start_index (int, optional): The starting index for the range.
+ Defaults to 0.
+ end_index (int, optional): The ending index for the range.
+ Defaults to -1.
+
+ Returns:
+ list[Any]: A list of parsed JSON values from the list.
+ """
+ results = redis_cache.lrange(key, start_index, end_index)
+ if not results:
+ return []
+ return [
+ json.loads(r.decode("utf-8") if isinstance(r, bytes) else r) for r in results
+ ]
+
@staticmethod
def remove_all_session_keys(
user_id: str | None = None,
diff --git a/backend/utils/date/enums.py b/backend/utils/date/enums.py
index 9a1b3a874..d7d6a4858 100644
--- a/backend/utils/date/enums.py
+++ b/backend/utils/date/enums.py
@@ -1,6 +1,7 @@
+from __future__ import annotations
+
from datetime import datetime, timedelta
from enum import Enum
-from typing import Optional
from django.utils import timezone
@@ -44,7 +45,7 @@ class DateRangePresets(Enum):
return self.get_start_date(), self.get_end_date()
@classmethod
- def from_value(cls, value: str) -> Optional["DateRangePresets"]:
+ def from_value(cls, value: str) -> DateRangePresets | None:
try:
return next(preset for preset in cls if preset.key == value)
except StopIteration as e:
diff --git a/backend/utils/dto.py b/backend/utils/dto.py
index a2be3237e..af10f475f 100644
--- a/backend/utils/dto.py
+++ b/backend/utils/dto.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
+
import json
import logging
from datetime import UTC, datetime
-from typing import Any, Optional
+from typing import Any
from django.utils import timezone as dj_timezone
@@ -43,7 +45,7 @@ class LogDataDTO:
self.data: dict[str, Any] = data
@classmethod
- def from_json(cls, json_data: str) -> Optional["LogDataDTO"]:
+ def from_json(cls, json_data: str) -> LogDataDTO | None:
try:
json_data = json.loads(json_data)
execution_id = json_data.get(LogFieldName.EXECUTION_ID)
diff --git a/backend/uv.lock b/backend/uv.lock
index ba36766f0..3bbe12f6e 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -1699,15 +1699,16 @@ wheels = [
[[package]]
name = "kombu"
-version = "5.3.7"
+version = "5.5.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "amqp" },
+ { name = "tzdata" },
{ name = "vine" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/99/3a/2fb09708fef243e35c286414f2bf78543dc311ae7d3de5d343bd8437e38d/kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf", size = 439344 }
+sdist = { url = "https://files.pythonhosted.org/packages/60/0a/128b65651ed8120460fc5af754241ad595eac74993115ec0de4f2d7bc459/kombu-5.5.3.tar.gz", hash = "sha256:021a0e11fcfcd9b0260ef1fb64088c0e92beb976eb59c1dfca7ddd4ad4562ea2", size = 461784 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/b4/9a/1951f2261271d6994f0df5a55b3e9cdad42ed1fc3020a0dc7f6de80a4566/kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9", size = 200190 },
+ { url = "https://files.pythonhosted.org/packages/5d/35/1407fb0b2f5b07b50cbaf97fce09ad87d3bfefbf64f7171a8651cd8d2f68/kombu-5.5.3-py3-none-any.whl", hash = "sha256:5b0dbceb4edee50aa464f59469d34b97864be09111338cfb224a10b6a163909b", size = 209921 },
]
[[package]]
@@ -4020,7 +4021,7 @@ dependencies = [
[package.metadata]
requires-dist = [
{ name = "flask", marker = "extra == 'flask'", specifier = "~=3.1.0" },
- { name = "kombu", specifier = "==5.3.7" },
+ { name = "kombu", specifier = "~=5.5.3" },
{ name = "redis", specifier = "~=5.2.1" },
{ name = "requests", specifier = "==2.31.0" },
]
diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.py
index 199a5d31b..66fec358f 100644
--- a/backend/workflow_manager/endpoint_v2/base_connector.py
+++ b/backend/workflow_manager/endpoint_v2/base_connector.py
@@ -13,13 +13,19 @@ from unstract.workflow_execution.execution_file_handler import ExecutionFileHand
class BaseConnector(ExecutionFileHandler):
"""Base class for connectors providing common methods and utilities."""
- def __init__(self, workflow_id: str, execution_id: str, organization_id: str) -> None:
+ def __init__(
+ self,
+ workflow_id: str,
+ execution_id: str,
+ organization_id: str,
+ file_execution_id: str | None = None,
+ ) -> None:
"""Initialize the BaseConnector class.
This class serves as a base for connectors and provides common
utilities.
"""
- super().__init__(workflow_id, execution_id, organization_id)
+ super().__init__(workflow_id, execution_id, organization_id, file_execution_id)
def get_fsspec(
self, settings: dict[str, Any], connector_id: str
@@ -95,3 +101,20 @@ class BaseConnector(ExecutionFileHandler):
workflow_id, execution_id, organization_id
)
return api_storage_dir
+
+ @classmethod
+ def get_execution_dir_path(cls, workflow_id: str, execution_id: str) -> str:
+ """Get the directory path for storing execution-related files.
+
+ Parameters:
+ - workflow_id (str): Identifier for the workflow.
+ - execution_id (str): Identifier for the execution.
+
+ Returns:
+ str: The directory path for the execution.
+ """
+ organization_id = UserContext.get_organization_identifier()
+ execution_dir: str = cls.get_execution_dir(
+ workflow_id, execution_id, organization_id
+ )
+ return execution_dir
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 7f704b55f..a1795945f 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -14,10 +14,9 @@ from workflow_manager.endpoint_v2.constants import (
ApiDeploymentResultStatus,
DestinationKey,
QueueResultStatus,
- WorkflowFileType,
)
from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
-from workflow_manager.endpoint_v2.dto import FileHash
+from workflow_manager.endpoint_v2.dto import DestinationConfig, FileHash
from workflow_manager.endpoint_v2.exceptions import (
DestinationConnectorNotConfigured,
InvalidDestinationConnectionType,
@@ -27,8 +26,8 @@ from workflow_manager.endpoint_v2.exceptions import (
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtils
+from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.enums import ExecutionStatus
-from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -59,7 +58,9 @@ class DestinationConnector(BaseConnector):
self,
workflow: Workflow,
execution_id: str,
- execution_service: WorkflowExecutionServiceHelper | None = None,
+ workflow_log: WorkflowLog,
+ use_file_history: bool,
+ file_execution_id: str | None = None,
) -> None:
"""Initialize a DestinationConnector object.
@@ -67,16 +68,17 @@ class DestinationConnector(BaseConnector):
workflow (Workflow): _description_
"""
organization_id = UserContext.get_organization_identifier()
- super().__init__(workflow.id, execution_id, organization_id)
+ super().__init__(workflow.id, execution_id, organization_id, file_execution_id)
self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)
self.execution_id = execution_id
- self.api_results: list[dict[str, Any]] = []
self.queue_results: list[dict[str, Any]] = []
- self.execution_service = execution_service
self.is_api: bool = (
self.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
)
+ self.workflow_log = workflow_log
+ self.use_file_history = use_file_history
+ self.workflow = workflow
def _get_endpoint_for_workflow(
self,
@@ -162,7 +164,7 @@ class DestinationConnector(BaseConnector):
file_execution_id: str,
) -> bool:
"""Handles HITL processing, returning True if data was pushed to the queue."""
- execution_result = self.get_result()
+ execution_result = self.get_tool_execution_result()
if WorkflowUtil.validate_db_rule(
execution_result, workflow, file_hash.file_destination
):
@@ -182,7 +184,7 @@ class DestinationConnector(BaseConnector):
input_file_path: str,
file_execution_id: str,
) -> None:
- result = self.get_result()
+ result = self.get_tool_execution_result()
meta_data = self.get_metadata()
self._push_to_queue(
file_name=file_name,
@@ -199,21 +201,15 @@ class DestinationConnector(BaseConnector):
file_hash: FileHash,
workflow: Workflow,
input_file_path: str,
- error: str | None = None,
- use_file_history: bool = True,
file_execution_id: str = None,
- ) -> None:
+ ) -> str | None:
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
- result: str | None = None
+ tool_execution_result: str | None = None
metadata: str | None = None
- if error:
- if connection_type == WorkflowEndpoint.ConnectionType.API:
- self._handle_api_result(file_name=file_name, error=error, result=result)
- return
file_history = None
- if use_file_history:
+ if self.use_file_history:
file_history = FileHistoryHelper.get_file_history(
workflow=workflow, cache_key=file_hash.file_hash
)
@@ -229,11 +225,7 @@ class DestinationConnector(BaseConnector):
):
self.insert_into_db(input_file_path=input_file_path)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
- result = self.get_result(file_history)
- exec_metadata = self.get_metadata(file_history)
- self._handle_api_result(
- file_name=file_name, error=error, result=result, metadata=exec_metadata
- )
+ tool_execution_result = self.get_tool_execution_result(file_history)
elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
self._push_data_to_queue(
file_name,
@@ -241,20 +233,20 @@ class DestinationConnector(BaseConnector):
input_file_path,
file_execution_id,
)
- if self.execution_service:
- self.execution_service.publish_log(
- message=f"File '{file_name}' processed successfully"
- )
+ self.workflow_log.publish_log(
+ message=f"File '{file_name}' processed successfully"
+ )
- if use_file_history and not file_history:
+ if self.use_file_history and not file_history:
FileHistoryHelper.create_file_history(
file_hash=file_hash,
workflow=workflow,
status=ExecutionStatus.COMPLETED,
- result=result,
+ result=tool_execution_result,
metadata=metadata,
file_name=file_name,
)
+ return tool_execution_result
def copy_output_to_output_directory(self) -> None:
"""Copy output to the destination directory."""
@@ -273,7 +265,9 @@ class DestinationConnector(BaseConnector):
input_dir=output_directory, root_path=root_path
)
logger.debug(f"destination output directory {output_directory}")
- destination_volume_path = os.path.join(self.execution_dir, ToolExecKey.OUTPUT_DIR)
+ destination_volume_path = os.path.join(
+ self.file_execution_dir, ToolExecKey.OUTPUT_DIR
+ )
try:
destination_fs.create_dir_if_not_exists(input_dir=output_directory)
@@ -328,7 +322,7 @@ class DestinationConnector(BaseConnector):
execution_id_name = str(
destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
)
- data = self.get_result()
+ data = self.get_tool_execution_result()
# If data is None, don't execute CREATE or INSERT query
if not data:
return
@@ -418,7 +412,10 @@ class DestinationConnector(BaseConnector):
Returns:
None
"""
- api_result: dict[str, Any] = {"file": file_name}
+ api_result: dict[str, Any] = {
+ "file": file_name,
+ "file_execution_id": self.file_execution_id,
+ }
if error:
api_result.update(
{"status": ApiDeploymentResultStatus.FAILED, "error": error}
@@ -436,7 +433,7 @@ class DestinationConnector(BaseConnector):
api_result.update(
{"status": ApiDeploymentResultStatus.SUCCESS, "result": ""}
)
- self.api_results.append(api_result)
+ self.update_api_results(api_result)
def parse_string(self, original_string: str) -> Any:
"""Parse the given string, attempting to evaluate it as a Python
@@ -472,15 +469,17 @@ class DestinationConnector(BaseConnector):
# assume it's a plain string
return original_string
- def get_result(self, file_history: FileHistory | None = None) -> Any | None:
+ def get_tool_execution_result(
+ self, file_history: FileHistory | None = None
+ ) -> Any | None:
"""Get result data from the output file.
Returns:
Union[dict[str, Any], str]: Result data.
"""
- return self.get_result_with_file_storage(file_history=file_history)
+ return self.get_tool_execution_result_from_metadata(file_history=file_history)
- def get_result_with_file_storage(
+ def get_tool_execution_result_from_metadata(
self, file_history: FileHistory | None = None
) -> Any | None:
"""Get result data from the output file.
@@ -490,7 +489,7 @@ class DestinationConnector(BaseConnector):
"""
if file_history and file_history.result:
return self.parse_string(file_history.result)
- output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
+ output_file = self.infile
metadata: dict[str, Any] = self.get_workflow_metadata()
output_type = self.get_output_type(metadata)
result: dict[str, Any] | str = ""
@@ -536,25 +535,28 @@ class DestinationConnector(BaseConnector):
return metadata
- def delete_execution_directory(self) -> None:
- """Delete the execution directory.
+ def delete_file_execution_directory(self) -> None:
+ """Delete the file execution directory.
Returns:
None
"""
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
- if file_storage.exists(self.execution_dir):
- file_storage.rm(self.execution_dir, recursive=True)
+ if self.file_execution_dir and file_storage.exists(self.file_execution_dir):
+ file_storage.rm(self.file_execution_dir, recursive=True)
- def delete_execution_and_api_storage_dir(self) -> None:
+ @classmethod
+ def delete_execution_and_api_storage_dir(
+ cls, workflow_id: str, execution_id: str
+ ) -> None:
"""Delete the execution and api storage directories.
Returns:
None
"""
- self.delete_execution_directory()
- self.delete_api_storage_dir(self.workflow_id, self.execution_id)
+ cls.delete_execution_directory(workflow_id, execution_id)
+ cls.delete_api_storage_dir(workflow_id, execution_id)
@classmethod
def delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:
@@ -570,6 +572,23 @@ class DestinationConnector(BaseConnector):
file_storage = file_system.get_file_storage()
if file_storage.exists(api_storage_dir):
file_storage.rm(api_storage_dir, recursive=True)
+ logger.info(f"API storage directory deleted: {api_storage_dir}")
+
+ @classmethod
+ def delete_execution_directory(cls, workflow_id: str, execution_id: str) -> None:
+ """Delete the execution directory.
+
+ Returns:
+ None
+ """
+ execution_dir = cls.get_execution_dir_path(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ if file_storage.exists(execution_dir):
+ file_storage.rm(execution_dir, recursive=True)
+ logger.info(f"Execution directory deleted: {execution_dir}")
@classmethod
def create_endpoint_for_workflow(
@@ -621,6 +640,44 @@ class DestinationConnector(BaseConnector):
)
return cls.get_json_schema(file_path=schema_path)
+ def get_config(self) -> DestinationConfig:
+ """Get serializable configuration for the destination connector.
+
+ Returns:
+ DestinationConfig: Configuration containing all necessary data to reconstruct the connector
+ """
+ return DestinationConfig(
+ workflow_id=self.workflow.id,
+ execution_id=self.execution_id,
+ use_file_history=self.use_file_history,
+ )
+
+ @classmethod
+ def from_config(
+ cls, workflow_log: WorkflowLog, config: DestinationConfig
+ ) -> "DestinationConnector":
+ """Create a DestinationConnector instance from configuration.
+
+ Args:
+ config (DestinationConfig): Configuration containing all necessary data to reconstruct the connector
+
+ Returns:
+ DestinationConnector: New instance
+ """
+ # Reconstruct workflow
+ workflow = Workflow.objects.get(id=config.workflow_id)
+
+ # Create destination connector instance
+ destination = cls(
+ workflow=workflow,
+ execution_id=config.execution_id,
+ workflow_log=workflow_log,
+ use_file_history=config.use_file_history,
+ file_execution_id=config.file_execution_id,
+ )
+
+ return destination
+
def _push_to_queue(
self,
file_name: str,
diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.py
index 3f21f8ef5..ead8e5c02 100644
--- a/backend/workflow_manager/endpoint_v2/dto.py
+++ b/backend/workflow_manager/endpoint_v2/dto.py
@@ -2,6 +2,10 @@ import json
from dataclasses import dataclass
from typing import Any
+from workflow_manager.endpoint_v2.constants import (
+ ApiDeploymentResultStatus,
+)
+
@dataclass
class FileHash:
@@ -42,3 +46,99 @@ class FileHash:
# Otherwise, assume it's a JSON string
data = json.loads(json_str_or_dict)
return FileHash(**data)
+
+
+@dataclass
+class SourceConfig:
+ workflow_id: str
+ execution_id: str
+ organization_id: str
+ use_file_history: bool
+ file_execution_id: str | None = None
+
+ def to_json(self) -> dict[str, Any]:
+ """Serialize the SourceConfig instance to a JSON string."""
+ file_execution_id = (
+ str(self.file_execution_id) if self.file_execution_id else None
+ )
+ return {
+ "workflow_id": str(self.workflow_id),
+ "execution_id": str(self.execution_id),
+ "organization_id": str(self.organization_id),
+ "use_file_history": self.use_file_history,
+ "file_execution_id": file_execution_id,
+ }
+
+ @staticmethod
+ def from_json(json_str_or_dict: Any) -> "SourceConfig":
+ """Deserialize a JSON string or dictionary to a SourceConfig instance."""
+ if isinstance(json_str_or_dict, dict):
+ data = json_str_or_dict
+ else:
+ data = json.loads(json_str_or_dict)
+ return SourceConfig(**data)
+
+
+@dataclass
+class DestinationConfig:
+ workflow_id: str
+ execution_id: str
+ use_file_history: bool
+ file_execution_id: str | None = None
+
+ def to_json(self) -> dict[str, Any]:
+ """Serialize the DestinationConfig instance to a JSON string."""
+ file_execution_id = (
+ str(self.file_execution_id) if self.file_execution_id else None
+ )
+ return {
+ "workflow_id": str(self.workflow_id),
+ "execution_id": str(self.execution_id),
+ "use_file_history": self.use_file_history,
+ "file_execution_id": file_execution_id,
+ }
+
+ @staticmethod
+ def from_json(json_str_or_dict: Any) -> "DestinationConfig":
+ """Deserialize a JSON string or dictionary to a DestinationConfig instance."""
+ if isinstance(json_str_or_dict, dict):
+ data = json_str_or_dict
+ else:
+ data = json.loads(json_str_or_dict)
+ return DestinationConfig(**data)
+
+
+@dataclass
+class FileExecutionResult:
+ file: str
+ file_execution_id: str
+ status: str | None = None
+ result: str | None = None
+ error: str | None = None
+ metadata: dict[str, Any] | None = None
+
+ def __post_init__(self) -> None:
+ if self.error:
+ self.status = ApiDeploymentResultStatus.FAILED
+ else:
+ self.status = ApiDeploymentResultStatus.SUCCESS
+
+ def to_json(self) -> dict[str, Any]:
+ """Serialize the FileExecutionResult instance to a JSON string."""
+ return {
+ "file": self.file,
+ "file_execution_id": self.file_execution_id,
+ "status": self.status,
+ "result": self.result,
+ "error": self.error,
+ "metadata": self.metadata,
+ }
+
+ @staticmethod
+ def from_json(json_str_or_dict: Any) -> "FileExecutionResult":
+ """Deserialize a JSON string or dictionary to a FileExecutionResult instance."""
+ if isinstance(json_str_or_dict, dict):
+ data = json_str_or_dict
+ else:
+ data = json.loads(json_str_or_dict)
+ return FileExecutionResult(**data)
diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.py
index 4eef455b9..e994627c8 100644
--- a/backend/workflow_manager/endpoint_v2/exceptions.py
+++ b/backend/workflow_manager/endpoint_v2/exceptions.py
@@ -107,3 +107,8 @@ class UnstractQueueException(APIException):
def __init__(self, detail: str = default_detail) -> None:
status_code = 500
super().__init__(detail=detail, code=status_code)
+
+
+class SourceFileOrInfilePathNotFound(APIException):
+ status_code = 500
+ default_detail = "Unable to obtain file (Source or Infile) for execution."
diff --git a/backend/workflow_manager/endpoint_v2/result_cache_utils.py b/backend/workflow_manager/endpoint_v2/result_cache_utils.py
new file mode 100644
index 000000000..6fe94a48c
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/result_cache_utils.py
@@ -0,0 +1,40 @@
+from typing import Any
+
+from django.conf import settings
+from utils.cache_service import CacheService
+from workflow_manager.endpoint_v2.dto import FileExecutionResult
+
+
+class ResultCacheUtils:
+ expire_time = int(settings.EXECUTION_RESULT_TTL_SECONDS)
+
+ @staticmethod
+ def _get_api_results_cache_key(workflow_id: str, execution_id: str) -> str:
+ """Get Redis cache key for api_results."""
+ return f"api_results:{workflow_id}:{execution_id}"
+
+ @classmethod
+ def get_api_results(cls, workflow_id: str, execution_id: str) -> list[dict[str, Any]]:
+ """Get api_results from Redis cache."""
+ cache_key = cls._get_api_results_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ return CacheService.lrange_json(cache_key)
+
+ @classmethod
+ def update_api_results(
+ cls, workflow_id: str, execution_id: str, api_result: FileExecutionResult
+ ) -> None:
+ """Update api_results in Redis cache."""
+ cache_key = cls._get_api_results_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ CacheService.rpush_with_expire(cache_key, api_result.to_json(), cls.expire_time)
+
+ @classmethod
+ def delete_api_results(cls, workflow_id: str, execution_id: str) -> None:
+ """Delete api_results from Redis cache."""
+ cache_key = cls._get_api_results_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ CacheService.delete_a_key(cache_key)
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 02835e564..46ae39553 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -19,19 +19,19 @@ from workflow_manager.endpoint_v2.constants import (
FileType,
SourceConstant,
SourceKey,
- WorkflowFileType,
)
-from workflow_manager.endpoint_v2.dto import FileHash
+from workflow_manager.endpoint_v2.dto import FileHash, SourceConfig
from workflow_manager.endpoint_v2.exceptions import (
InvalidInputDirectory,
InvalidSourceConnectionType,
MissingSourceConnectionType,
OrganizationIdNotFound,
SourceConnectorNotConfigured,
+ SourceFileOrInfilePathNotFound,
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.file_execution.models import WorkflowFileExecution
-from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -65,8 +65,10 @@ class SourceConnector(BaseConnector):
self,
workflow: Workflow,
execution_id: str,
+ workflow_log: WorkflowLog,
+ use_file_history: bool,
organization_id: str | None = None,
- execution_service: WorkflowExecutionServiceHelper | None = None,
+ file_execution_id: str | None = None,
) -> None:
"""Create a SourceConnector.
@@ -84,13 +86,14 @@ class SourceConnector(BaseConnector):
organization_id = organization_id or UserContext.get_organization_identifier()
if not organization_id:
raise OrganizationIdNotFound()
- super().__init__(workflow.id, execution_id, organization_id)
+ super().__init__(workflow.id, execution_id, organization_id, file_execution_id)
self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
self.workflow = workflow
self.execution_id = execution_id
self.organization_id = organization_id
self.hash_value_of_file_content: str | None = None
- self.execution_service = execution_service
+ self.workflow_log = workflow_log
+ self.use_file_history = use_file_history
def _get_endpoint_for_workflow(
self,
@@ -226,36 +229,30 @@ class SourceConnector(BaseConnector):
msg (str): Message to log
"""
logger.info(msg)
- if self.execution_service:
- self.execution_service.publish_log(msg)
+ self.workflow_log.publish_log(message=msg)
def publish_input_output_list_file_logs(
self, folders: list[str], matched_files: dict[str, FileHash], count: int
) -> None:
- if not self.execution_service:
- return None
-
folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)
input_log = f"## Folders to process:\n\n{folders_list}\n\n"
- self.execution_service.publish_update_log(
+ self.workflow_log.publish_update_log(
state=LogState.INPUT_UPDATE, message=input_log
)
output_log = self._matched_files_component_log(matched_files, count)
- self.execution_service.publish_update_log(
+ self.workflow_log.publish_update_log(
state=LogState.OUTPUT_UPDATE, message=output_log
)
def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:
- if not self.execution_service:
- return None
output_log_message = f"## Input text:\n\n```text\n{input_text}\n```\n\n"
input_log_message = (
f"## Input file:\n\n```text\n{os.path.basename(input_file_path)}\n```\n\n"
)
- self.execution_service.publish_update_log(
+ self.workflow_log.publish_update_log(
state=LogState.INPUT_UPDATE, message=input_log_message
)
- self.execution_service.publish_update_log(
+ self.workflow_log.publish_update_log(
state=LogState.OUTPUT_UPDATE, message=output_log_message
)
@@ -357,7 +354,7 @@ class SourceConnector(BaseConnector):
if matches_blocked:
message = f"Skipping '{file_name}' as it has an unsupported file format"
logger.debug(message)
- self.execution_service.publish_log(message)
+ self.workflow_log.publish_log(message)
return False
return True
@@ -369,12 +366,8 @@ class SourceConnector(BaseConnector):
file_history = self._get_file_history(workflow, source_fs, file_path)
# In case of ETL pipelines, its necessary to skip files which have
# already been processed
- if (
- self.execution_service.use_file_history
- and file_history
- and file_history.is_completed()
- ):
- self.execution_service.publish_log(
+ if self.use_file_history and file_history and file_history.is_completed():
+ self.workflow_log.publish_log(
f"Skipping file {file_path} as it has already been processed. "
"Clear the file markers to process it again."
)
@@ -505,6 +498,8 @@ class SourceConnector(BaseConnector):
input_file_path = file_hash.file_path
source_file_path = self.source_file
infile_path = self.infile
+ if not source_file_path or not infile_path:
+ raise SourceFileOrInfilePathNotFound()
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
source_fs = self.get_fsspec(
@@ -544,8 +539,10 @@ class SourceConnector(BaseConnector):
def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:
"""Add input file to execution directory from api storage."""
- infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
- source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
+ source_file_path = self.source_file
+ infile_path = self.infile
+ if not source_file_path or not infile_path:
+ raise SourceFileOrInfilePathNotFound()
api_file_system = FileSystem(FileStorageType.API_EXECUTION)
api_file_storage = api_file_system.get_file_storage()
@@ -555,7 +552,7 @@ class SourceConnector(BaseConnector):
source_storage=api_file_storage,
destination_storage=workflow_file_storage,
source_path=input_file_path,
- destination_paths=[infile_path, source_path],
+ destination_paths=[infile_path, source_file_path],
)
logger.info(f"File {input_file_path} added to execution directory")
return file_content_hash
@@ -770,3 +767,45 @@ class SourceConnector(BaseConnector):
os.path.dirname(__file__), "static", "src", "file.json"
)
return cls.get_json_schema(file_path=schema_path)
+
+ def get_config(self) -> SourceConfig:
+ """Get serializable configuration for the source connector.
+
+ Returns:
+ SourceConfig: Configuration containing all necessary data to reconstruct the connector
+ """
+ source_config = SourceConfig(
+ workflow_id=self.workflow.id,
+ execution_id=self.execution_id,
+ organization_id=self.organization_id,
+ use_file_history=self.use_file_history,
+ )
+ return source_config
+
+ @classmethod
+ def from_config(
+ cls, workflow_log: WorkflowLog, config: SourceConfig
+ ) -> "SourceConnector":
+ """Create a SourceConnector instance from configuration.
+
+ Args:
+ workflow_log (WorkflowLog): Workflow log instance
+ config (SourceConfig): Configuration containing all necessary data to reconstruct the connector
+
+ Returns:
+ SourceConnector: New instance
+ """
+ # Reconstruct workflow
+ workflow = Workflow.objects.get(id=config.workflow_id)
+
+ # Create source connector instance
+ source = cls(
+ workflow=workflow,
+ execution_id=config.execution_id,
+ workflow_log=workflow_log,
+ use_file_history=config.use_file_history,
+ organization_id=config.organization_id,
+ file_execution_id=config.file_execution_id,
+ )
+
+ return source
diff --git a/backend/workflow_manager/execution/dto.py b/backend/workflow_manager/execution/dto.py
new file mode 100644
index 000000000..ee3915f66
--- /dev/null
+++ b/backend/workflow_manager/execution/dto.py
@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import Any
+
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+
+
+class ExecutionCacheFields:
+ WORKFLOW_ID = "workflow_id"
+ EXECUTION_ID = "execution_id"
+ STATUS = "status"
+ TOTAL_FILES = "total_files"
+ COMPLETED_FILES = "completed_files"
+ FAILED_FILES = "failed_files"
+
+
+@dataclass
+class ExecutionCache:
+ workflow_id: str
+ execution_id: str
+ status: ExecutionStatus
+ total_files: int = 0
+ completed_files: int = 0
+ failed_files: int = 0
+
+ def __post_init__(self):
+ self.workflow_id = str(self.workflow_id)
+ self.execution_id = str(self.execution_id)
+ self.status = ExecutionStatus(self.status)
+
+ def to_json(self) -> dict[str, Any]:
+ return {
+ ExecutionCacheFields.WORKFLOW_ID: self.workflow_id,
+ ExecutionCacheFields.EXECUTION_ID: self.execution_id,
+ ExecutionCacheFields.STATUS: self.status.value,
+ ExecutionCacheFields.TOTAL_FILES: self.total_files,
+ ExecutionCacheFields.COMPLETED_FILES: self.completed_files,
+ ExecutionCacheFields.FAILED_FILES: self.failed_files,
+ }
diff --git a/backend/workflow_manager/execution/execution_cache_utils.py b/backend/workflow_manager/execution/execution_cache_utils.py
new file mode 100644
index 000000000..a20d1e1a2
--- /dev/null
+++ b/backend/workflow_manager/execution/execution_cache_utils.py
@@ -0,0 +1,95 @@
+from django.conf import settings
+from utils.cache_service import CacheService
+
+from workflow_manager.execution.dto import ExecutionCache, ExecutionCacheFields
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+
+
+class ExecutionCacheUtils:
+ """Utility class for accessing and managing workflow execution status and
+ related information from cache to reduce database load.
+ """
+
+ expire_time = int(settings.EXECUTION_CACHE_TTL_SECONDS)
+
+ @staticmethod
+ def _get_execution_cache_key(workflow_id: str, execution_id: str) -> str:
+ """Get Redis cache key for execution."""
+ return f"execution:{workflow_id}:{execution_id}"
+
+ @classmethod
+ def get_execution(cls, workflow_id: str, execution_id: str) -> ExecutionCache | None:
+ """Get or create execution."""
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ execution = CacheService.hgetall(cache_key)
+ if execution:
+ # Decode keys and values from bytes to strings
+ decoded = {k.decode(): v.decode() for k, v in execution.items()}
+ return ExecutionCache(**decoded)
+ else:
+ return None
+
+ @classmethod
+ def is_execution_exists(cls, workflow_id: str, execution_id: str) -> bool:
+ """Check if execution exists."""
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ return CacheService.check_a_key_exist(cache_key)
+
+ @classmethod
+ def create_execution(cls, execution: ExecutionCache) -> None:
+ """Create execution."""
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=execution.workflow_id, execution_id=execution.execution_id
+ )
+ CacheService.hset(
+ cache_key, mapping=execution.to_json(), expire_time=cls.expire_time
+ )
+
+ @classmethod
+ def update_status(
+ cls, workflow_id: str, execution_id: str, status: ExecutionStatus
+ ) -> None:
+ """Update execution status."""
+ try:
+ status_enum = ExecutionStatus(status)
+ except ValueError:
+ raise ValueError(
+ f"Invalid status: {status}. Must be a valid ExecutionStatus."
+ )
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ CacheService.hset(
+ cache_key,
+ field=ExecutionCacheFields.STATUS,
+ value=status_enum.value,
+ expire_time=cls.expire_time,
+ )
+
+ @classmethod
+ def increment_completed_files(cls, workflow_id: str, execution_id: str) -> None:
+ """Increment completed files."""
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ CacheService.hincrby(cache_key, ExecutionCacheFields.COMPLETED_FILES, 1)
+
+ @classmethod
+ def increment_failed_files(cls, workflow_id: str, execution_id: str) -> None:
+ """Increment failed files."""
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ CacheService.hincrby(cache_key, ExecutionCacheFields.FAILED_FILES, 1)
+
+ @classmethod
+ def delete_execution(cls, workflow_id: str, execution_id: str) -> None:
+ """Delete execution."""
+ cache_key = cls._get_execution_cache_key(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ CacheService.delete_a_key(cache_key)
diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.py
index d96f1f0f4..ce8434b77 100644
--- a/backend/workflow_manager/file_execution/models.py
+++ b/backend/workflow_manager/file_execution/models.py
@@ -7,7 +7,6 @@ from utils.common_utils import CommonUtils
from utils.models.base_model import BaseModel
from workflow_manager.endpoint_v2.dto import FileHash
-from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.workflow_v2.enums import ExecutionStatus
FILE_NAME_LENGTH = 255
@@ -21,7 +20,7 @@ class WorkflowFileExecutionManager(models.Manager):
self,
workflow_execution: Any,
file_hash: FileHash,
- connection_type: WorkflowEndpoint.ConnectionType,
+ is_api: bool = False,
) -> "WorkflowFileExecution":
"""Retrieves or creates a new input file record for a workflow execution.
@@ -34,7 +33,6 @@ class WorkflowFileExecutionManager(models.Manager):
Returns:
The `WorkflowFileExecution` object.
"""
- is_api = connection_type == WorkflowEndpoint.ConnectionType.API
# Determine file path based on connection type
execution_file_path = file_hash.file_path if not is_api else None
diff --git a/backend/workflow_manager/utils/__init__.py b/backend/workflow_manager/utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.py
new file mode 100644
index 000000000..1769f527e
--- /dev/null
+++ b/backend/workflow_manager/utils/workflow_log.py
@@ -0,0 +1,91 @@
+import logging
+
+from account_v2.constants import Common
+from utils.local_context import StateStore
+
+from unstract.core.pubsub_helper import LogPublisher
+from unstract.workflow_execution.enums import (
+ LogComponent,
+ LogLevel,
+ LogStage,
+ LogState,
+)
+
+
+class WorkflowLog:
+ def __init__(
+ self,
+ execution_id: str,
+ log_stage: LogStage,
+ file_execution_id: str | None = None,
+ organization_id: str | None = None,
+ pipeline_id: str | None = None,
+ ):
+ log_events_id: str | None = StateStore.get(Common.LOG_EVENTS_ID)
+ self.messaging_channel = log_events_id if log_events_id else pipeline_id
+ self.execution_id = str(execution_id)
+ self.file_execution_id = str(file_execution_id) if file_execution_id else None
+ self.organization_id = str(organization_id) if organization_id else None
+ self.log_stage = log_stage
+
+ def publish_log(
+ self,
+ message: str,
+ level: LogLevel = LogLevel.INFO,
+ step: int | None = None,
+ ) -> None:
+ log_details = LogPublisher.log_workflow(
+ stage=self.log_stage.value,
+ message=message,
+ level=level.value,
+ step=step,
+ execution_id=self.execution_id,
+ file_execution_id=self.file_execution_id,
+ organization_id=self.organization_id,
+ )
+ LogPublisher.publish(self.messaging_channel, log_details)
+
+ def log_error(self, logger: logging.Logger, message: str) -> None:
+ self.publish_log(message, level=LogLevel.ERROR)
+ logger.error(message, exc_info=True, stack_info=True)
+
+ def log_info(self, message: str) -> None:
+ self.publish_log(message, level=LogLevel.INFO)
+
+ def publish_update_log(
+ self,
+ state: LogState,
+ message: str,
+ component: str | LogComponent | None = None,
+ ) -> None:
+ """Publishes update logs for monitoring the execution of a workflow.
+
+ Args:
+ state (LogState): The state of the log, such as "RUN" or "COMPILE".
+ message (str): The log message to be published.
+ component (LogComponent, optional): The component associated
+ with the log. Defaults to None.
+
+ Returns:
+ None
+ """
+ if isinstance(component, LogComponent):
+ component = component.value
+
+ log_details = LogPublisher.log_workflow_update(state.value, message, component)
+ LogPublisher.publish(self.messaging_channel, log_details)
+
+ def publish_initial_workflow_logs(self, total_files: int) -> None:
+ """Publishes the initial logs for the workflow.
+
+ Args:
+ total_files (int): The total number of matched files.
+
+ Returns:
+ None
+ """
+ self.publish_log(f"Total matched files: {total_files}")
+ self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)
+ self.publish_update_log(
+ LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW
+ )
diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.py
index 7d2859166..fba0f3460 100644
--- a/backend/workflow_manager/workflow_v2/dto.py
+++ b/backend/workflow_manager/workflow_v2/dto.py
@@ -1,8 +1,13 @@
-from dataclasses import dataclass
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, fields
from typing import Any
from celery.result import AsyncResult
+from workflow_manager.endpoint_v2.dto import DestinationConfig, SourceConfig
+from workflow_manager.file_execution.models import WorkflowFileExecution
+from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.constants import WorkflowKey
@@ -135,3 +140,73 @@ class AsyncResultData:
"status": self.status,
"result": self.result,
}
+
+
+@dataclass
+class FileData:
+ workflow_id: str
+ source_config: dict[str, Any]
+ destination_config: dict[str, Any]
+ execution_id: str
+ single_step: bool
+ organization_id: str
+ pipeline_id: str
+ scheduled: bool
+ execution_mode: str
+ use_file_history: bool
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> FileData:
+ field_names = {f.name for f in fields(cls)}
+ filtered_data = {k: v for k, v in data.items() if k in field_names}
+ return cls(**filtered_data)
+
+ def __str__(self) -> str:
+ return f"FileData({self.__dict__})"
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+@dataclass
+class FileBatchData:
+ files: list
+ file_data: FileData
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> FileBatchData:
+ file_data = FileData.from_dict(data["file_data"])
+ return cls(files=data["files"], file_data=file_data)
+
+ def __str__(self) -> str:
+ return f"FileBatchData(files={self.files}, file_data={self.file_data})"
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+@dataclass
+class ToolExecutionResult:
+ error: str | None
+ result: Any | None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "error": self.error,
+ "result": self.result,
+ }
+
+
+@dataclass
+class FinalOutputResult:
+ output: Any | None
+ metadata: dict[str, Any] | None
+ error: str | None
+
+
+@dataclass
+class ExecutionContext:
+ workflow_log: WorkflowLog
+ workflow_file_execution: WorkflowFileExecution
+ source_config: SourceConfig
+ destination_config: DestinationConfig
diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.py
index 8cb77e9ae..9dbe78321 100644
--- a/backend/workflow_manager/workflow_v2/enums.py
+++ b/backend/workflow_manager/workflow_v2/enums.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
from enum import Enum
from django.db.models import TextChoices
@@ -31,6 +33,17 @@ class ExecutionStatus(TextChoices):
STOPPED = "STOPPED"
ERROR = "ERROR"
+ @classmethod
+ def is_completed(cls, status: str | ExecutionStatus) -> bool:
+ """Check if the execution status is completed."""
+ try:
+ status_enum = cls(status)
+ except ValueError:
+ raise ValueError(
+ f"Invalid status: {status}. Must be a valid ExecutionStatus."
+ )
+ return status_enum in [cls.COMPLETED, cls.STOPPED, cls.ERROR]
+
class SchemaType(Enum):
"""Possible types for workflow module's JSON schema.
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index 469b39497..720108612 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -45,7 +45,9 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
workflow_execution: WorkflowExecution | None = None,
use_file_history: bool = True,
+ file_execution_id: str | None = None,
) -> None:
+ self.file_execution_id = file_execution_id
tool_instances_as_dto = []
for tool_instance in tool_instances:
tool_instances_as_dto.append(
@@ -66,6 +68,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
tool_instances=tool_instances_as_dto,
platform_service_api_key=str(platform_key.key),
ignore_processed_entities=False,
+ file_execution_id=file_execution_id,
)
if not workflow_execution:
# Use pipline_id for pipelines / API deployment
@@ -146,7 +149,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
else WorkflowExecution.Type.COMPLETE
)
execution_log_id = log_events_id if log_events_id else pipeline_id
-
# Create the workflow execution
workflow_execution = WorkflowExecution.objects.create(
id=execution_id if execution_id else uuid.uuid4(),
@@ -253,7 +255,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
start_time = time.time()
try:
self.execute_workflow(
- file_execution_id=file_execution_id,
execution_type=execution_type,
)
end_time = time.time()
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
new file mode 100644
index 000000000..49fc52d9b
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -0,0 +1,638 @@
+import logging
+from typing import Any
+from uuid import UUID
+
+from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil
+from tool_instance_v2.constants import ToolInstanceKey
+from tool_instance_v2.models import ToolInstance
+from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
+from utils.constants import Account
+from utils.local_context import StateStore
+
+from backend.workers.file_processing.constants import QueueNames
+from backend.workers.file_processing.file_processing import app as file_processing_app
+from unstract.workflow_execution.enums import LogComponent, LogStage, LogState
+from unstract.workflow_execution.exceptions import StopExecution
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.dto import (
+ DestinationConfig,
+ FileExecutionResult,
+ FileHash,
+ SourceConfig,
+)
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
+from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
+from workflow_manager.file_execution.models import WorkflowFileExecution
+from workflow_manager.utils.workflow_log import WorkflowLog
+from workflow_manager.workflow_v2.dto import (
+ ExecutionContext,
+ FileBatchData,
+ FileData,
+ FinalOutputResult,
+ ToolExecutionResult,
+)
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.exceptions import (
+ WorkflowDoesNotExistError,
+ WorkflowExecutionError,
+)
+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+logger = logging.getLogger(__name__)
+
+
+class FileExecutionTasks:
+ @staticmethod
+ def get_queue_name(source: SourceConnector) -> str:
+ is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
+ return QueueNames.API_FILE_PROCESSING if is_api else QueueNames.FILE_PROCESSING
+
+ @staticmethod
+ def get_workflow_by_id(id: str) -> Workflow:
+ try:
+ workflow: Workflow = Workflow.objects.get(pk=id)
+ return workflow
+ except Workflow.DoesNotExist:
+ logger.error(f"Error getting workflow: {id}")
+ raise WorkflowDoesNotExistError()
+
+ @staticmethod
+ def _build_workflow_execution_service(
+ organization_id: str | None,
+ workflow: Workflow,
+ tool_instances: list[ToolInstance],
+ pipeline_id: str | None,
+ single_step: bool,
+ scheduled: bool,
+ execution_mode: tuple[str, str],
+ workflow_execution: WorkflowExecution,
+ use_file_history: bool = True, # Will be False for API deployment alone
+ file_execution_id: str | None = None,
+ ) -> WorkflowExecutionServiceHelper:
+ workflow_execution_service = WorkflowExecutionServiceHelper(
+ organization_id=organization_id,
+ workflow=workflow,
+ tool_instances=tool_instances,
+ pipeline_id=pipeline_id,
+ single_step=single_step,
+ scheduled=scheduled,
+ mode=execution_mode,
+ workflow_execution=workflow_execution,
+ use_file_history=use_file_history,
+ file_execution_id=file_execution_id,
+ )
+ workflow_execution_service.build()
+ return workflow_execution_service
+
+ @file_processing_app.task(
+ bind=True,
+ max_retries=0, # Maximum number of retries
+ ignore_result=False, # result is passed to the callback task
+ retry_backoff=True,
+ retry_backoff_max=500, # Max delay between retries (500 seconds)
+ retry_jitter=True, # Add random jitter to prevent thundering herd
+ default_retry_delay=5, # Initial retry delay (5 seconds)
+ )
+ def process_file_batch(self, file_batch_data) -> dict[str, Any]:
+ """Process a batch of files in parallel using Celery.
+
+ Args:
+ file_batch_data (dict): Contains all necessary data to process files in the batch
+ - files: List of (file_name, file_hash) tuples
+ - workflow_id: ID of the workflow
+ - source_config: Source connector configuration
+ - destination_config: Destination connector configuration
+ - execution_id: ID of execution service
+ - single_step: Whether to process in single step mode
+ """
+ file_batch_data = FileBatchData.from_dict(file_batch_data)
+ file_data = file_batch_data.file_data
+ logger.info(f"File processing context {file_data}")
+
+ organization_id = file_data.organization_id
+ StateStore.set(Account.ORGANIZATION_ID, organization_id)
+
+ # Use proper logger instead of print
+ logger.info(
+ f"Starting to process file batch for execution {file_data.execution_id}"
+ )
+ logger.debug(f"Batch data received: {file_batch_data}")
+
+ successful_files = 0
+ failed_files = 0
+ execution_id = file_data.execution_id
+ workflow_id = file_data.workflow_id
+ # Reconstruct necessary objects
+ workflow = FileExecutionTasks.get_workflow_by_id(str(workflow_id))
+ workflow_execution = WorkflowExecution.objects.get(id=UUID(execution_id))
+
+ total_files = len(file_batch_data.files)
+ q_file_no_list = (
+ WorkflowUtil.get_q_no_list(workflow, total_files) if total_files > 0 else []
+ )
+
+ logger.info(
+ f"Processing {total_files} files of execution {execution_id} in a batch"
+ )
+
+ for file_number, (file_name, file_hash_dict) in enumerate(
+ file_batch_data.files, 1
+ ):
+ file_hash = FileHash(
+ file_path=file_hash_dict.get("file_path"),
+ file_name=file_hash_dict.get("file_name"),
+ source_connection_type=file_hash_dict.get("source_connection_type"),
+ file_hash=file_hash_dict.get("file_hash"),
+ file_size=file_hash_dict.get("file_size"),
+ provider_file_uuid=file_hash_dict.get("provider_file_uuid"),
+ mime_type=file_hash_dict.get("mime_type"),
+ fs_metadata=file_hash_dict.get("fs_metadata"),
+ file_destination=file_hash_dict.get("file_destination"),
+ is_executed=file_hash_dict.get("is_executed"),
+ )
+ file_hash = WorkflowUtil.add_file_destination_filehash(
+ file_number,
+ q_file_no_list,
+ file_hash,
+ )
+ file_execution_result = FileExecutionTasks._process_file(
+ current_file_idx=file_number,
+ total_files=total_files,
+ file_data=file_data,
+ file_hash=file_hash,
+ workflow_execution=workflow_execution,
+ )
+ if file_execution_result.error:
+ failed_files += 1
+ ExecutionCacheUtils.increment_failed_files(
+ workflow_id=workflow.id,
+ execution_id=execution_id,
+ )
+ else:
+ successful_files += 1
+ ExecutionCacheUtils.increment_completed_files(
+ workflow_id=workflow.id,
+ execution_id=execution_id,
+ )
+ return {
+ "successful_files": successful_files,
+ "failed_files": failed_files,
+ }
+
+ @file_processing_app.task(
+ bind=True,
+ max_retries=0, # Maximum number of retries
+ retry_backoff=True,
+ retry_backoff_max=500, # Max delay between retries (500 seconds)
+ retry_jitter=True, # Add random jitter to prevent thundering herd
+ default_retry_delay=5, # Initial retry delay (5 seconds)
+ )
+ def process_batch_callback(self, results, **kwargs):
+ """Callback task to handle batch processing results.
+
+ Args:
+ results (list): List of results from each batch
+ Each result is a dictionary containing:
+ - successful_files: Number of successfully processed files
+ - failed_files: Number of failed files
+ - error_messages: List of error messages
+ - execution_id: ID of the execution
+ """
+ execution_id = kwargs.get("execution_id")
+ workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(
+ id=execution_id
+ )
+ workflow = workflow_execution.workflow
+ organization = workflow.organization
+ organization_id = organization.organization_id
+
+ # Set organization ID in StateStore
+ StateStore.set(Account.ORGANIZATION_ID, organization_id)
+
+ if not results:
+ return None
+
+ # Aggregate results
+ total_successful = sum(result["successful_files"] for result in results)
+ total_failed = sum(result["failed_files"] for result in results)
+
+ total_files = total_successful + total_failed
+
+ # Update final status
+ final_status = (
+ ExecutionStatus.COMPLETED if total_successful else ExecutionStatus.ERROR
+ )
+ error_message = None
+ # Update execution service with final results
+ workflow_execution.update_execution(
+ status=final_status,
+ error=error_message,
+ )
+ # clean up execution and api storage directories
+ DestinationConnector.delete_execution_and_api_storage_dir(
+ workflow_id=workflow.id, execution_id=execution_id
+ )
+ return {
+ "execution_id": execution_id,
+ "total_files": total_files,
+ "successful_files": total_successful,
+ "failed_files": total_failed,
+ "error_message": error_message,
+ "status": final_status,
+ }
+
+ @classmethod
+ def _process_file(
+ cls,
+ current_file_idx: int,
+ total_files: int,
+ file_data: FileData,
+ file_hash: FileHash,
+ workflow_execution: WorkflowExecution,
+ ) -> FileExecutionResult:
+ """Process a single file in the workflow.
+
+ Args:
+ current_file_idx (int): Index of the current file
+ total_files (int): Total number of files
+ file_data (FileData): File data
+ file_hash (FileHash): File hash
+ workflow_execution (WorkflowExecution): Workflow execution instance
+
+ Returns:
+ FileExecutionResult: Result of the file execution
+ """
+ try:
+ # Initialization Phase
+ execution_context = cls._initialize_execution_context(
+ file_data, file_hash, workflow_execution
+ )
+ workflow_log = execution_context.workflow_log
+ workflow_file_execution = execution_context.workflow_file_execution
+ source_config = execution_context.source_config
+ destination_config = execution_context.destination_config
+ source = SourceConnector.from_config(workflow_log, source_config)
+ destination = DestinationConnector.from_config(
+ workflow_log, destination_config
+ )
+ destination.delete_file_execution_directory()
+
+ # File Preparation Phase
+ content_hash = cls._prepare_file_for_processing(
+ source,
+ workflow_log,
+ workflow_file_execution,
+ file_hash,
+ workflow_execution,
+ )
+
+ # History Check Phase
+ if early_result := cls._check_processing_history(
+ destination,
+ workflow_execution.workflow,
+ content_hash,
+ file_hash,
+ workflow_log,
+ workflow_file_execution,
+ ):
+ return early_result
+
+ # Core Execution Phase
+ execution_result = cls._execute_workflow_steps(
+ file_data,
+ workflow_execution,
+ workflow_file_execution,
+ file_hash,
+ current_file_idx,
+ total_files,
+ )
+
+ # Finalization Phase
+ return cls._finalize_execution(
+ workflow_execution,
+ workflow_file_execution,
+ file_hash,
+ workflow_log,
+ destination,
+ execution_result,
+ )
+ except Exception as error:
+ error_msg = f"File execution failed: {error}"
+ workflow_log.log_error(logger=logger, message=error_msg)
+ workflow_file_execution.update_status(
+ status=ExecutionStatus.ERROR, execution_error=error_msg[:500]
+ )
+ result = FinalOutputResult(output=None, metadata=None, error=error_msg)
+ return cls._build_final_result(
+ workflow_execution=workflow_execution,
+ workflow_file_execution=workflow_file_execution,
+ file_hash=file_hash,
+ error=error_msg,
+ result=result,
+ )
+
+ @classmethod
+ def _initialize_execution_context(
+ cls,
+ file_data: FileData,
+ file_hash: FileHash,
+ workflow_execution: WorkflowExecution,
+ ) -> ExecutionContext:
+ """Set up all required execution context objects and configurations."""
+ workflow = workflow_execution.workflow
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+
+ is_api = endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
+
+ # Create execution record
+ file_execution: WorkflowFileExecution = (
+ WorkflowFileExecution.objects.get_or_create_file_execution(
+ workflow_execution=workflow_execution,
+ file_hash=file_hash,
+ is_api=is_api,
+ )
+ )
+
+ # Create configurations
+ source_config = SourceConfig.from_json(file_data.source_config)
+ destination_config = DestinationConfig.from_json(file_data.destination_config)
+ source_config.file_execution_id = destination_config.file_execution_id = str(
+ file_execution.id
+ )
+
+ # Initialize logging
+ workflow_log = WorkflowLog(
+ execution_id=str(workflow_execution.id),
+ log_stage=LogStage.PROCESSING,
+ organization_id=file_data.organization_id,
+ pipeline_id=str(workflow_execution.pipeline_id),
+ file_execution_id=str(file_execution.id),
+ )
+
+ return ExecutionContext(
+ workflow_log=workflow_log,
+ workflow_file_execution=file_execution,
+ source_config=source_config,
+ destination_config=destination_config,
+ )
+
+ @classmethod
+ def _prepare_file_for_processing(
+ cls,
+ source: SourceConnector,
+ workflow_log: WorkflowLog,
+ workflow_file_exec: WorkflowFileExecution,
+ file_hash: FileHash,
+ workflow_execution: WorkflowExecution,
+ ) -> str:
+ """Handle file preparation and volume storage."""
+ workflow_file_exec.update_status(ExecutionStatus.EXECUTING)
+
+ try:
+ content_hash = source.add_file_to_volume(
+ workflow_file_execution=workflow_file_exec,
+ tags=workflow_execution.tag_names,
+ file_hash=file_hash,
+ )
+ file_hash.file_hash = content_hash
+ workflow_file_exec.update(file_hash=content_hash)
+ return content_hash
+ except FileNotFoundError as error:
+ error_msg = f"File not Found in execution dir: {error}"
+ workflow_log.log_error(logger=logger, message=error_msg)
+ raise WorkflowExecutionError(error_msg) from error
+ except Exception as error:
+ error_msg = f"File preparation failed: {error}"
+ workflow_log.log_error(logger=logger, message=error_msg)
+ raise WorkflowExecutionError(error_msg) from error
+
+ @classmethod
+ def _check_processing_history(
+ cls,
+ destination: DestinationConnector,
+ workflow: Workflow,
+ content_hash: str,
+ file_hash: FileHash,
+ workflow_log: WorkflowLog,
+ workflow_file_exec: WorkflowFileExecution,
+ ) -> FileExecutionResult | None:
+ """Check for existing file processing history and return early result if found."""
+ if destination.is_api:
+ return None
+
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=content_hash
+ )
+ if not file_history:
+ return None
+
+ workflow_log.log_info(f"Skipping duplicate '{file_hash.file_name}'")
+ return FileExecutionResult(
+ file=file_hash.file_name,
+ file_execution_id=str(workflow_file_exec.id),
+ error=None,
+ result=file_history.result,
+ metadata=file_history.metadata,
+ )
+
+ @classmethod
+ def _execute_workflow_steps(
+ cls,
+ file_data: FileData,
+ workflow_execution: WorkflowExecution,
+ workflow_file_execution: WorkflowFileExecution,
+ file_hash: FileHash,
+ current_file_idx: int,
+ total_files: int,
+ ) -> ToolExecutionResult:
+ """Execute main workflow processing steps with proper error handling."""
+ tool_instances: list[ToolInstance] = (
+ ToolInstanceHelper.get_tool_instances_by_workflow(
+ file_data.workflow_id, ToolInstanceKey.STEP
+ )
+ )
+ execution_service = cls._build_workflow_execution_service(
+ organization_id=file_data.organization_id,
+ workflow=workflow_execution.workflow,
+ tool_instances=tool_instances,
+ pipeline_id=file_data.pipeline_id,
+ single_step=file_data.single_step,
+ scheduled=file_data.scheduled,
+ execution_mode=file_data.execution_mode,
+ workflow_execution=workflow_execution,
+ use_file_history=file_data.use_file_history,
+ file_execution_id=str(workflow_file_execution.id),
+ )
+
+ try:
+ execution_service.initiate_tool_execution(
+ current_file_idx=current_file_idx,
+ total_files=total_files,
+ file_name=file_hash.file_name,
+ single_step=file_data.single_step,
+ )
+
+ if not file_hash.is_executed:
+ execution_service.execute_input_file(
+ file_execution_id=str(workflow_file_execution.id),
+ file_name=file_hash.file_name,
+ single_step=file_data.single_step,
+ workflow_file_execution=workflow_file_execution,
+ )
+ # Log execution costs
+ execution_service.log_total_cost_per_file(
+ run_id=str(workflow_file_execution.id), file_name=file_hash.file_name
+ )
+ tool_execution_result = ToolExecutionResult(error=None, result=None)
+ return tool_execution_result
+ except StopExecution:
+ workflow_file_execution.update_status(status=ExecutionStatus.STOPPED)
+ raise
+ except WorkflowExecutionError as error:
+ tool_execution_result = ToolExecutionResult(
+ error=str(error),
+ result=None,
+ )
+ logger.error(f"Workflow execution failed: {str(error)}", exc_info=True)
+ return tool_execution_result
+ except Exception as error:
+ tool_execution_result = ToolExecutionResult(
+ error=f"Unexpected error: {error}",
+ result=None,
+ )
+ logger.error(f"Workflow execution failed: {str(error)}", exc_info=True)
+ return tool_execution_result
+
+ @classmethod
+ def _finalize_execution(
+ cls,
+ workflow_execution: WorkflowExecution,
+ workflow_file_execution: WorkflowFileExecution,
+ file_hash: FileHash,
+ workflow_log: WorkflowLog,
+ destination: DestinationConnector,
+ execution_result: ToolExecutionResult,
+ ) -> FileExecutionResult:
+ """Handle final processing steps and result generation."""
+ result = cls._process_final_output(
+ destination,
+ workflow_execution.workflow,
+ file_hash,
+ str(workflow_file_execution.id),
+ execution_result.error,
+ )
+
+ cls._complete_execution(
+ workflow_file_execution=workflow_file_execution,
+ workflow_log=workflow_log,
+ error=execution_result.error,
+ )
+
+ return cls._build_final_result(
+ workflow_execution=workflow_execution,
+ workflow_file_execution=workflow_file_execution,
+ file_hash=file_hash,
+ error=execution_result.error,
+ result=result,
+ )
+
+ @classmethod
+ def _process_final_output(
+ cls,
+ destination: DestinationConnector,
+ workflow: Workflow,
+ file_hash: FileHash,
+ file_execution_id: str,
+ processing_error: str | None,
+ ) -> FinalOutputResult:
+ """Handle final output processing and metadata collection."""
+ output_result = None
+ execution_metadata = None
+
+ try:
+ if not processing_error:
+ # Process final output through destination
+ output_result = destination.handle_output(
+ file_name=file_hash.file_name,
+ file_hash=file_hash,
+ workflow=workflow,
+ input_file_path=file_hash.file_path,
+ file_execution_id=file_execution_id,
+ )
+
+ # Collect metadata from file history if available
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash.file_hash
+ )
+ if file_history and destination.is_api:
+ execution_metadata = destination.get_metadata(file_history)
+
+ except Exception as e:
+ error_msg = f"Final output processing failed: {str(e)}"
+ logger.error(error_msg, exc_info=True)
+ return FinalOutputResult(output=None, metadata=None, error=error_msg)
+
+ return FinalOutputResult(
+ output=output_result, metadata=execution_metadata, error=None
+ )
+
+ @classmethod
+ def _complete_execution(
+ cls,
+ workflow_file_execution: WorkflowFileExecution,
+ workflow_log: WorkflowLog,
+ error: str | None,
+ ) -> None:
+ """Final cleanup and status updates for the execution."""
+ try:
+ # Update execution status
+ final_status = ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED
+ workflow_file_execution.update_status(
+ status=final_status, execution_error=error
+ )
+
+ # Publish final log status
+ log_state = LogState.ERROR if error else LogState.SUCCESS
+ workflow_log.publish_update_log(
+ state=log_state,
+ message=f"Execution {'failed' if error else 'completed'} for {workflow_file_execution.file_name}",
+ component=LogComponent.DESTINATION,
+ )
+
+ except Exception as e:
+ logger.error(f"Completion cleanup failed: {str(e)}", exc_info=True)
+
+ @classmethod
+ def _build_final_result(
+ cls,
+ workflow_execution: WorkflowExecution,
+ workflow_file_execution: WorkflowFileExecution,
+ file_hash: FileHash,
+ error: str | None,
+ result: FinalOutputResult,
+ ) -> FileExecutionResult:
+ """Construct and cache the final execution result."""
+ final_result = FileExecutionResult(
+ file=file_hash.file_name,
+ file_execution_id=str(workflow_file_execution.id),
+ error=error,
+ result=result.output,
+ metadata=result.metadata,
+ )
+
+ # Update cache with final result
+ ResultCacheUtils.update_api_results(
+ workflow_id=workflow_execution.workflow.id,
+ execution_id=str(workflow_execution.id),
+ api_result=final_result,
+ )
+
+ return final_result
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index 19d04d9d6..31c8988fe 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -6,6 +6,7 @@ from api_v2.models import APIDeployment
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.db.models import QuerySet, Sum
+from django.utils import timezone
from pipeline_v2.models import Pipeline
from tags.models import Tag
from usage_v2.constants import UsageKeys
@@ -13,6 +14,8 @@ from usage_v2.models import Usage
from utils.common_utils import CommonUtils
from utils.models.base_model import BaseModel
+from workflow_manager.execution.dto import ExecutionCache
+from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models import Workflow
@@ -220,6 +223,10 @@ class WorkflowExecution(BaseModel):
return total_cost
+ @property
+ def is_completed(self) -> bool:
+ return ExecutionStatus.is_completed(self.status)
+
def __str__(self) -> str:
return (
f"Workflow execution: {self.id} ("
@@ -229,3 +236,67 @@ class WorkflowExecution(BaseModel):
f"files: {self.total_files}, "
f"error message: {self.error_message})"
)
+
+ def update_execution(
+ self,
+ status: ExecutionStatus | None = None,
+ error: str | None = None,
+ increment_attempt: bool = False,
+ ) -> None:
+ """Update the execution status and related fields.
+
+ Args:
+ status (Optional[ExecutionStatus], optional): New execution status. Defaults to None.
+ error (Optional[str], optional): Error message if any. Defaults to None.
+ increment_attempt (bool, optional): Whether to increment attempt counter. Defaults to False.
+ """
+ if status is not None:
+ self.status = status.value
+ if (
+ status
+ in [
+ ExecutionStatus.COMPLETED,
+ ExecutionStatus.ERROR,
+ ExecutionStatus.STOPPED,
+ ]
+ and not self.execution_time
+ ):
+ self.execution_time = round(
+ (timezone.now() - self.created_at).total_seconds(), 3
+ )
+ if error:
+ self.error_message = error[:EXECUTION_ERROR_LENGTH]
+ if increment_attempt:
+ self.attempts += 1
+
+ self.save()
+
+ def update_execution_err(self, err_msg: str = "") -> None:
+ """Update execution status to ERROR with an error message.
+
+ Args:
+ err_msg (str, optional): Error message to store. Defaults to "".
+ """
+ self.update_execution(status=ExecutionStatus.ERROR, error=err_msg)
+
+ def _handle_execution_cache(self):
+ if not ExecutionCacheUtils.is_execution_exists(
+ workflow_id=self.workflow.id, execution_id=self.id
+ ):
+ execution_cache = ExecutionCache(
+ workflow_id=self.workflow.id,
+ execution_id=self.id,
+ total_files=self.total_files,
+ status=self.status,
+ )
+ ExecutionCacheUtils.create_execution(
+ execution=execution_cache,
+ )
+ else:
+ ExecutionCacheUtils.update_status(
+ workflow_id=self.workflow.id, execution_id=self.id, status=self.status
+ )
+
+ def save(self, *args, **kwargs):
+ super().save(*args, **kwargs)
+ self._handle_execution_cache()
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index f1a57fac8..97a49682f 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -1,19 +1,21 @@
import json
import logging
+import math
import os
+import time
import traceback
from typing import Any
from account_v2.constants import Common
from api_v2.models import APIDeployment
from api_v2.utils import APIDeploymentUtils
-from celery import current_task, shared_task
+from celery import chord, current_task
from celery import exceptions as celery_exceptions
from celery.result import AsyncResult
+from django.conf import settings
from django.db import IntegrityError
from pipeline_v2.models import Pipeline
from pipeline_v2.pipeline_processor import PipelineProcessor
-from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil
from rest_framework import serializers
from tool_instance_v2.constants import ToolInstanceKey
from tool_instance_v2.models import ToolInstance
@@ -24,19 +26,26 @@ from utils.local_context import StateStore
from utils.user_context import UserContext
from backend.celery_service import app as celery_app
-from unstract.workflow_execution.enums import LogComponent, LogLevel, LogState
-from unstract.workflow_execution.exceptions import StopExecution
+from unstract.workflow_execution.enums import LogStage
from workflow_manager.endpoint_v2.destination import DestinationConnector
-from workflow_manager.endpoint_v2.dto import FileHash
+from workflow_manager.endpoint_v2.dto import (
+ FileHash,
+)
+from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
from workflow_manager.endpoint_v2.source import SourceConnector
-from workflow_manager.file_execution.models import WorkflowFileExecution
+from workflow_manager.execution.dto import ExecutionCache
+from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
+from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.constants import (
- CeleryConfigurations,
WorkflowErrors,
WorkflowExecutionKey,
WorkflowMessages,
)
-from workflow_manager.workflow_v2.dto import AsyncResultData, ExecutionResponse
+from workflow_manager.workflow_v2.dto import (
+ ExecutionResponse,
+ FileBatchData,
+ FileData,
+)
from workflow_manager.workflow_v2.enums import ExecutionStatus, SchemaEntity, SchemaType
from workflow_manager.workflow_v2.exceptions import (
InvalidRequest,
@@ -46,6 +55,7 @@ from workflow_manager.workflow_v2.exceptions import (
WorkflowExecutionNotExist,
)
from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow_v2.file_execution_tasks import FileExecutionTasks
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -58,8 +68,6 @@ class WorkflowHelper:
def get_workflow_by_id(id: str) -> Workflow:
try:
workflow: Workflow = Workflow.objects.get(pk=id)
- if not workflow or workflow is None:
- raise WorkflowDoesNotExistError()
return workflow
except Workflow.DoesNotExist:
logger.error(f"Error getting workflow: {id}")
@@ -84,221 +92,123 @@ class WorkflowHelper:
workflow.save()
return workflow
- @staticmethod
- def build_workflow_execution_service(
- organization_id: str | None,
- workflow: Workflow,
- tool_instances: list[ToolInstance],
- pipeline_id: str | None,
- single_step: bool,
- scheduled: bool,
- execution_mode: tuple[str, str],
- workflow_execution: WorkflowExecution,
- use_file_history: bool = True, # Will be False for API deployment alone
- ) -> WorkflowExecutionServiceHelper:
- workflow_execution_service = WorkflowExecutionServiceHelper(
- organization_id=organization_id,
- workflow=workflow,
- tool_instances=tool_instances,
- pipeline_id=pipeline_id,
- single_step=single_step,
- scheduled=scheduled,
- mode=execution_mode,
- workflow_execution=workflow_execution,
- use_file_history=use_file_history,
- )
- workflow_execution_service.build()
- return workflow_execution_service
-
@classmethod
- def process_input_files(
- cls,
- workflow: Workflow,
- source: SourceConnector,
- destination: DestinationConnector,
- execution_service: WorkflowExecutionServiceHelper,
- single_step: bool,
- input_files: dict[str, FileHash],
- ) -> WorkflowExecution:
- total_files = len(input_files)
- error_message = None
- successful_files = 0
- failed_files = 0
- execution_service.publish_initial_workflow_logs(total_files)
- execution_service.update_execution(
- ExecutionStatus.EXECUTING, increment_attempt=True
- )
- q_file_no_list = (
- WorkflowUtil.get_q_no_list(workflow, total_files) if total_files > 0 else []
- )
+ def get_file_batches(
+ cls, input_files: dict[str, FileHash]
+ ) -> list[list[tuple[str, FileHash]]]:
+ """_summary_
- for index, (file_name, file_hash) in enumerate(input_files.items()):
- # Get workflow execution file
- file_number = index + 1
- file_hash = WorkflowUtil.add_file_destination_filehash(
- file_number,
- q_file_no_list,
- file_hash,
- )
- try:
- error = cls._process_file(
- current_file_idx=file_number,
- total_files=total_files,
- input_file=file_hash.file_path,
- workflow=workflow,
- source=source,
- destination=destination,
- execution_service=execution_service,
- single_step=single_step,
- file_hash=file_hash,
- )
- if error:
- failed_files += 1
- else:
- successful_files += 1
- except StopExecution as e:
- execution_service.update_execution(ExecutionStatus.STOPPED, error=str(e))
- break
- except Exception as e:
- failed_files += 1
- error_message = f"Error processing file '{file_name}'. {e}"
- logger.error(error_message, stack_info=True, exc_info=True)
- execution_service.publish_log(message=error_message, level=LogLevel.ERROR)
- # TODO: Store only generic WF errors here (concerning all failed files)
- # TODO: Review if we need partial success
- final_status = (
- ExecutionStatus.ERROR
- if total_files > 0 and failed_files >= total_files
- else ExecutionStatus.COMPLETED
- )
- execution_service.update_execution(final_status, error=error_message)
+ Args:
+ input_files (dict[str, FileHash]): input files
- workflow_execution = execution_service.get_execution_instance()
+ Returns:
+ batches: batches of input files
+ """
+ json_serializable_files = {
+ file_name: file_hash.to_json() for file_name, file_hash in input_files.items()
+ }
- execution_service.publish_final_workflow_logs(
- total_files=total_files,
- successful_files=successful_files,
- failed_files=failed_files,
- )
+ # Prepare batches of files for parallel processing
+ BATCH_SIZE = settings.MAX_PARALLEL_FILE_BATCHES # Max number of batches
+ file_items = list(json_serializable_files.items())
+
+ # Calculate how many items per batch
+ num_files = len(file_items)
+ num_batches = min(BATCH_SIZE, num_files)
+ items_per_batch = math.ceil(num_files / num_batches)
- return workflow_execution
+ # Split into batches
+ batches = []
+ for start_index in range(0, len(file_items), items_per_batch):
+ end_index = start_index + items_per_batch
+ batch = file_items[start_index:end_index]
+ batches.append(batch)
+
+ return batches
@classmethod
- def _process_file(
+ def process_input_files(
cls,
- current_file_idx: int,
- total_files: int,
- input_file: str,
workflow: Workflow,
source: SourceConnector,
destination: DestinationConnector,
- execution_service: WorkflowExecutionServiceHelper,
+ workflow_log: WorkflowLog,
+ workflow_execution: WorkflowExecution,
single_step: bool,
- file_hash: FileHash,
+ input_files: dict[str, FileHash],
+ organization_id: str,
+ pipeline_id: str,
+ scheduled: bool,
+ execution_mode: tuple[str, str],
+ use_file_history: bool,
) -> str | None:
- # Delete previous execution directory to avoid conflicts
- destination.delete_execution_directory()
-
- # Multiple run_ids are linked to an execution_id
- # Each run_id corresponds to workflow runs for a single file
- # It should e uuid of workflow_file_execution
- workflow_file_execution: WorkflowFileExecution = (
- WorkflowFileExecution.objects.get_or_create_file_execution(
- workflow_execution=execution_service.workflow_execution,
- file_hash=file_hash,
- connection_type=source.endpoint.connection_type,
- )
- )
- workflow_file_execution.update_status(status=ExecutionStatus.EXECUTING)
- file_execution_id = str(workflow_file_execution.id)
- file_name = file_hash.file_name
-
- # This will add the file to the volume
- file_content_hash = source.add_file_to_volume(
- workflow_file_execution=workflow_file_execution,
- tags=execution_service.tags,
- file_hash=file_hash,
- )
+ total_files = len(input_files)
+ workflow_log.publish_initial_workflow_logs(total_files=total_files)
- # Update file_hash after adding to volume
- file_hash.file_hash = file_content_hash
- workflow_file_execution.update(file_hash=file_content_hash)
+ workflow_execution.update_execution(
+ status=ExecutionStatus.EXECUTING, increment_attempt=True
+ )
- # Ensure no duplicate file processing
- if (
- FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_content_hash
- )
- and not destination.is_api
- ):
- logger.info(f"Skipping '{file_name}', already processed.")
- execution_service.publish_log(
- f"Skipping '{file_name}', already processed.", LogLevel.INFO
+ if not input_files:
+ logger.info(f"Execution {workflow_execution.id} no files to process")
+ workflow_execution.update_execution(
+ status=ExecutionStatus.COMPLETED,
)
- return None
+ return
- try:
- execution_service.file_execution_id = file_execution_id
- execution_service.initiate_tool_execution(
- current_file_idx, total_files, file_name, single_step
+ batches = cls.get_file_batches(input_files=input_files)
+ batch_tasks = []
+ mode = (
+ execution_mode[1]
+ if isinstance(execution_mode, tuple)
+ else str(execution_mode)
+ )
+ result = None
+ logger.info(
+ f"Execution {workflow_execution.id} processing {total_files} files in {len(batches)} batches"
+ )
+ for batch in batches:
+ # Convert all UUIDs to strings in batch_data
+ file_data = FileData(
+ workflow_id=str(workflow.id),
+ source_config=source.get_config().to_json(),
+ destination_config=destination.get_config().to_json(),
+ execution_id=str(workflow_execution.id),
+ single_step=single_step,
+ organization_id=str(organization_id),
+ pipeline_id=str(pipeline_id),
+ scheduled=scheduled,
+ execution_mode=mode,
+ use_file_history=use_file_history,
)
- workflow_file_execution.update_status(status=ExecutionStatus.EXECUTING)
- if not file_hash.is_executed:
- execution_service.execute_input_file(
- file_execution_id=file_execution_id,
- file_name=file_name,
- single_step=single_step,
- workflow_file_execution=workflow_file_execution,
+ batch_data = FileBatchData(files=batch, file_data=file_data)
+
+ # Determine the appropriate queue based on execution_mode
+ file_processing_queue = FileExecutionTasks.get_queue_name(source)
+
+ # Send each batch to the dedicated file_processing queue
+ batch_tasks.append(
+ FileExecutionTasks.process_file_batch.s(batch_data.to_dict()).set(
+ queue=file_processing_queue
)
- except StopExecution:
- workflow_file_execution.update_status(status=ExecutionStatus.STOPPED)
- raise
- except WorkflowExecutionError as e:
- error_message = f"Error processing file '{file_name}': {e}"
- execution_service.publish_log(error_message, level=LogLevel.ERROR)
- error = error_message
- except Exception as e:
- error_message = f"Unexpected error processing file '{file_name}': {str(e)}"
- logger.error(error_message, stack_info=True, exc_info=True)
- execution_service.publish_log(error_message, level=LogLevel.ERROR)
- error = error_message
- # Handling error based on destination and continuing for other files
- else:
- error = None
- execution_service.publish_update_log(
- LogState.RUNNING,
- f"Processing output for {file_name}",
- LogComponent.DESTINATION,
- )
- try:
- destination.handle_output(
- file_name=file_name,
- file_hash=file_hash,
- workflow=workflow,
- input_file_path=input_file,
- error=error,
- use_file_history=execution_service.use_file_history,
- file_execution_id=file_execution_id,
)
+ try:
+ result = chord(batch_tasks)(
+ FileExecutionTasks.process_batch_callback.s(
+ execution_id=str(workflow_execution.id)
+ ).set(queue=file_processing_queue)
+ )
+ if not result.id:
+ exception = f"Failed to queue execution task {workflow_execution.id}"
+ logger.error(exception)
+ raise WorkflowExecutionError(exception)
+ logger.info(f"Execution {workflow_execution.id} task queued successfully")
except Exception as e:
- error_message = f"Error during output processing for '{file_name}': {str(e)}"
- logger.error(error_message, stack_info=True, exc_info=True)
- execution_service.publish_log(error_message, level=LogLevel.ERROR)
- error = error_message
- finally:
- execution_service.log_total_cost_per_file(
- run_id=file_execution_id, file_name=file_name
- )
- execution_service.publish_update_log(
- LogState.SUCCESS,
- f"{file_name}'s output is processed successfully",
- LogComponent.DESTINATION,
- )
- workflow_file_execution.update_status(
- status=ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED,
- execution_error=error if error else None,
+ workflow_execution.update_execution(
+ status=ExecutionStatus.ERROR,
+ error=f"Error while processing files: {str(e)}",
)
- return error
+ return result.id
@staticmethod
def validate_tool_instances_meta(
@@ -328,31 +238,27 @@ class WorkflowHelper:
workflow.id, ToolInstanceKey.STEP
)
)
-
WorkflowHelper.validate_tool_instances_meta(tool_instances=tool_instances)
execution_mode = execution_mode or WorkflowExecution.Mode.INSTANT
- execution_service = WorkflowHelper.build_workflow_execution_service(
+ execution_id = str(workflow_execution.id)
+ workflow_log = WorkflowLog(
+ execution_id=workflow_execution.id,
organization_id=organization_id,
- workflow=workflow,
- tool_instances=tool_instances,
+ log_stage=LogStage.INITIALIZE,
pipeline_id=pipeline_id,
- single_step=single_step,
- scheduled=scheduled,
- execution_mode=execution_mode,
- workflow_execution=workflow_execution,
- use_file_history=use_file_history,
)
- execution_id = execution_service.execution_id
source = SourceConnector(
- organization_id=organization_id,
workflow=workflow,
execution_id=execution_id,
- execution_service=execution_service,
+ workflow_log=workflow_log,
+ use_file_history=use_file_history,
+ organization_id=organization_id,
)
destination = DestinationConnector(
workflow=workflow,
execution_id=execution_id,
- execution_service=execution_service,
+ workflow_log=workflow_log,
+ use_file_history=use_file_history,
)
try:
# Validating endpoints
@@ -362,40 +268,44 @@ class WorkflowHelper:
input_files, total_files = source.list_files_from_source(hash_values_of_files)
workflow_execution.total_files = total_files
workflow_execution.save()
- workflow_execution = WorkflowHelper.process_input_files(
- workflow,
- source,
- destination,
- execution_service,
+ WorkflowHelper.process_input_files(
+ workflow=workflow,
+ source=source,
+ destination=destination,
+ workflow_log=workflow_log,
+ workflow_execution=workflow_execution,
single_step=single_step,
input_files=input_files,
+ organization_id=organization_id,
+ pipeline_id=pipeline_id,
+ scheduled=scheduled,
+ use_file_history=use_file_history,
+ execution_mode=execution_mode,
)
WorkflowHelper._update_pipeline_status(
pipeline_id=pipeline_id, workflow_execution=workflow_execution
)
+ api_results = []
return ExecutionResponse(
str(workflow.id),
str(workflow_execution.id),
workflow_execution.status,
- log_id=str(execution_service.execution_log_id),
+ log_id=workflow_log.messaging_channel,
error=workflow_execution.error_message,
mode=workflow_execution.execution_mode,
- result=destination.api_results,
+ result=api_results,
)
except Exception as e:
logger.error(f"Error executing workflow {workflow}: {e}")
logger.error(f"Error {traceback.format_exc()}")
- workflow_execution = WorkflowExecutionServiceHelper.update_execution_err(
- execution_id, str(e)
+ workflow_execution.update_execution(
+ status=ExecutionStatus.ERROR,
+ error=str(e),
)
WorkflowHelper._update_pipeline_status(
pipeline_id=pipeline_id, workflow_execution=workflow_execution
)
raise
- finally:
- # TODO: Handle error gracefully during delete
- # Mark status as an ERROR correctly
- destination.delete_execution_and_api_storage_dir()
@staticmethod
def _update_pipeline_status(
@@ -449,26 +359,26 @@ class WorkflowHelper:
Returns:
ExecutionResponse: _description_
"""
- execution = WorkflowExecution.objects.get(id=execution_id)
+ execution: WorkflowExecution = WorkflowExecution.objects.get(id=execution_id)
if not execution.task_id:
raise TaskDoesNotExistError(f"No task ID found for execution: {execution_id}")
- result = celery_app.AsyncResult(str(execution.task_id))
- task = AsyncResultData(async_result=result)
-
+ task_result = None
+ result_acknowledged = execution.result_acknowledged
# Prepare the initial response with the task's current status and result.
+ if execution.is_completed:
+ task_result = ResultCacheUtils.get_api_results(
+ workflow_id=str(execution.workflow.id), execution_id=execution_id
+ )
+ cls._set_result_acknowledge(execution)
+
result_response = ExecutionResponse(
- execution.workflow_id,
- execution_id,
- execution.status,
- result=task.result,
- result_acknowledged=execution.result_acknowledged,
+ workflow_id=str(execution.workflow.id),
+ execution_id=execution_id,
+ execution_status=execution.status,
+ result=task_result,
+ result_acknowledged=result_acknowledged,
)
-
- # If task is complete, handle acknowledgment and forgetting the
- if result.ready():
- result.forget() # Remove the result from the result backend.
- cls._set_result_acknowledge(execution)
return result_response
@staticmethod
@@ -487,6 +397,37 @@ class WorkflowHelper:
logger.info(
f"ExecutionID [{execution.id}] - Task {execution.task_id} acknowledged"
)
+ # Delete api results from cache
+ ResultCacheUtils.delete_api_results(
+ workflow_id=str(execution.workflow.id), execution_id=str(execution.id)
+ )
+ ExecutionCacheUtils.delete_execution(
+ workflow_id=str(execution.workflow.id), execution_id=str(execution.id)
+ )
+
+ @classmethod
+ def _get_execution_status(
+ cls, workflow_id: str, execution_id: str
+ ) -> ExecutionStatus:
+ execution_cache = ExecutionCacheUtils.get_execution(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ if not execution_cache:
+ execution_model: WorkflowExecution = WorkflowExecution.objects.get(
+ id=execution_id
+ )
+ execution_cache = ExecutionCache(
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ status=ExecutionStatus(execution_model.status),
+ total_files=execution_model.total_files,
+ completed_files=execution_model.completed_files,
+ failed_files=execution_model.failed_files,
+ )
+ ExecutionCacheUtils.create_execution(
+ execution=execution_cache,
+ )
+ return execution_cache.status
@classmethod
def execute_workflow_async(
@@ -540,29 +481,37 @@ class WorkflowHelper:
f"[{org_schema}] Job '{async_execution}' has been enqueued for "
f"execution_id '{execution_id}', '{len(hash_values_of_files)}' files"
)
- workflow_execution = WorkflowExecution.objects.get(id=execution_id)
+ workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(
+ id=execution_id
+ )
workflow_execution.task_id = async_execution.id
workflow_execution.save()
+ execution_status = workflow_execution.status
if timeout > -1:
- async_execution.wait(
- timeout=timeout,
- interval=CeleryConfigurations.INTERVAL,
+ while not ExecutionStatus.is_completed(execution_status) and timeout > 0:
+ time.sleep(2)
+ timeout -= 2
+
+ execution_status = cls._get_execution_status(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ if ExecutionStatus.is_completed(execution_status):
+ # Fetch the object agian to get the latest status.
+ workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(
+ id=execution_id
+ )
+ task_result = ResultCacheUtils.get_api_results(
+ workflow_id=workflow_id, execution_id=execution_id
)
- task = AsyncResultData(async_result=async_execution)
- celery_result = task.to_dict()
- task_result = celery_result.get("result")
- # Fetch the object agian to get the latest status.
- workflow_execution = WorkflowExecution.objects.get(id=execution_id)
+ cls._set_result_acknowledge(workflow_execution)
+ else:
+ task_result = None
execution_response = ExecutionResponse(
workflow_id,
execution_id,
- workflow_execution.status,
+ execution_status,
result=task_result,
)
- # If task is complete, handle acknowledgment and forgetting the
- if async_execution.ready():
- async_execution.forget() # Remove the result from the result backend.
- cls._set_result_acknowledge(workflow_execution)
return execution_response
except celery_exceptions.TimeoutError:
return ExecutionResponse(
@@ -587,7 +536,7 @@ class WorkflowHelper:
)
@staticmethod
- @shared_task(
+ @celery_app.task(
name="async_execute_bin",
autoretry_for=(Exception,),
max_retries=0,
@@ -630,7 +579,7 @@ class WorkflowHelper:
task_id = current_task.request.id
# Set organization in state store for execution
StateStore.set(Account.ORGANIZATION_ID, schema_name)
- return WorkflowHelper.execute_workflow(
+ WorkflowHelper.execute_workflow(
organization_id=schema_name,
task_id=task_id,
workflow_id=workflow_id,
@@ -775,6 +724,9 @@ class WorkflowHelper:
use_file_history=use_file_history,
log_events_id=log_events_id,
)
+ ExecutionCacheUtils.delete_execution(
+ workflow_id=str(workflow.id), execution_id=str(execution_id)
+ )
updated_execution = WorkflowExecution.objects.get(id=execution_id)
execution_response = ExecutionResponse(
workflow.id,
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 6fd958231..e23afc9fa 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -93,6 +93,47 @@ services:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
+ # Celery worker for handling file processing tasks
+ worker-file-processing:
+ image: unstract/backend:${VERSION}
+ container_name: unstract-worker-file-processing
+ restart: unless-stopped
+ entrypoint: .venv/bin/celery
+ command: "-A backend.workers.file_processing worker --loglevel=info -Q file_processing --autoscale=${WORKER_FILE_PROCESSING_AUTOSCALE}"
+ env_file:
+ - ../backend/.env
+ depends_on:
+ - redis
+ environment:
+ - ENVIRONMENT=development
+ - APPLICATION_NAME=unstract-worker-file-processing
+ labels:
+ - traefik.enable=false
+ volumes:
+ - ./workflow_data:/data
+ - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
+
+
+ # Celery worker for handling API file processing tasks
+ worker-api-file-processing:
+ image: unstract/backend:${VERSION}
+ container_name: unstract-worker-api-file-processing
+ restart: unless-stopped
+ entrypoint: .venv/bin/celery
+ command: "-A backend.workers.file_processing worker --loglevel=info -Q api_file_processing --autoscale=${WORKER_API_FILE_PROCESSING_AUTOSCALE}"
+ env_file:
+ - ../backend/.env
+ depends_on:
+ - redis
+ environment:
+ - ENVIRONMENT=development
+ - APPLICATION_NAME=unstract-worker-api-file-processing
+ labels:
+ - traefik.enable=false
+ volumes:
+ - ./workflow_data:/data
+ - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
+
# Celery Flower
celery-flower:
image: unstract/backend:${VERSION}
diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yaml
index f4ac352b2..f6e92877c 100644
--- a/docker/sample.compose.override.yaml
+++ b/docker/sample.compose.override.yaml
@@ -4,6 +4,8 @@
services:
worker:
command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue,celery_api_deployments --autoscale=${WORKER_AUTOSCALE}"
+ worker-file-processing:
+ command: "-A backend.workers.file_processing worker --loglevel=info -Q file_processing,api_file_processing --autoscale=${WORKER_FILE_PROCESSING_AUTOSCALE}"
worker-logging:
profiles:
diff --git a/docker/sample.env b/docker/sample.env
index 9af5d82fd..269bca95f 100644
--- a/docker/sample.env
+++ b/docker/sample.env
@@ -10,3 +10,5 @@ TOOL_REGISTRY_CONFIG_SRC_PATH="${PWD}/../unstract/tool-registry/tool_registry_co
WORKER_API_DEPLOYMENTS_AUTOSCALE=4,1
WORKER_LOGGING_AUTOSCALE=4,1
WORKER_AUTOSCALE=4,1
+WORKER_FILE_PROCESSING_AUTOSCALE=4,1
+WORKER_API_FILE_PROCESSING_AUTOSCALE=4,1
diff --git a/runner/src/unstract/runner/clients/interface.py b/runner/src/unstract/runner/clients/interface.py
index b1cd9b42c..09b6fbbee 100644
--- a/runner/src/unstract/runner/clients/interface.py
+++ b/runner/src/unstract/runner/clients/interface.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
+
import logging
from abc import ABC, abstractmethod
from collections.abc import Iterator
-from typing import Any, Optional
+from typing import Any
class ContainerInterface(ABC):
@@ -24,7 +26,7 @@ class ContainerInterface(ABC):
pass
@abstractmethod
- def cleanup(self, client: Optional["ContainerClientInterface"] = None) -> None:
+ def cleanup(self, client: ContainerClientInterface | None = None) -> None:
"""Stops and removes the running container."""
pass
diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.py
index 870b2b069..0d253096a 100644
--- a/runner/src/unstract/runner/runner.py
+++ b/runner/src/unstract/runner/runner.py
@@ -335,6 +335,7 @@ class UnstractRunner:
organization_id,
workflow_id,
execution_id,
+ file_execution_id,
)
envs[Env.WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS] = os.getenv(
Env.WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS, "{}"
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py
index b41818dea..dc141bb61 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/enums.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/enums.py
@@ -33,6 +33,10 @@ class LogStage(Enum):
BUILD = "BUILD"
TOOL = "TOOL"
RUN = "RUN"
+ SOURCE = "SOURCE"
+ DESTINATION = "DESTINATION"
+ INITIALIZE = "INITIALIZE"
+ PROCESSING = "PROCESSING"
class LogState(Enum):
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py b/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py
index 7ece4b5bc..4706cda34 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/exceptions.py
@@ -54,3 +54,24 @@ class ToolMetadataNotFound(Exception):
self, message: str = "Internal server error: Tool metadata not found."
) -> None:
self.message = message
+
+
+class ExecutionDirectoryNotFound(Exception):
+ def __init__(
+ self, message: str = "Internal server error: Execution directory not set."
+ ) -> None:
+ self.message = message
+
+
+class FileExecutionNotFound(Exception):
+ def __init__(
+ self, message: str = "Internal server error: File execution not found."
+ ) -> None:
+ self.message = message
+
+
+class FileMetadataJsonNotFound(Exception):
+ def __init__(
+ self, message: str = "Internal server error: File metadata JSON not found."
+ ) -> None:
+ self.message = message
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
index 30a5301e7..9a8ec919e 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
@@ -12,25 +12,36 @@ from unstract.workflow_execution.constants import (
ToolRuntimeVariable,
WorkflowFileType,
)
-from unstract.workflow_execution.exceptions import ToolMetadataNotFound
+from unstract.workflow_execution.exceptions import (
+ ExecutionDirectoryNotFound,
+ FileExecutionNotFound,
+ FileMetadataJsonNotFound,
+ ToolMetadataNotFound,
+)
from unstract.workflow_execution.tools_utils import ToolsUtils
logger = logging.getLogger(__name__)
class ExecutionFileHandler:
- def __init__(self, workflow_id: str, execution_id: str, organization_id: str) -> None:
+ def __init__(
+ self,
+ workflow_id: str,
+ execution_id: str,
+ organization_id: str,
+ file_execution_id: str | None = None,
+ ) -> None:
self.organization_id = organization_id
self.workflow_id = workflow_id
self.execution_id = execution_id
+ self.file_execution_id = file_execution_id
self.execution_dir = self.get_execution_dir(
workflow_id, execution_id, organization_id
)
- self.source_file = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
- self.infile = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
- self.metadata_file = os.path.join(
- self.execution_dir, WorkflowFileType.METADATA_JSON
- )
+ self.file_execution_dir = self._get_file_execution_dir()
+ self.source_file = self._get_source_file_path()
+ self.infile = self._get_infile_path()
+ self.metadata_file = self._get_metadata_file_path()
def get_workflow_metadata(self) -> dict[str, Any]:
"""Get metadata for the workflow.
@@ -38,6 +49,8 @@ class ExecutionFileHandler:
Returns:
dict[str, Any]: Workflow metadata.
"""
+ if not self.metadata_file:
+ raise FileMetadataJsonNotFound()
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
metadata_content = file_storage.read(path=self.metadata_file, mode="r")
@@ -106,7 +119,11 @@ class ExecutionFileHandler:
Raises:
None
"""
- metadata_path = os.path.join(self.execution_dir, WorkflowFileType.METADATA_JSON)
+ if not self.file_execution_dir:
+ raise FileExecutionNotFound()
+ metadata_path = self.metadata_file
+ if not metadata_path:
+ raise FileMetadataJsonNotFound()
filename = os.path.basename(input_file_path)
content = {
MetaDataKey.SOURCE_NAME: filename,
@@ -125,6 +142,18 @@ class ExecutionFileHandler:
f"metadata for {input_file_path} is " "added in to execution directory"
)
+ def _get_file_execution_dir(self) -> str | None:
+ """Get the directory path for a specific file execution.
+
+ Returns:
+ str: (Optional) The directory path for the file execution.
+ """
+ if not self.execution_dir:
+ raise ExecutionDirectoryNotFound()
+ if not self.file_execution_id:
+ return None
+ return os.path.join(self.execution_dir, self.file_execution_id)
+
@classmethod
def get_execution_dir(
cls, workflow_id: str, execution_id: str, organization_id: str
@@ -134,7 +163,7 @@ class ExecutionFileHandler:
Parameters:
- workflow_id (str): Identifier for the workflow.
- execution_id (str): Identifier for the execution.
- - organization_id (Optional[str]):
+ - organization_id (str | None):
Identifier for the organization (default: None).
Returns:
@@ -158,7 +187,7 @@ class ExecutionFileHandler:
Parameters:
- workflow_id (str): Identifier for the workflow.
- execution_id (str): Identifier for the execution.
- - organization_id (Optional[str]):
+ - organization_id (str | None):
Identifier for the organization (default: None).
Returns:
@@ -169,3 +198,31 @@ class ExecutionFileHandler:
Path(path_prefix) / organization_id / str(workflow_id) / str(execution_id)
)
return str(execution_dir)
+
+ def _get_source_file_path(self) -> str | None:
+ """Returns:
+ str: (Optional) The path to the source file.
+ """
+ if not self.file_execution_dir:
+ return None
+ else:
+ return os.path.join(self.file_execution_dir, WorkflowFileType.SOURCE)
+
+ def _get_infile_path(self) -> str | None:
+ """Returns:
+ str: (Optional) The path to the infile.
+ """
+ if not self.file_execution_dir:
+ return None
+ return os.path.join(self.file_execution_dir, WorkflowFileType.INFILE)
+
+ def _get_metadata_file_path(self) -> str | None:
+ """Get the path to the metadata file.
+
+ Args:
+ None
+ Returns: str: (Optional) The path to the metadata file.
+ """
+ if not self.file_execution_dir:
+ return None
+ return os.path.join(self.file_execution_dir, WorkflowFileType.METADATA_JSON)
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 7c117b02f..acd7a64fe 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -44,6 +44,7 @@ class WorkflowExecutionService:
tool_instances: list[ToolInstance],
platform_service_api_key: str,
ignore_processed_entities: bool = False,
+ file_execution_id: str | None = None,
) -> None:
self.organization_id = organization_id
self.workflow_id = workflow_id
@@ -60,7 +61,7 @@ class WorkflowExecutionService:
self.ignore_processed_entities = ignore_processed_entities
self.override_single_step = False
self.execution_id: str = ""
- self.file_execution_id: str | None = None
+ self.file_execution_id: str | None = file_execution_id
self.messaging_channel: str | None = None
self.input_files: list[str] = []
self.log_stage: LogStage = LogStage.COMPILE
@@ -89,7 +90,10 @@ class WorkflowExecutionService:
try:
self.execution_id = str(execution_id)
self.file_handler = ExecutionFileHandler(
- self.workflow_id, self.execution_id, self.organization_id
+ self.workflow_id,
+ self.execution_id,
+ self.organization_id,
+ file_execution_id=self.file_execution_id,
)
logger.info(f"Execution {execution_id}: compilation completed")
@@ -136,9 +140,7 @@ class WorkflowExecutionService:
logger.info(f"Execution {self.execution_id}: Build completed")
- def execute_workflow(
- self, file_execution_id: str, execution_type: ExecutionType
- ) -> None:
+ def execute_workflow(self, execution_type: ExecutionType) -> None:
"""Executes the complete workflow by running each tools one by one.
Returns the result from final tool in a dictionary.
@@ -156,10 +158,10 @@ class WorkflowExecutionService:
Eg:- {"result": "RESULT_FROM_FINAL_TOOL"}
"""
self.log_stage = LogStage.RUN
+
self._initialize_execution()
total_steps = len(self.tool_sandboxes)
self.total_steps = total_steps
- self.file_execution_id = file_execution_id
# Currently each tool is run serially for files and workflows contain 1 tool
# only. While supporting more tools in a workflow, correct the tool container
# name to avoid conflicts.
@@ -315,6 +317,8 @@ class WorkflowExecutionService:
"""
if not self.execution_id:
raise BadRequestException("Execution Id not found")
+ if not self.file_execution_id:
+ raise BadRequestException("File Execution Id not found")
def _finalize_execution(self, execution_type: ExecutionType) -> None:
"""Finalize the execution process.
@@ -339,11 +343,14 @@ class WorkflowExecutionService:
self._handling_step_execution()
def validate_execution_result(self, step: int) -> bool:
- workflow_metadata = self.file_handler.get_workflow_metadata()
- metadata_list = self.file_handler.get_list_of_tool_metadata(workflow_metadata)
- if len(metadata_list) == step:
- return True
- return False
+ try:
+ workflow_metadata = self.file_handler.get_workflow_metadata()
+ metadata_list = self.file_handler.get_list_of_tool_metadata(workflow_metadata)
+ if len(metadata_list) == step:
+ return True
+ return False
+ except Exception:
+ return False
def publish_log(
self,