LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Worker check runner container status (#1342)
ali730f79ecb8ae
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 2e3cd9ebc..47aa0680c 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -143,6 +143,13 @@ EXECUTION_RESULT_TTL_SECONDS = int(EXECUTION_CACHE_TTL_SECONDS = int(os.environ.get("EXECUTION_CACHE_TTL_SECONDS", 10800)) # 3 hours+FILE_EXECUTION_TRACKER_TTL_IN_SECOND = int(+ os.environ.get("FILE_EXECUTION_TRACKER_TTL_IN_SECOND", 60 * 60 * 5)+)+FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND = int(+ os.environ.get("FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND", 60 * 10)+) # 10 minutes+INSTANT_WF_POLLING_TIMEOUT = int(os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")) # 5 minutesdiff --git a/backend/sample.env b/backend/sample.envindex 82f2674e4..bebae5891 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -178,3 +178,11 @@ INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes# Maximum number of batches (i.e., parallel tasks) created for a single workflow executionMAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time++# File execution tracker ttl in seconds+FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hours+FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600 # 10 minutes++# Runner polling timeout+MAX_RUNNER_POLLING_WAIT_SECONDS=10800 # 3 hours+RUNNER_POLLING_INTERVAL_SECONDS=2 # 2 secondsdiff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex fe496170d..e38c67454 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -3,6 +3,7 @@ from typing import Anyfrom uuid import UUIDfrom account_v2.constants import Common+from django.conf import settingsfrom plugins.workflow_manager.workflow_v2.utils import WorkflowUtilfrom tool_instance_v2.constants import ToolInstanceKeyfrom tool_instance_v2.models import ToolInstance@@ -20,6 +21,13 @@ from backend.workers.file_processing_callback.constants import (from backend.workers.file_processing_callback.file_processing_callback import (app as file_processing_callback_app,)+from unstract.core.file_execution_tracker import (+ FileExecutionData,+ FileExecutionStage,+ FileExecutionStageData,+ FileExecutionStageStatus,+ FileExecutionStatusTracker,+)from unstract.workflow_execution.enums import LogComponent, LogStage, LogStatefrom unstract.workflow_execution.exceptions import StopExecutionfrom workflow_manager.endpoint_v2.destination import DestinationConnector@@ -350,7 +358,50 @@ class FileExecutionTasks:destination = DestinationConnector.from_config(workflow_log, destination_config)- destination.delete_file_execution_directory()++ # Set file execution tracker+ file_execution_data = cls._set_file_execution_tracker(+ execution_id=str(workflow_execution.id),+ file_execution_id=str(workflow_file_execution.id),+ organization_id=str(workflow_execution.workflow.organization_id),+ )++ # Check if file execution is already in progress+ stage = file_execution_data.stage_status.stage+ if stage.is_after(FileExecutionStage.INITIALIZATION):+ logger.info(+ f"File already tracked. Resuming execution for execution_id: {workflow_execution.id}, "+ f"file_execution_id: {workflow_file_execution.id}, current_stage: {stage.value}"+ )+ # Not required to prepare file again+ # Not required to check processing history again+ if stage.is_before(FileExecutionStage.COMPLETED):+ # Core Execution Phase+ execution_result = cls._execute_workflow_steps(+ file_data,+ workflow_execution,+ workflow_file_execution,+ file_hash,+ current_file_idx,+ total_files,+ )++ # Finalization Phase+ return cls._finalize_execution(+ workflow_execution,+ workflow_file_execution,+ file_hash,+ workflow_log,+ destination,+ execution_result,+ )+ # If stage is already completed, skip execution+ return cls._build_final_result(+ workflow_execution,+ file_hash,+ FinalOutputResult.skipped(),+ workflow_file_execution,+ )# File Preparation Phasecontent_hash = cls._prepare_file_for_processing(@@ -432,6 +483,82 @@ class FileExecutionTasks:is_api=destination.is_api,)+ @classmethod+ def _set_file_execution_tracker(+ cls,+ execution_id: str,+ file_execution_id: str,+ organization_id: str,+ ) -> FileExecutionData:+ # Initialize file execution tracker+ file_execution_tracker = FileExecutionStatusTracker()+ file_execution_stage_data = FileExecutionStageData(+ stage=FileExecutionStage.INITIALIZATION,+ status=FileExecutionStageStatus.IN_PROGRESS,+ )+ file_execution_data = FileExecutionData(+ execution_id=str(execution_id),+ file_execution_id=str(file_execution_id),+ organization_id=str(organization_id),+ stage_status=file_execution_stage_data,+ status_history=[],+ )+ if not file_execution_tracker.exists(execution_id, file_execution_id):+ file_execution_tracker.set_data(file_execution_data)+ else:+ file_execution_data = file_execution_tracker.get_data(+ execution_id, file_execution_id+ )+ logger.info(+ f"File execution tracker already exists for execution_id: {execution_id}, file_execution_id: {file_execution_id}"+ )+ return file_execution_data++ @classmethod+ def _mark_file_execution_tracker_finalization_status(+ cls,+ execution_id: str,+ file_execution_id: str,+ status: FileExecutionStageStatus,+ error: str | None = None,+ ) -> None:+ file_execution_tracker = FileExecutionStatusTracker()+ stage_data = FileExecutionStageData(+ stage=FileExecutionStage.FINALIZATION,+ status=status,+ error=error,+ )+ file_execution_tracker.update_stage_status(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ stage_status=stage_data,+ )++ @classmethod+ def _mark_file_execution_tracker_completed_status(+ cls,+ execution_id: str,+ file_execution_id: str,+ status: FileExecutionStageStatus,+ error: str | None = None,+ ) -> None:+ # This is the cache ttl for completed file execution to prevent long term cache+ COMPLETED_EXECUTION_CACHE_TTL_IN_SECOND = (+ settings.FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND+ )+ file_execution_tracker = FileExecutionStatusTracker()+ stage_data = FileExecutionStageData(+ stage=FileExecutionStage.COMPLETED,+ status=status,+ error=error,+ )+ file_execution_tracker.update_stage_status(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ stage_status=stage_data,+ ttl_in_second=COMPLETED_EXECUTION_CACHE_TTL_IN_SECOND,+ )+@classmethoddef _initialize_execution_context(cls,@@ -669,7 +796,7 @@ class FileExecutionTasks:workflow_log=workflow_log,error=result.error or execution_result.error,)-+ destination.delete_file_execution_directory()return cls._build_final_result(workflow_execution=workflow_execution,file_hash=file_hash,@@ -795,6 +922,18 @@ class FileExecutionTasks:component=LogComponent.DESTINATION,)+ file_execution_tracker_status = (+ FileExecutionStageStatus.SUCCESS+ if not error+ else FileExecutionStageStatus.FAILED+ )+ cls._mark_file_execution_tracker_finalization_status(+ execution_id=str(workflow_file_execution.workflow_execution.id),+ file_execution_id=str(workflow_file_execution.id),+ status=file_execution_tracker_status,+ error=error,+ )+except Exception as e:logger.error(f"Completion status update failed: {str(e)}", exc_info=True)@@ -825,4 +964,16 @@ class FileExecutionTasks:api_result=final_result,)+ status = (+ FileExecutionStageStatus.SUCCESS+ if not error+ else FileExecutionStageStatus.FAILED+ )+ cls._mark_file_execution_tracker_completed_status(+ execution_id=str(workflow_execution.id),+ file_execution_id=str(workflow_file_execution.id),+ status=status,+ error=error,+ )+return final_resultdiff --git a/runner/sample.env b/runner/sample.envindex 331e6c514..9a4fd2972 100644--- a/runner/sample.env+++ b/runner/sample.env@@ -54,3 +54,6 @@ TOOL_SIDECAR_ENABLED=FalseTOOL_SIDECAR_IMAGE_NAME="unstract/tool-sidecar"TOOL_SIDECAR_IMAGE_TAG="0.2.0"TOOL_EXECUTION_CACHE_TTL_IN_SECOND=86400 # 24 Hours++# File Execution Tracker+FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hoursdiff --git a/runner/src/unstract/runner/clients/docker_client.py b/runner/src/unstract/runner/clients/docker_client.pyindex 605a6e55f..62dcaaa6d 100644--- a/runner/src/unstract/runner/clients/docker_client.py+++ b/runner/src/unstract/runner/clients/docker_client.py@@ -5,8 +5,9 @@ from collections.abc import Iteratorfrom typing import Anyfrom docker import DockerClient-from docker.errors import APIError, ImageNotFound+from docker.errors import APIError, ImageNotFound, NotFoundfrom docker.models.containers import Container+from unstract.core.runner.enum import ContainerStatusfrom unstract.core.utilities import UnstractUtilsfrom unstract.runner.clients.interface import (ContainerClientInterface,@@ -328,3 +329,61 @@ class Client(ContainerClientInterface):if not container:return Nonereturn container.wait_until_stop(main_container_status)++ def get_container_status(self, container_name: str) -> str:+ """Get status of a Docker container by name.++ Returns one of the defined values in ContainerStatus enum.+ """+ try:+ container = self.client.containers.get(container_name)+ status = container.status.lower()+ except NotFound:+ return ContainerStatus.NOT_FOUND.value+ except Exception as e:+ self.logger.error(f"Error checking status of {container_name}: {e}")+ return ContainerStatus.ERROR.value++ try:+ return ContainerStatus(status.upper()).value+ except ValueError:+ self.logger.warning(+ f"Unexpected container status '{status}' for {container_name}"+ )+ return status++ def remove_container_by_name(+ self, container_name: str, with_sidecar: bool = False, force: bool = True+ ) -> None:+ """Remove a container by its name."""+ try:+ container = self.client.containers.get(container_name)+ container.remove(force=force)+ self.logger.info(f"Container '{container_name}' removed successfully.")+ except NotFound:+ self.logger.warning(f"Container '{container_name}' not found.")+ except APIError as e:+ self.logger.error(+ f"An API error occurred while removing container '{container_name}': {e}"+ )+ except Exception as e:+ self.logger.error(+ f"An unexpected error occurred while removing container '{container_name}': {e}"+ )++ if with_sidecar:+ sidecar_name = Utils.get_sidecar_container_name(container_name)+ try:+ sidecar = self.client.containers.get(sidecar_name)+ sidecar.remove(force=force)+ self.logger.info(f"Sidecar '{sidecar_name}' removed successfully.")+ except NotFound:+ self.logger.warning(f"Sidecar '{sidecar_name}' not found.")+ except APIError as e:+ self.logger.error(+ f"An API error occurred while removing sidecar '{sidecar_name}': {e}"+ )+ except Exception as e:+ self.logger.error(+ f"An unexpected error occurred while removing sidecar '{sidecar_name}': {e}"+ )diff --git a/runner/src/unstract/runner/clients/interface.py b/runner/src/unstract/runner/clients/interface.pyindex 09b6fbbee..da999b78c 100644--- a/runner/src/unstract/runner/clients/interface.py+++ b/runner/src/unstract/runner/clients/interface.py@@ -122,3 +122,24 @@ class ContainerClientInterface(ABC):def cleanup_volume(self) -> None:"""Cleans up the shared volume"""pass++ @abstractmethod+ def get_container_status(self, container_name: str) -> str:+ """Get the status of the container."""+ pass++ @abstractmethod+ def remove_container_by_name(+ self, container_name: str, with_sidecar: bool = False, force: bool = True+ ) -> bool:+ """Remove a container by its name.++ Args:+ container_name (str): Name of the container to remove.+ with_sidecar (bool, optional): Remove sidecar container if running. Defaults to False.+ force (bool, optional): Force removal if running. Defaults to True.++ Returns:+ bool: True if container was removed or not found, False if error.+ """+ passdiff --git a/runner/src/unstract/runner/controller/run.py b/runner/src/unstract/runner/controller/run.pyindex 6917f3133..3ff495a40 100644--- a/runner/src/unstract/runner/controller/run.py+++ b/runner/src/unstract/runner/controller/run.py@@ -38,6 +38,26 @@ def run_container() -> Any | None:return result+@run_bp.route("container/run-status", methods=["GET"])+def run_status() -> Any | None:+ data = request.args+ container_name = data.get("container_name")+ runner = UnstractRunner(None, None, app)+ status = runner.get_container_status(+ container_name=container_name,+ )+ return {"status": status}+++@run_bp.route("container/remove", methods=["POST"])+def remove_container() -> Any | None:+ data = request.get_json()+ container_name = data["container_name"]+ runner = UnstractRunner(None, None, app)+ result = runner.remove_container_by_name(container_name)+ return result++@run_bp.route("container/<command>", methods=["GET"])def run_command(command: str) -> Any | None:"""Endpoint which will can execute any of the below commands.diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.pyindex 82cb4f530..30c2c89f9 100644--- a/runner/src/unstract/runner/runner.py+++ b/runner/src/unstract/runner/runner.py@@ -8,6 +8,12 @@ from dotenv import load_dotenvfrom flask import Flaskfrom unstract.core.constants import LogFieldName+from unstract.core.file_execution_tracker import (+ FileExecutionStage,+ FileExecutionStageData,+ FileExecutionStageStatus,+ FileExecutionStatusTracker,+)from unstract.core.pubsub_helper import LogPublisherfrom unstract.core.tool_execution_status import (ToolExecutionData,@@ -362,6 +368,20 @@ class UnstractRunner:# Delete the status from cache since it is no longer neededtool_execution_tracker.delete_status(tool_execution_data)+ def get_container_status(+ self,+ container_name: str,+ ) -> str:+ """Get container status."""+ return self.client.get_container_status(container_name)++ def remove_container_by_name(self, container_name: str) -> dict[str, Any]:+ """Remove container by name."""+ self.client.remove_container_by_name(+ container_name, with_sidecar=self.sidecar_enabled+ )+ return {"status": "success"}+def run_container(self,organization_id: str,@@ -407,6 +427,22 @@ class UnstractRunner:settings=settings,)+ # Update Execution Tracker status+ file_execution_tracker = FileExecutionStatusTracker()+ file_execution_tracker.update_stage_status(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ stage_status=FileExecutionStageData(+ stage=FileExecutionStage(FileExecutionStage.TOOL_EXECUTION),+ status=FileExecutionStageStatus(FileExecutionStageStatus.IN_PROGRESS),+ ),+ )+ file_execution_tracker.update_tool_container_name(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ tool_container_name=container_name,+ )+container_config = self.client.get_container_run_config(command=["/bin/sh", "-c", container_command],file_execution_id=file_execution_id,@@ -444,35 +480,14 @@ class UnstractRunner:# Run the Docker containercontainer = Nonesidecar = None- result = {"type": "RESULT", "result": None}+ result = {"type": "RESULT", "result": None, "status": "RUNNING"}try:self.logger.info(f"Execution ID: {execution_id}, running docker "f"container: {container_name}")if sidecar_config:- containers: tuple[ContainerInterface, ContainerInterface | None] = (- self.client.run_container_with_sidecar(- container_config, sidecar_config- )- )- container, sidecar = containers- status = self.client.wait_for_container_stop(container)- self.logger.info(- f"Execution ID: {execution_id}, docker "- f"container: {container_name} ran with status: {status}"- )- self.client.wait_for_container_stop(sidecar, main_container_status=status)- self.logger.info(- f"Execution ID: {execution_id}, docker "- f"container: {container_name} completed execution"- )- self._handle_tool_execution_status(- execution_id=execution_id,- file_execution_id=file_execution_id,- container_name=container_name,- )-+ self.client.run_container_with_sidecar(container_config, sidecar_config)else:container: ContainerInterface = self.client.run_container(container_config@@ -495,19 +510,29 @@ class UnstractRunner:f"Execution ID: {execution_id}, docker "f"container: {container_name} ran successfully")-+ result = {"type": "RESULT", "result": None, "status": "SUCCESS"}except ToolRunException as te:self.logger.error(f"Error while running docker container {container_config.get('name')}: {te}",stack_info=True,exc_info=True,)- result = {"type": "RESULT", "result": None, "error": str(te.message)}+ result = {+ "type": "RESULT",+ "result": None,+ "error": str(te.message),+ "status": "ERROR",+ }except Exception as e:self.logger.error(f"Failed to run docker container: {e}", stack_info=True, exc_info=True)- result = {"type": "RESULT", "result": None, "error": str(e)}+ result = {+ "type": "RESULT",+ "result": None,+ "error": str(e),+ "status": "ERROR",+ }if container:container.cleanup(client=self.client)if sidecar:diff --git a/runner/src/unstract/runner/utils.py b/runner/src/unstract/runner/utils.pyindex d97b54edc..4bec83a2c 100644--- a/runner/src/unstract/runner/utils.py+++ b/runner/src/unstract/runner/utils.py@@ -103,3 +103,15 @@ class Utils:"""raw_timeout = os.getenv(Env.TOOL_SIDECAR_CONTAINER_WAIT_TIMEOUT)return Utils.str_to_int(raw_timeout, default=5)++ @staticmethod+ def get_sidecar_container_name(container_name: str) -> str:+ """Get sidecar container name from tool container name.++ Args:+ container_name (str): Container name++ Returns:+ str: Sidecar container name+ """+ return f"{container_name}-sidecar"diff --git a/unstract/core/src/unstract/core/exceptions.py b/unstract/core/src/unstract/core/exceptions.pyindex 19de7768b..a74ac6636 100644--- a/unstract/core/src/unstract/core/exceptions.py+++ b/unstract/core/src/unstract/core/exceptions.py@@ -10,3 +10,24 @@ class ToolExecutionValueException(Exception):def __init__(self, message: str = "Execution ID and file execution ID are required"):super().__init__(message)+++class FileExecutionTrackerValueException(Exception):+ """Raised when required execution identifiers are missing."""++ def __init__(self, message: str = "Execution ID and file execution ID are required"):+ super().__init__(message)+++class FileExecutionTrackerNotFound(Exception):+ """Raised when FileExecutionTracker is not found."""++ def __init__(self, message: str = "FileExecutionTracker not found"):+ super().__init__(message)+++class FileExecutionStageException(Exception):+ """Raised when an invalid file execution stage is provided."""++ def __init__(self, message: str = "Invalid file execution stage"):+ super().__init__(message)diff --git a/unstract/core/src/unstract/core/file_execution_tracker.py b/unstract/core/src/unstract/core/file_execution_tracker.pynew file mode 100644index 000000000..4ccce5f0a--- /dev/null+++ b/unstract/core/src/unstract/core/file_execution_tracker.py@@ -0,0 +1,312 @@+import json+import logging+import os+from dataclasses import asdict, dataclass+from enum import Enum+from typing import Any++import redis++from unstract.core.exceptions import (+ FileExecutionStageException,+ FileExecutionTrackerNotFound,+ FileExecutionTrackerValueException,+)++logger = logging.getLogger(__name__)+++class FileExecutionStage(Enum):+ INITIALIZATION = "INITIALIZATION"+ TOOL_EXECUTION = "TOOL_EXECUTION"+ FINALIZATION = "FINALIZATION"+ COMPLETED = "COMPLETED"++ @property+ def order(self) -> int:+ return FILE_EXECUTION_STAGE_ORDER[self]++ def can_move_to(self, other: "FileExecutionStage") -> bool:+ """Check if the stage can move to the other stage."""+ return self.order < other.order++ def is_before(self, other: "FileExecutionStage") -> bool:+ """Check if the stage is before the other stage."""+ return self.order < other.order++ def is_after(self, other: "FileExecutionStage") -> bool:+ """Check if the stage is after the other stage."""+ return self.order > other.order+++FILE_EXECUTION_STAGE_ORDER = {+ FileExecutionStage.INITIALIZATION: 0,+ FileExecutionStage.TOOL_EXECUTION: 1,+ FileExecutionStage.FINALIZATION: 2,+ FileExecutionStage.COMPLETED: 3,+}+++class FileExecutionStageStatus(Enum):+ IN_PROGRESS = "IN_PROGRESS"+ SUCCESS = "SUCCESS"+ FAILED = "FAILED"+++class FileExecutionField:+ EXECUTION_ID = "execution_id"+ FILE_EXECUTION_ID = "file_execution_id"+ ORGANIZATION_ID = "organization_id"+ STAGE = "stage"+ STATUS = "status"+ ERROR = "error"+ STAGE_STATUS = "stage_status"+ TOOL_CONTAINER_NAME = "tool_container_name"+ STATUS_HISTORY = "status_history"+++@dataclass+class FileExecutionStageData:+ stage: FileExecutionStage+ status: FileExecutionStageStatus+ error: str | None = None++ def __post_init__(self):+ self.validate()++ def validate(self) -> None:+ if not self.stage or not self.status:+ raise FileExecutionTrackerValueException("Stage and status are required")++ def to_dict(self) -> dict[str, Any]:+ return asdict(self)++ def to_serializable(self) -> dict[str, Any]:+ data = {+ FileExecutionField.STAGE: self.stage.value,+ FileExecutionField.STATUS: self.status.value,+ FileExecutionField.ERROR: self.error,+ }+ # Remove any keys with value None (Redis doesn't support None in HSET)+ return {k: v for k, v in data.items() if v is not None}+++@dataclass+class FileExecutionData:+ execution_id: str+ file_execution_id: str+ organization_id: str+ stage_status: FileExecutionStageData+ status_history: list[FileExecutionStageData]+ tool_container_name: str | None = None+ error: str | None = None++ def __post_init__(self):+ self.validate()++ def validate(self) -> None:+ if not self.execution_id or not self.file_execution_id:+ raise FileExecutionTrackerValueException()++ def to_dict(self) -> dict[str, Any]:+ return asdict(self)++ def to_serializable(self) -> dict[str, Any]:+ data = {+ FileExecutionField.EXECUTION_ID: self.execution_id,+ FileExecutionField.FILE_EXECUTION_ID: self.file_execution_id,+ FileExecutionField.ORGANIZATION_ID: self.organization_id,+ FileExecutionField.STAGE_STATUS: json.dumps(+ self.stage_status.to_serializable()+ ),+ FileExecutionField.STATUS_HISTORY: json.dumps(+ [history.to_serializable() for history in self.status_history]+ ),+ FileExecutionField.TOOL_CONTAINER_NAME: self.tool_container_name,+ FileExecutionField.ERROR: self.error,+ }+ # Remove any keys with value None (Redis doesn't support None in HSET)+ return {k: v for k, v in data.items() if v is not None}+++class FileExecutionStatusTracker:+ """File execution status tracker.++ This class is used to track the status of a file execution.+ """++ CACHE_TTL_IN_SECOND = int(+ os.environ.get("FILE_EXECUTION_TRACKER_TTL_IN_SECOND", 60 * 60 * 24)+ )++ def __init__(self):+ self.redis_client = redis.Redis(+ host=os.environ.get("REDIS_HOST"),+ port=int(os.environ.get("REDIS_PORT", 6379)),+ username=os.environ.get("REDIS_USER"),+ password=os.environ.get("REDIS_PASSWORD"),+ decode_responses=True, # ensures hgetall returns str instead of bytes+ )++ def _resolve_field(+ self,+ field_name: str,+ new_data: FileExecutionData,+ existing_data: FileExecutionData | None,+ ) -> str:+ """Resolves the value of a field based on the new data and existing data.++ Args:+ field_name (str): Name of the field+ new_data (ToolExecutionData): New data+ existing_data (ToolExecutionData | None): Existing data++ Returns:+ str: Value of the field+ """+ new_value = getattr(new_data, field_name)+ if not new_value:+ if not existing_data:+ return ""+ new_value = getattr(existing_data, field_name)+ return new_value or ""++ def get_cache_key(self, execution_id: str, file_execution_id: str) -> str:+ return f"file_execution:{execution_id}:{file_execution_id}"++ def set_data(self, data: FileExecutionData, ttl_in_second: int | None = None) -> None:+ data.validate()+ key = self.get_cache_key(data.execution_id, data.file_execution_id)+ logger.info(f"Setting file execution data for {key}: {data.to_serializable()}")+ self.redis_client.hset(key, mapping=data.to_serializable())+ logger.info(+ f"Setting file execution data for {key} to expire in {ttl_in_second} seconds"+ )+ self.redis_client.expire(key, ttl_in_second or self.CACHE_TTL_IN_SECOND)++ def exists(self, execution_id: str, file_execution_id: str) -> bool:+ """Check if file execution tracker data exists."""+ key = self.get_cache_key(execution_id, file_execution_id)+ return self.redis_client.exists(key)++ def update_stage_status(+ self,+ execution_id: str,+ file_execution_id: str,+ stage_status: FileExecutionStageData,+ ttl_in_second: int | None = None,+ ) -> None:+ key = self.get_cache_key(execution_id, file_execution_id)++ # For Existing Execution Data [backward compatibility]+ if not self.redis_client.exists(key):+ self.set_data(+ FileExecutionData(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ organization_id="",+ stage_status=stage_status,+ status_history=[stage_status],+ )+ )++ existing_data = self.get_data(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ )+ logger.info(f"Existing data for {key}: {existing_data}")+ if not existing_data:+ raise FileExecutionTrackerNotFound()++ existing_stage_status = existing_data.stage_status++ if existing_stage_status.stage == stage_status.stage:+ existing_stage_status.status = stage_status.status+ existing_stage_status.error = stage_status.error+ existing_data.stage_status = existing_stage_status++ if existing_stage_status.stage != stage_status.stage:+ # Chekck the stage order+ if not existing_stage_status.stage.can_move_to(stage_status.stage):+ raise FileExecutionStageException(+ "Cannot move to stage: " + stage_status.stage.value+ )++ # Update stage status+ existing_data.stage_status = stage_status++ # Update status history+ existing_data.status_history = [+ existing_stage_status+ ] + existing_data.status_history+ # Update error+ existing_data.error = stage_status.error or existing_data.error+ self.set_data(existing_data, ttl_in_second)++ def update_tool_container_name(+ self, execution_id: str, file_execution_id: str, tool_container_name: str+ ) -> None:+ key = self.get_cache_key(execution_id, file_execution_id)+ with self.redis_client.pipeline() as pipe:+ pipe.hset(key, FileExecutionField.TOOL_CONTAINER_NAME, tool_container_name)+ pipe.expire(key, self.CACHE_TTL_IN_SECOND)+ pipe.execute()++ def update_error(self, execution_id: str, file_execution_id: str, error: str) -> None:+ key = self.get_cache_key(execution_id, file_execution_id)+ with self.redis_client.pipeline() as pipe:+ pipe.hset(key, FileExecutionField.ERROR, error)+ pipe.expire(key, self.CACHE_TTL_IN_SECOND)+ pipe.execute()++ def get_data(+ self, execution_id: str, file_execution_id: str+ ) -> FileExecutionData | None:+ """Get the status of a file execution.++ Args:+ execution_id (str): Execution id of the file execution+ file_execution_id (str): File execution id of the file execution++ Returns:+ FileExecutionData | None: Status of the file execution+ """+ data = self.redis_client.hgetall(+ self.get_cache_key(execution_id, file_execution_id)+ )+ if not data:+ return None++ stage_status_dict = json.loads(data.get(FileExecutionField.STAGE_STATUS, {}))+ status_history_list = json.loads(data.get(FileExecutionField.STATUS_HISTORY, []))+ return FileExecutionData(+ execution_id=execution_id,+ organization_id=data.get(FileExecutionField.ORGANIZATION_ID),+ file_execution_id=file_execution_id,+ stage_status=FileExecutionStageData(+ stage=FileExecutionStage(stage_status_dict.get(FileExecutionField.STAGE)),+ status=FileExecutionStageStatus(+ stage_status_dict.get(FileExecutionField.STATUS)+ ),+ error=stage_status_dict.get(FileExecutionField.ERROR),+ ),+ tool_container_name=data.get(FileExecutionField.TOOL_CONTAINER_NAME),+ status_history=[+ FileExecutionStageData(+ stage=FileExecutionStage(entry.get(FileExecutionField.STAGE)),+ status=FileExecutionStageStatus(entry.get(FileExecutionField.STATUS)),+ error=entry.get(FileExecutionField.ERROR),+ )+ for entry in status_history_list+ ],+ error=data.get(FileExecutionField.ERROR) or None,+ )++ def delete_data(self, execution_id: str, file_execution_id: str) -> None:+ """Delete the status of a file execution.++ Args:+ execution_id (str): Execution id of the file execution+ file_execution_id (str): File execution id of the file execution+ """+ self.redis_client.delete(self.get_cache_key(execution_id, file_execution_id))diff --git a/unstract/core/src/unstract/core/runner/__init__.py b/unstract/core/src/unstract/core/runner/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/unstract/core/src/unstract/core/runner/enum.py b/unstract/core/src/unstract/core/runner/enum.pynew file mode 100644index 000000000..e248c32e2--- /dev/null+++ b/unstract/core/src/unstract/core/runner/enum.py@@ -0,0 +1,15 @@+from enum import Enum+++class ContainerStatus(Enum):+ RUNNING = "RUNNING"+ EXITED = "EXITED"+ CREATED = "CREATED"+ PAUSED = "PAUSED"+ RESTARTING = "RESTARTING"+ REMOVING = "REMOVING"+ DEAD = "DEAD"+ NOT_FOUND = "NOT_FOUND"+ ERROR = "ERROR"+ PENDING = "PENDING"+ UNKNOWN = "UNKNOWN"diff --git a/unstract/core/src/unstract/core/tool_execution_status.py b/unstract/core/src/unstract/core/tool_execution_status.pyindex f923b5300..edd561f3c 100644--- a/unstract/core/src/unstract/core/tool_execution_status.py+++ b/unstract/core/src/unstract/core/tool_execution_status.py@@ -167,5 +167,8 @@ class ToolExecutionTracker:Args:tool_execution_data (ToolExecutionData): Status of the tool execution"""- tool_execution_data.validate()- self.redis_client.delete(self.get_cache_key(tool_execution_data))+ try:+ tool_execution_data.validate()+ self.redis_client.delete(self.get_cache_key(tool_execution_data))+ except ToolExecutionValueException:+ returndiff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.pyindex bd126a5e0..99fb744b3 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py@@ -5,6 +5,8 @@ class UnstractRunner:PROPERTIES_API_ENDPOINT = "/container/properties"ICON_API_ENDPOINT = "/container/icon"VARIABLES_API_ENDPOINT = "/container/variables"+ RUN_STATUS_API_ENDPOINT = "/container/run-status"+ CLEANUP_TOOL_CONTAINER_API_ENDPOINT = "/container/remove"class ToolCommandKey:diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/dto.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/dto.pynew file mode 100644index 000000000..88e81be4a--- /dev/null+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/dto.py@@ -0,0 +1,44 @@+from dataclasses import dataclass+from enum import Enum+from typing import Any+++class RunnerContainerRunStatus(Enum):+ SUCCESS = "SUCCESS"+ ERROR = "ERROR"+ RUNNING = "RUNNING"+++@dataclass+class RunnerContainerRunResponse:+ type: str+ result: dict[str, Any] | None+ error: str | None+ status: RunnerContainerRunStatus++ def to_dict(self) -> dict[str, Any]:+ return {+ "type": self.type,+ "result": self.result,+ "error": self.error,+ "status": self.status.value,+ }++ @classmethod+ def from_dict(cls, data: dict[str, Any]) -> "RunnerContainerRunResponse":+ return cls(+ type=data.get("type"),+ result=data.get("result"),+ error=data.get("error"),+ status=RunnerContainerRunStatus(data.get("status")),+ )+++@dataclass+class RunnerContainerRunStatusResponse:+ status: str++ def to_dict(self) -> dict[str, Any]:+ return {+ "status": self.status,+ }diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/exceptions.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/exceptions.pynew file mode 100644index 000000000..a549c1348--- /dev/null+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/exceptions.py@@ -0,0 +1,4 @@+class ToolExecutionException(Exception):+ def __init__(self, message):+ self.message = message+ super().__init__(self.message)diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.pyindex 7d1f4bc96..bdb22f118 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py@@ -1,17 +1,43 @@import jsonimport loggingimport os+import time+from datetime import UTC, datetime, timedeltafrom typing import Anyimport requestsfrom requests import Responsefrom requests.exceptions import ConnectionError, RequestException+from unstract.core.file_execution_tracker import (+ FileExecutionData,+ FileExecutionStage,+ FileExecutionStageData,+ FileExecutionStageStatus,+ FileExecutionStatusTracker,+)+from unstract.core.runner.enum import ContainerStatus+from unstract.core.tool_execution_status import (+ ToolExecutionData,+ ToolExecutionStatus,+ ToolExecutionTracker,+)from unstract.core.utilities import UnstractUtilsfrom unstract.tool_sandbox.constants import UnstractRunner+from unstract.tool_sandbox.dto import (+ RunnerContainerRunResponse,+ RunnerContainerRunStatus,+)logger = logging.getLogger(__name__)+COMPLETED_FINAL_STATUSES = {+ ContainerStatus.EXITED.value,+ ContainerStatus.DEAD.value,+ ContainerStatus.ERROR.value,+ ContainerStatus.NOT_FOUND.value,+}+class ToolSanboxError(Exception):pass@@ -75,6 +101,64 @@ class ToolSandboxHelper:)return result+ def poll_tool_status(+ self,+ file_execution_id: str,+ file_execution_data: FileExecutionData | None = None,+ ) -> RunnerContainerRunResponse | None:+ if not file_execution_data:+ file_execution_tracker = FileExecutionStatusTracker()+ file_execution_data = file_execution_tracker.get_data(+ execution_id=self.execution_id, file_execution_id=file_execution_id+ )++ # Configurable polling values+ max_wait_seconds = int(os.getenv("MAX_RUNNER_POLLING_WAIT_SECONDS", 60 * 60 * 3))+ interval_seconds = int(os.getenv("RUNNER_POLLING_INTERVAL_SECONDS", 2))+ start_time = datetime.now(UTC)+ end_time = start_time + timedelta(seconds=max_wait_seconds)+ response: RunnerContainerRunResponse | None = None++ while datetime.now(UTC) < end_time:+ status = self._check_tool_run_status(file_execution_data.tool_container_name)+ elapsed = (datetime.now(UTC) - start_time).total_seconds()+ logger.info(+ f"Tool status {status} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id} - elapsed: {elapsed:.2f}s"+ )+ if status and status.get("status") in COMPLETED_FINAL_STATUSES:+ error = self._handle_tool_execution_status(+ execution_id=self.execution_id,+ file_execution_id=file_execution_id,+ container_name=file_execution_data.tool_container_name,+ )+ if error:+ response = self._create_run_response(+ status=RunnerContainerRunStatus.ERROR,+ error=error,+ )+ break+ response = self._create_run_response(+ status=RunnerContainerRunStatus.SUCCESS,+ )+ break+ time.sleep(interval_seconds)++ if not response:+ logger.error(+ f"Tool {file_execution_data.tool_container_name} is not completed within {max_wait_seconds} seconds"+ )+ response = self._create_run_response(+ status=RunnerContainerRunStatus.ERROR,+ error=f"Tool is not completed within {max_wait_seconds} seconds",+ )++ if file_execution_data and file_execution_data.tool_container_name:+ self.cleanup_tool_container(+ container_name=file_execution_data.tool_container_name,+ file_execution_id=file_execution_id,+ )+ return response+def call_tool_handler(self,file_execution_id: str,@@ -82,7 +166,219 @@ class ToolSandboxHelper:image_tag: str,settings: dict[str, Any],retry_count: int | None = None,- ) -> dict[str, Any] | None:+ ) -> RunnerContainerRunResponse | None:+ """Calling unstract runner to run the required tool.++ Args:+ image_name (str): image name+ image_tag (str): image tag+ params (dict[str, Any]): tool params+ settings (dict[str, Any]): tool settings++ Returns:+ Optional[dict[str, Any]]: tool response+ """+ file_execution_tracker = FileExecutionStatusTracker()+ file_execution_data = file_execution_tracker.get_data(+ execution_id=self.execution_id, file_execution_id=file_execution_id+ )++ if not file_execution_data:+ logger.warning(+ f"File execution data not found for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"+ )+ file_execution_tracker.set_data(+ execution_id=self.execution_id,+ file_execution_id=file_execution_id,+ file_execution_data=FileExecutionData(+ execution_id=self.execution_id,+ file_execution_id=file_execution_id,+ organization_id=self.organization_id,+ stage_status=FileExecutionStageData(+ stage=FileExecutionStage.INITIALIZATION,+ status=FileExecutionStageStatus.SUCCESS,+ ),+ status_history=[],+ ),+ )+ response = self._run_and_poll(+ file_execution_id=file_execution_id,+ image_name=image_name,+ image_tag=image_tag,+ settings=settings,+ retry_count=retry_count,+ )++ self._update_stage_status_for_tool_execution(file_execution_id, response)+ self._update_stage_status(+ status=FileExecutionStageStatus.IN_PROGRESS,+ stage=FileExecutionStage.FINALIZATION,+ file_execution_id=file_execution_id,+ )+ else:+ logger.info(+ f"File execution data {file_execution_data} found for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"+ )+ if file_execution_data.error:+ response = self._create_run_response(+ status=RunnerContainerRunStatus.ERROR,+ error=file_execution_data.error,+ )+ return response++ stage = file_execution_data.stage_status.stage+ logger.info(+ f"Current File execution stage {stage.value} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"+ )+ if stage == FileExecutionStage.TOOL_EXECUTION:+ response = self.poll_tool_status(+ file_execution_id=file_execution_id,+ file_execution_data=file_execution_data,+ )+ self._update_stage_status_for_tool_execution(file_execution_id, response)+ self._update_stage_status(+ status=FileExecutionStageStatus.IN_PROGRESS,+ stage=FileExecutionStage.FINALIZATION,+ file_execution_id=file_execution_id,+ )+ elif stage.is_before(FileExecutionStage.TOOL_EXECUTION):+ self._update_stage_status(+ status=FileExecutionStageStatus.SUCCESS,+ stage=stage,+ file_execution_id=file_execution_id,+ )+ response = self._run_and_poll(+ file_execution_id=file_execution_id,+ image_name=image_name,+ image_tag=image_tag,+ settings=settings,+ retry_count=retry_count,+ )+ self._update_stage_status_for_tool_execution(file_execution_id, response)+ self._update_stage_status(+ status=FileExecutionStageStatus.IN_PROGRESS,+ stage=FileExecutionStage.FINALIZATION,+ file_execution_id=file_execution_id,+ )+ else:+ logger.warning(+ f"File execution data stage {file_execution_data.stage_status.stage} is after tool execution for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"+ )+ # Assuming the tool execution is successful+ response = self._create_run_response(+ status=RunnerContainerRunStatus.SUCCESS,+ )+ logger.info(+ f"Tool execution response: {response} for execution_id={self.execution_id}, file_execution_id={file_execution_id}"+ )+ return response++ def _run_and_poll(+ self,+ file_execution_id: str,+ image_name: str,+ image_tag: str,+ settings: dict[str, Any],+ retry_count: int | None = None,+ ) -> RunnerContainerRunResponse:+ logger.info(+ f"Calling runner to run tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"+ )+ response = self.run_tool_container(+ file_execution_id=file_execution_id,+ image_name=image_name,+ image_tag=image_tag,+ settings=settings,+ retry_count=retry_count,+ )+ logger.info(+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"+ )++ if response.status == RunnerContainerRunStatus.RUNNING:+ logger.info(+ f"Polling tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"+ )+ return self.poll_tool_status(file_execution_id)++ logger.info(+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"+ )+ return response++ def cleanup_tool_container(+ self,+ container_name: str,+ file_execution_id: str,+ ) -> None:+ url = f"{self.base_url}{UnstractRunner.CLEANUP_TOOL_CONTAINER_API_ENDPOINT}"+ headers = {+ "X-Request-ID": file_execution_id,+ }+ data = self.cleanup_tool_container_request_data(container_name=container_name)+ response = requests.post(url, headers=headers, json=data)+ if response.status_code != 200:+ logger.error(+ f"Error while calling tool {container_name} reason: {response.reason}"+ )+ return response.json()++ def _update_stage_status(+ self,+ status: FileExecutionStageStatus,+ stage: FileExecutionStage,+ file_execution_id: str,+ ) -> None:+ file_execution_tracker = FileExecutionStatusTracker()++ file_execution_data = file_execution_tracker.get_data(+ execution_id=self.execution_id,+ file_execution_id=file_execution_id,+ )+ if not file_execution_data:+ logger.warning(+ f"File execution data not found for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"+ )+ return+ if file_execution_data.stage_status.stage != stage:+ logger.warning(+ f"File execution data stage {file_execution_data.stage_status.stage} is not {stage} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"+ )+ return++ stage_status = file_execution_data.stage_status+ stage_status.status = status++ file_execution_tracker.update_stage_status(+ execution_id=self.execution_id,+ file_execution_id=file_execution_id,+ stage_status=stage_status,+ )++ def _update_stage_status_for_tool_execution(+ self,+ file_execution_id: str,+ response: RunnerContainerRunResponse,+ ) -> None:+ stage_status = (+ FileExecutionStageStatus.SUCCESS+ if response.status == RunnerContainerRunStatus.SUCCESS+ else FileExecutionStageStatus.FAILED+ )+ self._update_stage_status(+ status=stage_status,+ stage=FileExecutionStage.TOOL_EXECUTION,+ file_execution_id=file_execution_id,+ )++ def run_tool_container(+ self,+ file_execution_id: str,+ image_name: str,+ image_tag: str,+ settings: dict[str, Any],+ retry_count: int | None = None,+ ) -> RunnerContainerRunResponse | None:"""Calling unstract runner to run the required tool.Args:@@ -121,7 +417,7 @@ class ToolSandboxHelper:error_message = response.textlogger.error(f"Error from runner: {error_message}")raise ToolSanboxError(error_message) from e- return response.json()+ return RunnerContainerRunResponse.from_dict(response.json())def create_tool_request_data(self,@@ -150,3 +446,94 @@ class ToolSandboxHelper:"messaging_channel": self.messaging_channel,}return data++ def _create_run_response(+ self,+ status: RunnerContainerRunStatus = RunnerContainerRunStatus.SUCCESS,+ error: str | None = None,+ ) -> RunnerContainerRunResponse:+ return RunnerContainerRunResponse(+ type="RESULT",+ result=None,+ error=error,+ status=status,+ )++ def cleanup_tool_container_request_data(self, container_name: str) -> dict[str, Any]:+ """Creating cleanup tool container request data."""+ data = {"container_name": container_name}+ return data++ def _check_tool_run_status(+ self,+ container_name: str,+ ) -> dict[str, Any] | None:+ params = self._create_tool_run_status_check_request_data(container_name)+ response = requests.get(+ url=f"{self.base_url}{UnstractRunner.RUN_STATUS_API_ENDPOINT}",+ params=params,+ )+ result: dict[str, Any] | None = None+ if response.status_code == 200:+ result = response.json()+ else:+ logger.error(+ f"Error while calling tool {container_name} status code: {response.status_code} reason: {response.reason}"+ )+ return result++ def _create_tool_run_status_check_request_data(+ self,+ container_name: str,+ ) -> dict[str, Any]:+ params = {"container_name": container_name}+ return params++ def _handle_tool_execution_status(+ self, execution_id: str, file_execution_id: str, container_name: str+ ) -> str | None:+ """Get the tool execution status data from the tool execution tracker."""+ error = None+ tool_execution_data = ToolExecutionData(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ )+ tool_execution_tracker = ToolExecutionTracker()+ try:+ tool_execution_status_data = tool_execution_tracker.get_status(+ tool_execution_data+ )+ if not tool_execution_status_data:+ logger.warning(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - failed to fetch execution status"+ )+ return error+ status = tool_execution_status_data.status+ error = tool_execution_status_data.error+ if status == ToolExecutionStatus.FAILED:+ logger.error(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - tool run failed. Error: {error}"+ )+ elif status == ToolExecutionStatus.SUCCESS:+ logger.info(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - tool execution completed successfully"+ )+ else:+ logger.warning(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - unexpected tool status: {status}"+ )+ return error+ except Exception as error:+ logger.error(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - failed to fetch execution status. Error: {error}",+ exc_info=True,+ )+ return str(error)+ finally:+ # Delete the status from cache since it is no longer needed+ tool_execution_tracker.delete_status(tool_execution_data)diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.pyindex 8738e2807..2eee55f61 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py@@ -1,6 +1,7 @@from typing import Anyfrom unstract.tool_sandbox.constants import ToolCommandKey, UnstractRunner+from unstract.tool_sandbox.dto import RunnerContainerRunResponsefrom unstract.tool_sandbox.helper import ToolSandboxHelper@@ -95,7 +96,7 @@ class ToolSandbox:def run_tool(self, file_execution_id: str, retry_count: int | None = None- ) -> dict[str, Any] | None:+ ) -> RunnerContainerRunResponse | None:return self.helper.call_tool_handler( # type: ignorefile_execution_id,self.image_name,diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex feffce305..ef05ffd77 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -7,6 +7,7 @@ from redis import Redisfrom unstract.core.pubsub_helper import LogPublisherfrom unstract.tool_registry import ToolRegistryfrom unstract.tool_sandbox import ToolSandbox+from unstract.tool_sandbox.dto import RunnerContainerRunResponsefrom unstract.workflow_execution.constants import ToolExecutionfrom unstract.workflow_execution.constants import ToolRuntimeVariable as ToolRVfrom unstract.workflow_execution.dto import ToolInstance, WorkflowDto@@ -177,7 +178,7 @@ class ToolsUtils:self,file_execution_id: str,tool_sandbox: ToolSandbox,- ) -> dict[str, Any] | None:+ ) -> RunnerContainerRunResponse | None:return self.run_tool_with_retry(file_execution_id, tool_sandbox)def run_tool_with_retry(@@ -185,7 +186,7 @@ class ToolsUtils:file_execution_id: str,tool_sandbox: ToolSandbox,max_retries: int = ToolExecution.MAXIMUM_RETRY,- ) -> dict[str, Any] | None:+ ) -> RunnerContainerRunResponse | None:for retry_count in range(max_retries):try:response = tool_sandbox.run_tool(file_execution_id, retry_count)diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex acd7a64fe..c0a2d0715 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -212,8 +212,8 @@ class WorkflowExecutionService:result = self.tool_utils.run_tool(file_execution_id=self.file_execution_id, tool_sandbox=sandbox)- if result and result.get("error"):- raise ToolOutputNotFoundException(result.get("error"))+ if result and result.error:+ raise ToolOutputNotFoundException(result.error)if not self.validate_execution_result(step + 1):raise ToolOutputNotFoundException(f"Error running tool '{tool_uid}' for run "