Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Worker check runner container status (#1342)

ali730f79ecb8ae
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 2e3cd9ebc..47aa0680c 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -143,6 +143,13 @@ EXECUTION_RESULT_TTL_SECONDS = int(
EXECUTION_CACHE_TTL_SECONDS = int(
os.environ.get("EXECUTION_CACHE_TTL_SECONDS", 10800)
) # 3 hours
+FILE_EXECUTION_TRACKER_TTL_IN_SECOND = int(
+ os.environ.get("FILE_EXECUTION_TRACKER_TTL_IN_SECOND", 60 * 60 * 5)
+)
+FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND = int(
+ os.environ.get("FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND", 60 * 10)
+) # 10 minutes
+
INSTANT_WF_POLLING_TIMEOUT = int(
os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")
) # 5 minutes
diff --git a/backend/sample.env b/backend/sample.env
index 82f2674e4..bebae5891 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -178,3 +178,11 @@ INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes
# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution
MAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time
+
+# File execution tracker ttl in seconds
+FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hours
+FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600 # 10 minutes
+
+# Runner polling timeout
+MAX_RUNNER_POLLING_WAIT_SECONDS=10800 # 3 hours
+RUNNER_POLLING_INTERVAL_SECONDS=2 # 2 seconds
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index fe496170d..e38c67454 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -3,6 +3,7 @@ from typing import Any
from uuid import UUID
from account_v2.constants import Common
+from django.conf import settings
from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil
from tool_instance_v2.constants import ToolInstanceKey
from tool_instance_v2.models import ToolInstance
@@ -20,6 +21,13 @@ from backend.workers.file_processing_callback.constants import (
from backend.workers.file_processing_callback.file_processing_callback import (
app as file_processing_callback_app,
)
+from unstract.core.file_execution_tracker import (
+ FileExecutionData,
+ FileExecutionStage,
+ FileExecutionStageData,
+ FileExecutionStageStatus,
+ FileExecutionStatusTracker,
+)
from unstract.workflow_execution.enums import LogComponent, LogStage, LogState
from unstract.workflow_execution.exceptions import StopExecution
from workflow_manager.endpoint_v2.destination import DestinationConnector
@@ -350,7 +358,50 @@ class FileExecutionTasks:
destination = DestinationConnector.from_config(
workflow_log, destination_config
)
- destination.delete_file_execution_directory()
+
+ # Set file execution tracker
+ file_execution_data = cls._set_file_execution_tracker(
+ execution_id=str(workflow_execution.id),
+ file_execution_id=str(workflow_file_execution.id),
+ organization_id=str(workflow_execution.workflow.organization_id),
+ )
+
+ # Check if file execution is already in progress
+ stage = file_execution_data.stage_status.stage
+ if stage.is_after(FileExecutionStage.INITIALIZATION):
+ logger.info(
+ f"File already tracked. Resuming execution for execution_id: {workflow_execution.id}, "
+ f"file_execution_id: {workflow_file_execution.id}, current_stage: {stage.value}"
+ )
+ # Not required to prepare file again
+ # Not required to check processing history again
+ if stage.is_before(FileExecutionStage.COMPLETED):
+ # Core Execution Phase
+ execution_result = cls._execute_workflow_steps(
+ file_data,
+ workflow_execution,
+ workflow_file_execution,
+ file_hash,
+ current_file_idx,
+ total_files,
+ )
+
+ # Finalization Phase
+ return cls._finalize_execution(
+ workflow_execution,
+ workflow_file_execution,
+ file_hash,
+ workflow_log,
+ destination,
+ execution_result,
+ )
+ # If stage is already completed, skip execution
+ return cls._build_final_result(
+ workflow_execution,
+ file_hash,
+ FinalOutputResult.skipped(),
+ workflow_file_execution,
+ )
# File Preparation Phase
content_hash = cls._prepare_file_for_processing(
@@ -432,6 +483,82 @@ class FileExecutionTasks:
is_api=destination.is_api,
)
+ @classmethod
+ def _set_file_execution_tracker(
+ cls,
+ execution_id: str,
+ file_execution_id: str,
+ organization_id: str,
+ ) -> FileExecutionData:
+ # Initialize file execution tracker
+ file_execution_tracker = FileExecutionStatusTracker()
+ file_execution_stage_data = FileExecutionStageData(
+ stage=FileExecutionStage.INITIALIZATION,
+ status=FileExecutionStageStatus.IN_PROGRESS,
+ )
+ file_execution_data = FileExecutionData(
+ execution_id=str(execution_id),
+ file_execution_id=str(file_execution_id),
+ organization_id=str(organization_id),
+ stage_status=file_execution_stage_data,
+ status_history=[],
+ )
+ if not file_execution_tracker.exists(execution_id, file_execution_id):
+ file_execution_tracker.set_data(file_execution_data)
+ else:
+ file_execution_data = file_execution_tracker.get_data(
+ execution_id, file_execution_id
+ )
+ logger.info(
+ f"File execution tracker already exists for execution_id: {execution_id}, file_execution_id: {file_execution_id}"
+ )
+ return file_execution_data
+
+ @classmethod
+ def _mark_file_execution_tracker_finalization_status(
+ cls,
+ execution_id: str,
+ file_execution_id: str,
+ status: FileExecutionStageStatus,
+ error: str | None = None,
+ ) -> None:
+ file_execution_tracker = FileExecutionStatusTracker()
+ stage_data = FileExecutionStageData(
+ stage=FileExecutionStage.FINALIZATION,
+ status=status,
+ error=error,
+ )
+ file_execution_tracker.update_stage_status(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ stage_status=stage_data,
+ )
+
+ @classmethod
+ def _mark_file_execution_tracker_completed_status(
+ cls,
+ execution_id: str,
+ file_execution_id: str,
+ status: FileExecutionStageStatus,
+ error: str | None = None,
+ ) -> None:
+ # This is the cache ttl for completed file execution to prevent long term cache
+ COMPLETED_EXECUTION_CACHE_TTL_IN_SECOND = (
+ settings.FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND
+ )
+ file_execution_tracker = FileExecutionStatusTracker()
+ stage_data = FileExecutionStageData(
+ stage=FileExecutionStage.COMPLETED,
+ status=status,
+ error=error,
+ )
+ file_execution_tracker.update_stage_status(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ stage_status=stage_data,
+ ttl_in_second=COMPLETED_EXECUTION_CACHE_TTL_IN_SECOND,
+ )
+
@classmethod
def _initialize_execution_context(
cls,
@@ -669,7 +796,7 @@ class FileExecutionTasks:
workflow_log=workflow_log,
error=result.error or execution_result.error,
)
-
+ destination.delete_file_execution_directory()
return cls._build_final_result(
workflow_execution=workflow_execution,
file_hash=file_hash,
@@ -795,6 +922,18 @@ class FileExecutionTasks:
component=LogComponent.DESTINATION,
)
+ file_execution_tracker_status = (
+ FileExecutionStageStatus.SUCCESS
+ if not error
+ else FileExecutionStageStatus.FAILED
+ )
+ cls._mark_file_execution_tracker_finalization_status(
+ execution_id=str(workflow_file_execution.workflow_execution.id),
+ file_execution_id=str(workflow_file_execution.id),
+ status=file_execution_tracker_status,
+ error=error,
+ )
+
except Exception as e:
logger.error(f"Completion status update failed: {str(e)}", exc_info=True)
@@ -825,4 +964,16 @@ class FileExecutionTasks:
api_result=final_result,
)
+ status = (
+ FileExecutionStageStatus.SUCCESS
+ if not error
+ else FileExecutionStageStatus.FAILED
+ )
+ cls._mark_file_execution_tracker_completed_status(
+ execution_id=str(workflow_execution.id),
+ file_execution_id=str(workflow_file_execution.id),
+ status=status,
+ error=error,
+ )
+
return final_result
diff --git a/runner/sample.env b/runner/sample.env
index 331e6c514..9a4fd2972 100644
--- a/runner/sample.env
+++ b/runner/sample.env
@@ -54,3 +54,6 @@ TOOL_SIDECAR_ENABLED=False
TOOL_SIDECAR_IMAGE_NAME="unstract/tool-sidecar"
TOOL_SIDECAR_IMAGE_TAG="0.2.0"
TOOL_EXECUTION_CACHE_TTL_IN_SECOND=86400 # 24 Hours
+
+# File Execution Tracker
+FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hours
diff --git a/runner/src/unstract/runner/clients/docker_client.py b/runner/src/unstract/runner/clients/docker_client.py
index 605a6e55f..62dcaaa6d 100644
--- a/runner/src/unstract/runner/clients/docker_client.py
+++ b/runner/src/unstract/runner/clients/docker_client.py
@@ -5,8 +5,9 @@ from collections.abc import Iterator
from typing import Any
from docker import DockerClient
-from docker.errors import APIError, ImageNotFound
+from docker.errors import APIError, ImageNotFound, NotFound
from docker.models.containers import Container
+from unstract.core.runner.enum import ContainerStatus
from unstract.core.utilities import UnstractUtils
from unstract.runner.clients.interface import (
ContainerClientInterface,
@@ -328,3 +329,61 @@ class Client(ContainerClientInterface):
if not container:
return None
return container.wait_until_stop(main_container_status)
+
+ def get_container_status(self, container_name: str) -> str:
+ """Get status of a Docker container by name.
+
+ Returns one of the defined values in ContainerStatus enum.
+ """
+ try:
+ container = self.client.containers.get(container_name)
+ status = container.status.lower()
+ except NotFound:
+ return ContainerStatus.NOT_FOUND.value
+ except Exception as e:
+ self.logger.error(f"Error checking status of {container_name}: {e}")
+ return ContainerStatus.ERROR.value
+
+ try:
+ return ContainerStatus(status.upper()).value
+ except ValueError:
+ self.logger.warning(
+ f"Unexpected container status '{status}' for {container_name}"
+ )
+ return status
+
+ def remove_container_by_name(
+ self, container_name: str, with_sidecar: bool = False, force: bool = True
+ ) -> None:
+ """Remove a container by its name."""
+ try:
+ container = self.client.containers.get(container_name)
+ container.remove(force=force)
+ self.logger.info(f"Container '{container_name}' removed successfully.")
+ except NotFound:
+ self.logger.warning(f"Container '{container_name}' not found.")
+ except APIError as e:
+ self.logger.error(
+ f"An API error occurred while removing container '{container_name}': {e}"
+ )
+ except Exception as e:
+ self.logger.error(
+ f"An unexpected error occurred while removing container '{container_name}': {e}"
+ )
+
+ if with_sidecar:
+ sidecar_name = Utils.get_sidecar_container_name(container_name)
+ try:
+ sidecar = self.client.containers.get(sidecar_name)
+ sidecar.remove(force=force)
+ self.logger.info(f"Sidecar '{sidecar_name}' removed successfully.")
+ except NotFound:
+ self.logger.warning(f"Sidecar '{sidecar_name}' not found.")
+ except APIError as e:
+ self.logger.error(
+ f"An API error occurred while removing sidecar '{sidecar_name}': {e}"
+ )
+ except Exception as e:
+ self.logger.error(
+ f"An unexpected error occurred while removing sidecar '{sidecar_name}': {e}"
+ )
diff --git a/runner/src/unstract/runner/clients/interface.py b/runner/src/unstract/runner/clients/interface.py
index 09b6fbbee..da999b78c 100644
--- a/runner/src/unstract/runner/clients/interface.py
+++ b/runner/src/unstract/runner/clients/interface.py
@@ -122,3 +122,24 @@ class ContainerClientInterface(ABC):
def cleanup_volume(self) -> None:
"""Cleans up the shared volume"""
pass
+
+ @abstractmethod
+ def get_container_status(self, container_name: str) -> str:
+ """Get the status of the container."""
+ pass
+
+ @abstractmethod
+ def remove_container_by_name(
+ self, container_name: str, with_sidecar: bool = False, force: bool = True
+ ) -> bool:
+ """Remove a container by its name.
+
+ Args:
+ container_name (str): Name of the container to remove.
+ with_sidecar (bool, optional): Remove sidecar container if running. Defaults to False.
+ force (bool, optional): Force removal if running. Defaults to True.
+
+ Returns:
+ bool: True if container was removed or not found, False if error.
+ """
+ pass
diff --git a/runner/src/unstract/runner/controller/run.py b/runner/src/unstract/runner/controller/run.py
index 6917f3133..3ff495a40 100644
--- a/runner/src/unstract/runner/controller/run.py
+++ b/runner/src/unstract/runner/controller/run.py
@@ -38,6 +38,26 @@ def run_container() -> Any | None:
return result
+@run_bp.route("container/run-status", methods=["GET"])
+def run_status() -> Any | None:
+ data = request.args
+ container_name = data.get("container_name")
+ runner = UnstractRunner(None, None, app)
+ status = runner.get_container_status(
+ container_name=container_name,
+ )
+ return {"status": status}
+
+
+@run_bp.route("container/remove", methods=["POST"])
+def remove_container() -> Any | None:
+ data = request.get_json()
+ container_name = data["container_name"]
+ runner = UnstractRunner(None, None, app)
+ result = runner.remove_container_by_name(container_name)
+ return result
+
+
@run_bp.route("container/<command>", methods=["GET"])
def run_command(command: str) -> Any | None:
"""Endpoint which will can execute any of the below commands.
diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.py
index 82cb4f530..30c2c89f9 100644
--- a/runner/src/unstract/runner/runner.py
+++ b/runner/src/unstract/runner/runner.py
@@ -8,6 +8,12 @@ from dotenv import load_dotenv
from flask import Flask
from unstract.core.constants import LogFieldName
+from unstract.core.file_execution_tracker import (
+ FileExecutionStage,
+ FileExecutionStageData,
+ FileExecutionStageStatus,
+ FileExecutionStatusTracker,
+)
from unstract.core.pubsub_helper import LogPublisher
from unstract.core.tool_execution_status import (
ToolExecutionData,
@@ -362,6 +368,20 @@ class UnstractRunner:
# Delete the status from cache since it is no longer needed
tool_execution_tracker.delete_status(tool_execution_data)
+ def get_container_status(
+ self,
+ container_name: str,
+ ) -> str:
+ """Get container status."""
+ return self.client.get_container_status(container_name)
+
+ def remove_container_by_name(self, container_name: str) -> dict[str, Any]:
+ """Remove container by name."""
+ self.client.remove_container_by_name(
+ container_name, with_sidecar=self.sidecar_enabled
+ )
+ return {"status": "success"}
+
def run_container(
self,
organization_id: str,
@@ -407,6 +427,22 @@ class UnstractRunner:
settings=settings,
)
+ # Update Execution Tracker status
+ file_execution_tracker = FileExecutionStatusTracker()
+ file_execution_tracker.update_stage_status(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ stage_status=FileExecutionStageData(
+ stage=FileExecutionStage(FileExecutionStage.TOOL_EXECUTION),
+ status=FileExecutionStageStatus(FileExecutionStageStatus.IN_PROGRESS),
+ ),
+ )
+ file_execution_tracker.update_tool_container_name(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ tool_container_name=container_name,
+ )
+
container_config = self.client.get_container_run_config(
command=["/bin/sh", "-c", container_command],
file_execution_id=file_execution_id,
@@ -444,35 +480,14 @@ class UnstractRunner:
# Run the Docker container
container = None
sidecar = None
- result = {"type": "RESULT", "result": None}
+ result = {"type": "RESULT", "result": None, "status": "RUNNING"}
try:
self.logger.info(
f"Execution ID: {execution_id}, running docker "
f"container: {container_name}"
)
if sidecar_config:
- containers: tuple[ContainerInterface, ContainerInterface | None] = (
- self.client.run_container_with_sidecar(
- container_config, sidecar_config
- )
- )
- container, sidecar = containers
- status = self.client.wait_for_container_stop(container)
- self.logger.info(
- f"Execution ID: {execution_id}, docker "
- f"container: {container_name} ran with status: {status}"
- )
- self.client.wait_for_container_stop(sidecar, main_container_status=status)
- self.logger.info(
- f"Execution ID: {execution_id}, docker "
- f"container: {container_name} completed execution"
- )
- self._handle_tool_execution_status(
- execution_id=execution_id,
- file_execution_id=file_execution_id,
- container_name=container_name,
- )
-
+ self.client.run_container_with_sidecar(container_config, sidecar_config)
else:
container: ContainerInterface = self.client.run_container(
container_config
@@ -495,19 +510,29 @@ class UnstractRunner:
f"Execution ID: {execution_id}, docker "
f"container: {container_name} ran successfully"
)
-
+ result = {"type": "RESULT", "result": None, "status": "SUCCESS"}
except ToolRunException as te:
self.logger.error(
f"Error while running docker container {container_config.get('name')}: {te}",
stack_info=True,
exc_info=True,
)
- result = {"type": "RESULT", "result": None, "error": str(te.message)}
+ result = {
+ "type": "RESULT",
+ "result": None,
+ "error": str(te.message),
+ "status": "ERROR",
+ }
except Exception as e:
self.logger.error(
f"Failed to run docker container: {e}", stack_info=True, exc_info=True
)
- result = {"type": "RESULT", "result": None, "error": str(e)}
+ result = {
+ "type": "RESULT",
+ "result": None,
+ "error": str(e),
+ "status": "ERROR",
+ }
if container:
container.cleanup(client=self.client)
if sidecar:
diff --git a/runner/src/unstract/runner/utils.py b/runner/src/unstract/runner/utils.py
index d97b54edc..4bec83a2c 100644
--- a/runner/src/unstract/runner/utils.py
+++ b/runner/src/unstract/runner/utils.py
@@ -103,3 +103,15 @@ class Utils:
"""
raw_timeout = os.getenv(Env.TOOL_SIDECAR_CONTAINER_WAIT_TIMEOUT)
return Utils.str_to_int(raw_timeout, default=5)
+
+ @staticmethod
+ def get_sidecar_container_name(container_name: str) -> str:
+ """Get sidecar container name from tool container name.
+
+ Args:
+ container_name (str): Container name
+
+ Returns:
+ str: Sidecar container name
+ """
+ return f"{container_name}-sidecar"
diff --git a/unstract/core/src/unstract/core/exceptions.py b/unstract/core/src/unstract/core/exceptions.py
index 19de7768b..a74ac6636 100644
--- a/unstract/core/src/unstract/core/exceptions.py
+++ b/unstract/core/src/unstract/core/exceptions.py
@@ -10,3 +10,24 @@ class ToolExecutionValueException(Exception):
def __init__(self, message: str = "Execution ID and file execution ID are required"):
super().__init__(message)
+
+
+class FileExecutionTrackerValueException(Exception):
+ """Raised when required execution identifiers are missing."""
+
+ def __init__(self, message: str = "Execution ID and file execution ID are required"):
+ super().__init__(message)
+
+
+class FileExecutionTrackerNotFound(Exception):
+ """Raised when FileExecutionTracker is not found."""
+
+ def __init__(self, message: str = "FileExecutionTracker not found"):
+ super().__init__(message)
+
+
+class FileExecutionStageException(Exception):
+ """Raised when an invalid file execution stage is provided."""
+
+ def __init__(self, message: str = "Invalid file execution stage"):
+ super().__init__(message)
diff --git a/unstract/core/src/unstract/core/file_execution_tracker.py b/unstract/core/src/unstract/core/file_execution_tracker.py
new file mode 100644
index 000000000..4ccce5f0a
--- /dev/null
+++ b/unstract/core/src/unstract/core/file_execution_tracker.py
@@ -0,0 +1,312 @@
+import json
+import logging
+import os
+from dataclasses import asdict, dataclass
+from enum import Enum
+from typing import Any
+
+import redis
+
+from unstract.core.exceptions import (
+ FileExecutionStageException,
+ FileExecutionTrackerNotFound,
+ FileExecutionTrackerValueException,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class FileExecutionStage(Enum):
+ INITIALIZATION = "INITIALIZATION"
+ TOOL_EXECUTION = "TOOL_EXECUTION"
+ FINALIZATION = "FINALIZATION"
+ COMPLETED = "COMPLETED"
+
+ @property
+ def order(self) -> int:
+ return FILE_EXECUTION_STAGE_ORDER[self]
+
+ def can_move_to(self, other: "FileExecutionStage") -> bool:
+ """Check if the stage can move to the other stage."""
+ return self.order < other.order
+
+ def is_before(self, other: "FileExecutionStage") -> bool:
+ """Check if the stage is before the other stage."""
+ return self.order < other.order
+
+ def is_after(self, other: "FileExecutionStage") -> bool:
+ """Check if the stage is after the other stage."""
+ return self.order > other.order
+
+
+FILE_EXECUTION_STAGE_ORDER = {
+ FileExecutionStage.INITIALIZATION: 0,
+ FileExecutionStage.TOOL_EXECUTION: 1,
+ FileExecutionStage.FINALIZATION: 2,
+ FileExecutionStage.COMPLETED: 3,
+}
+
+
+class FileExecutionStageStatus(Enum):
+ IN_PROGRESS = "IN_PROGRESS"
+ SUCCESS = "SUCCESS"
+ FAILED = "FAILED"
+
+
+class FileExecutionField:
+ EXECUTION_ID = "execution_id"
+ FILE_EXECUTION_ID = "file_execution_id"
+ ORGANIZATION_ID = "organization_id"
+ STAGE = "stage"
+ STATUS = "status"
+ ERROR = "error"
+ STAGE_STATUS = "stage_status"
+ TOOL_CONTAINER_NAME = "tool_container_name"
+ STATUS_HISTORY = "status_history"
+
+
+@dataclass
+class FileExecutionStageData:
+ stage: FileExecutionStage
+ status: FileExecutionStageStatus
+ error: str | None = None
+
+ def __post_init__(self):
+ self.validate()
+
+ def validate(self) -> None:
+ if not self.stage or not self.status:
+ raise FileExecutionTrackerValueException("Stage and status are required")
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+ def to_serializable(self) -> dict[str, Any]:
+ data = {
+ FileExecutionField.STAGE: self.stage.value,
+ FileExecutionField.STATUS: self.status.value,
+ FileExecutionField.ERROR: self.error,
+ }
+ # Remove any keys with value None (Redis doesn't support None in HSET)
+ return {k: v for k, v in data.items() if v is not None}
+
+
+@dataclass
+class FileExecutionData:
+ execution_id: str
+ file_execution_id: str
+ organization_id: str
+ stage_status: FileExecutionStageData
+ status_history: list[FileExecutionStageData]
+ tool_container_name: str | None = None
+ error: str | None = None
+
+ def __post_init__(self):
+ self.validate()
+
+ def validate(self) -> None:
+ if not self.execution_id or not self.file_execution_id:
+ raise FileExecutionTrackerValueException()
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+ def to_serializable(self) -> dict[str, Any]:
+ data = {
+ FileExecutionField.EXECUTION_ID: self.execution_id,
+ FileExecutionField.FILE_EXECUTION_ID: self.file_execution_id,
+ FileExecutionField.ORGANIZATION_ID: self.organization_id,
+ FileExecutionField.STAGE_STATUS: json.dumps(
+ self.stage_status.to_serializable()
+ ),
+ FileExecutionField.STATUS_HISTORY: json.dumps(
+ [history.to_serializable() for history in self.status_history]
+ ),
+ FileExecutionField.TOOL_CONTAINER_NAME: self.tool_container_name,
+ FileExecutionField.ERROR: self.error,
+ }
+ # Remove any keys with value None (Redis doesn't support None in HSET)
+ return {k: v for k, v in data.items() if v is not None}
+
+
+class FileExecutionStatusTracker:
+ """File execution status tracker.
+
+ This class is used to track the status of a file execution.
+ """
+
+ CACHE_TTL_IN_SECOND = int(
+ os.environ.get("FILE_EXECUTION_TRACKER_TTL_IN_SECOND", 60 * 60 * 24)
+ )
+
+ def __init__(self):
+ self.redis_client = redis.Redis(
+ host=os.environ.get("REDIS_HOST"),
+ port=int(os.environ.get("REDIS_PORT", 6379)),
+ username=os.environ.get("REDIS_USER"),
+ password=os.environ.get("REDIS_PASSWORD"),
+ decode_responses=True, # ensures hgetall returns str instead of bytes
+ )
+
+ def _resolve_field(
+ self,
+ field_name: str,
+ new_data: FileExecutionData,
+ existing_data: FileExecutionData | None,
+ ) -> str:
+ """Resolves the value of a field based on the new data and existing data.
+
+ Args:
+ field_name (str): Name of the field
+ new_data (ToolExecutionData): New data
+ existing_data (ToolExecutionData | None): Existing data
+
+ Returns:
+ str: Value of the field
+ """
+ new_value = getattr(new_data, field_name)
+ if not new_value:
+ if not existing_data:
+ return ""
+ new_value = getattr(existing_data, field_name)
+ return new_value or ""
+
+ def get_cache_key(self, execution_id: str, file_execution_id: str) -> str:
+ return f"file_execution:{execution_id}:{file_execution_id}"
+
+ def set_data(self, data: FileExecutionData, ttl_in_second: int | None = None) -> None:
+ data.validate()
+ key = self.get_cache_key(data.execution_id, data.file_execution_id)
+ logger.info(f"Setting file execution data for {key}: {data.to_serializable()}")
+ self.redis_client.hset(key, mapping=data.to_serializable())
+ logger.info(
+ f"Setting file execution data for {key} to expire in {ttl_in_second} seconds"
+ )
+ self.redis_client.expire(key, ttl_in_second or self.CACHE_TTL_IN_SECOND)
+
+ def exists(self, execution_id: str, file_execution_id: str) -> bool:
+ """Check if file execution tracker data exists."""
+ key = self.get_cache_key(execution_id, file_execution_id)
+ return self.redis_client.exists(key)
+
+ def update_stage_status(
+ self,
+ execution_id: str,
+ file_execution_id: str,
+ stage_status: FileExecutionStageData,
+ ttl_in_second: int | None = None,
+ ) -> None:
+ key = self.get_cache_key(execution_id, file_execution_id)
+
+ # For Existing Execution Data [backward compatibility]
+ if not self.redis_client.exists(key):
+ self.set_data(
+ FileExecutionData(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ organization_id="",
+ stage_status=stage_status,
+ status_history=[stage_status],
+ )
+ )
+
+ existing_data = self.get_data(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ )
+ logger.info(f"Existing data for {key}: {existing_data}")
+ if not existing_data:
+ raise FileExecutionTrackerNotFound()
+
+ existing_stage_status = existing_data.stage_status
+
+ if existing_stage_status.stage == stage_status.stage:
+ existing_stage_status.status = stage_status.status
+ existing_stage_status.error = stage_status.error
+ existing_data.stage_status = existing_stage_status
+
+ if existing_stage_status.stage != stage_status.stage:
+ # Chekck the stage order
+ if not existing_stage_status.stage.can_move_to(stage_status.stage):
+ raise FileExecutionStageException(
+ "Cannot move to stage: " + stage_status.stage.value
+ )
+
+ # Update stage status
+ existing_data.stage_status = stage_status
+
+ # Update status history
+ existing_data.status_history = [
+ existing_stage_status
+ ] + existing_data.status_history
+ # Update error
+ existing_data.error = stage_status.error or existing_data.error
+ self.set_data(existing_data, ttl_in_second)
+
+ def update_tool_container_name(
+ self, execution_id: str, file_execution_id: str, tool_container_name: str
+ ) -> None:
+ key = self.get_cache_key(execution_id, file_execution_id)
+ with self.redis_client.pipeline() as pipe:
+ pipe.hset(key, FileExecutionField.TOOL_CONTAINER_NAME, tool_container_name)
+ pipe.expire(key, self.CACHE_TTL_IN_SECOND)
+ pipe.execute()
+
+ def update_error(self, execution_id: str, file_execution_id: str, error: str) -> None:
+ key = self.get_cache_key(execution_id, file_execution_id)
+ with self.redis_client.pipeline() as pipe:
+ pipe.hset(key, FileExecutionField.ERROR, error)
+ pipe.expire(key, self.CACHE_TTL_IN_SECOND)
+ pipe.execute()
+
+ def get_data(
+ self, execution_id: str, file_execution_id: str
+ ) -> FileExecutionData | None:
+ """Get the status of a file execution.
+
+ Args:
+ execution_id (str): Execution id of the file execution
+ file_execution_id (str): File execution id of the file execution
+
+ Returns:
+ FileExecutionData | None: Status of the file execution
+ """
+ data = self.redis_client.hgetall(
+ self.get_cache_key(execution_id, file_execution_id)
+ )
+ if not data:
+ return None
+
+ stage_status_dict = json.loads(data.get(FileExecutionField.STAGE_STATUS, {}))
+ status_history_list = json.loads(data.get(FileExecutionField.STATUS_HISTORY, []))
+ return FileExecutionData(
+ execution_id=execution_id,
+ organization_id=data.get(FileExecutionField.ORGANIZATION_ID),
+ file_execution_id=file_execution_id,
+ stage_status=FileExecutionStageData(
+ stage=FileExecutionStage(stage_status_dict.get(FileExecutionField.STAGE)),
+ status=FileExecutionStageStatus(
+ stage_status_dict.get(FileExecutionField.STATUS)
+ ),
+ error=stage_status_dict.get(FileExecutionField.ERROR),
+ ),
+ tool_container_name=data.get(FileExecutionField.TOOL_CONTAINER_NAME),
+ status_history=[
+ FileExecutionStageData(
+ stage=FileExecutionStage(entry.get(FileExecutionField.STAGE)),
+ status=FileExecutionStageStatus(entry.get(FileExecutionField.STATUS)),
+ error=entry.get(FileExecutionField.ERROR),
+ )
+ for entry in status_history_list
+ ],
+ error=data.get(FileExecutionField.ERROR) or None,
+ )
+
+ def delete_data(self, execution_id: str, file_execution_id: str) -> None:
+ """Delete the status of a file execution.
+
+ Args:
+ execution_id (str): Execution id of the file execution
+ file_execution_id (str): File execution id of the file execution
+ """
+ self.redis_client.delete(self.get_cache_key(execution_id, file_execution_id))
diff --git a/unstract/core/src/unstract/core/runner/__init__.py b/unstract/core/src/unstract/core/runner/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/unstract/core/src/unstract/core/runner/enum.py b/unstract/core/src/unstract/core/runner/enum.py
new file mode 100644
index 000000000..e248c32e2
--- /dev/null
+++ b/unstract/core/src/unstract/core/runner/enum.py
@@ -0,0 +1,15 @@
+from enum import Enum
+
+
+class ContainerStatus(Enum):
+ RUNNING = "RUNNING"
+ EXITED = "EXITED"
+ CREATED = "CREATED"
+ PAUSED = "PAUSED"
+ RESTARTING = "RESTARTING"
+ REMOVING = "REMOVING"
+ DEAD = "DEAD"
+ NOT_FOUND = "NOT_FOUND"
+ ERROR = "ERROR"
+ PENDING = "PENDING"
+ UNKNOWN = "UNKNOWN"
diff --git a/unstract/core/src/unstract/core/tool_execution_status.py b/unstract/core/src/unstract/core/tool_execution_status.py
index f923b5300..edd561f3c 100644
--- a/unstract/core/src/unstract/core/tool_execution_status.py
+++ b/unstract/core/src/unstract/core/tool_execution_status.py
@@ -167,5 +167,8 @@ class ToolExecutionTracker:
Args:
tool_execution_data (ToolExecutionData): Status of the tool execution
"""
- tool_execution_data.validate()
- self.redis_client.delete(self.get_cache_key(tool_execution_data))
+ try:
+ tool_execution_data.validate()
+ self.redis_client.delete(self.get_cache_key(tool_execution_data))
+ except ToolExecutionValueException:
+ return
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
index bd126a5e0..99fb744b3 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
@@ -5,6 +5,8 @@ class UnstractRunner:
PROPERTIES_API_ENDPOINT = "/container/properties"
ICON_API_ENDPOINT = "/container/icon"
VARIABLES_API_ENDPOINT = "/container/variables"
+ RUN_STATUS_API_ENDPOINT = "/container/run-status"
+ CLEANUP_TOOL_CONTAINER_API_ENDPOINT = "/container/remove"
class ToolCommandKey:
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/dto.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/dto.py
new file mode 100644
index 000000000..88e81be4a
--- /dev/null
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/dto.py
@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class RunnerContainerRunStatus(Enum):
+ SUCCESS = "SUCCESS"
+ ERROR = "ERROR"
+ RUNNING = "RUNNING"
+
+
+@dataclass
+class RunnerContainerRunResponse:
+ type: str
+ result: dict[str, Any] | None
+ error: str | None
+ status: RunnerContainerRunStatus
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "type": self.type,
+ "result": self.result,
+ "error": self.error,
+ "status": self.status.value,
+ }
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> "RunnerContainerRunResponse":
+ return cls(
+ type=data.get("type"),
+ result=data.get("result"),
+ error=data.get("error"),
+ status=RunnerContainerRunStatus(data.get("status")),
+ )
+
+
+@dataclass
+class RunnerContainerRunStatusResponse:
+ status: str
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "status": self.status,
+ }
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/exceptions.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/exceptions.py
new file mode 100644
index 000000000..a549c1348
--- /dev/null
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/exceptions.py
@@ -0,0 +1,4 @@
+class ToolExecutionException(Exception):
+ def __init__(self, message):
+ self.message = message
+ super().__init__(self.message)
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
index 7d1f4bc96..bdb22f118 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
@@ -1,17 +1,43 @@
import json
import logging
import os
+import time
+from datetime import UTC, datetime, timedelta
from typing import Any
import requests
from requests import Response
from requests.exceptions import ConnectionError, RequestException
+from unstract.core.file_execution_tracker import (
+ FileExecutionData,
+ FileExecutionStage,
+ FileExecutionStageData,
+ FileExecutionStageStatus,
+ FileExecutionStatusTracker,
+)
+from unstract.core.runner.enum import ContainerStatus
+from unstract.core.tool_execution_status import (
+ ToolExecutionData,
+ ToolExecutionStatus,
+ ToolExecutionTracker,
+)
from unstract.core.utilities import UnstractUtils
from unstract.tool_sandbox.constants import UnstractRunner
+from unstract.tool_sandbox.dto import (
+ RunnerContainerRunResponse,
+ RunnerContainerRunStatus,
+)
logger = logging.getLogger(__name__)
+COMPLETED_FINAL_STATUSES = {
+ ContainerStatus.EXITED.value,
+ ContainerStatus.DEAD.value,
+ ContainerStatus.ERROR.value,
+ ContainerStatus.NOT_FOUND.value,
+}
+
class ToolSanboxError(Exception):
pass
@@ -75,6 +101,64 @@ class ToolSandboxHelper:
)
return result
+ def poll_tool_status(
+ self,
+ file_execution_id: str,
+ file_execution_data: FileExecutionData | None = None,
+ ) -> RunnerContainerRunResponse | None:
+ if not file_execution_data:
+ file_execution_tracker = FileExecutionStatusTracker()
+ file_execution_data = file_execution_tracker.get_data(
+ execution_id=self.execution_id, file_execution_id=file_execution_id
+ )
+
+ # Configurable polling values
+ max_wait_seconds = int(os.getenv("MAX_RUNNER_POLLING_WAIT_SECONDS", 60 * 60 * 3))
+ interval_seconds = int(os.getenv("RUNNER_POLLING_INTERVAL_SECONDS", 2))
+ start_time = datetime.now(UTC)
+ end_time = start_time + timedelta(seconds=max_wait_seconds)
+ response: RunnerContainerRunResponse | None = None
+
+ while datetime.now(UTC) < end_time:
+ status = self._check_tool_run_status(file_execution_data.tool_container_name)
+ elapsed = (datetime.now(UTC) - start_time).total_seconds()
+ logger.info(
+ f"Tool status {status} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id} - elapsed: {elapsed:.2f}s"
+ )
+ if status and status.get("status") in COMPLETED_FINAL_STATUSES:
+ error = self._handle_tool_execution_status(
+ execution_id=self.execution_id,
+ file_execution_id=file_execution_id,
+ container_name=file_execution_data.tool_container_name,
+ )
+ if error:
+ response = self._create_run_response(
+ status=RunnerContainerRunStatus.ERROR,
+ error=error,
+ )
+ break
+ response = self._create_run_response(
+ status=RunnerContainerRunStatus.SUCCESS,
+ )
+ break
+ time.sleep(interval_seconds)
+
+ if not response:
+ logger.error(
+ f"Tool {file_execution_data.tool_container_name} is not completed within {max_wait_seconds} seconds"
+ )
+ response = self._create_run_response(
+ status=RunnerContainerRunStatus.ERROR,
+ error=f"Tool is not completed within {max_wait_seconds} seconds",
+ )
+
+ if file_execution_data and file_execution_data.tool_container_name:
+ self.cleanup_tool_container(
+ container_name=file_execution_data.tool_container_name,
+ file_execution_id=file_execution_id,
+ )
+ return response
+
def call_tool_handler(
self,
file_execution_id: str,
@@ -82,7 +166,219 @@ class ToolSandboxHelper:
image_tag: str,
settings: dict[str, Any],
retry_count: int | None = None,
- ) -> dict[str, Any] | None:
+ ) -> RunnerContainerRunResponse | None:
+ """Calling unstract runner to run the required tool.
+
+ Args:
+ image_name (str): image name
+ image_tag (str): image tag
+ params (dict[str, Any]): tool params
+ settings (dict[str, Any]): tool settings
+
+ Returns:
+ Optional[dict[str, Any]]: tool response
+ """
+ file_execution_tracker = FileExecutionStatusTracker()
+ file_execution_data = file_execution_tracker.get_data(
+ execution_id=self.execution_id, file_execution_id=file_execution_id
+ )
+
+ if not file_execution_data:
+ logger.warning(
+ f"File execution data not found for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
+ )
+ file_execution_tracker.set_data(
+ execution_id=self.execution_id,
+ file_execution_id=file_execution_id,
+ file_execution_data=FileExecutionData(
+ execution_id=self.execution_id,
+ file_execution_id=file_execution_id,
+ organization_id=self.organization_id,
+ stage_status=FileExecutionStageData(
+ stage=FileExecutionStage.INITIALIZATION,
+ status=FileExecutionStageStatus.SUCCESS,
+ ),
+ status_history=[],
+ ),
+ )
+ response = self._run_and_poll(
+ file_execution_id=file_execution_id,
+ image_name=image_name,
+ image_tag=image_tag,
+ settings=settings,
+ retry_count=retry_count,
+ )
+
+ self._update_stage_status_for_tool_execution(file_execution_id, response)
+ self._update_stage_status(
+ status=FileExecutionStageStatus.IN_PROGRESS,
+ stage=FileExecutionStage.FINALIZATION,
+ file_execution_id=file_execution_id,
+ )
+ else:
+ logger.info(
+ f"File execution data {file_execution_data} found for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
+ )
+ if file_execution_data.error:
+ response = self._create_run_response(
+ status=RunnerContainerRunStatus.ERROR,
+ error=file_execution_data.error,
+ )
+ return response
+
+ stage = file_execution_data.stage_status.stage
+ logger.info(
+ f"Current File execution stage {stage.value} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
+ )
+ if stage == FileExecutionStage.TOOL_EXECUTION:
+ response = self.poll_tool_status(
+ file_execution_id=file_execution_id,
+ file_execution_data=file_execution_data,
+ )
+ self._update_stage_status_for_tool_execution(file_execution_id, response)
+ self._update_stage_status(
+ status=FileExecutionStageStatus.IN_PROGRESS,
+ stage=FileExecutionStage.FINALIZATION,
+ file_execution_id=file_execution_id,
+ )
+ elif stage.is_before(FileExecutionStage.TOOL_EXECUTION):
+ self._update_stage_status(
+ status=FileExecutionStageStatus.SUCCESS,
+ stage=stage,
+ file_execution_id=file_execution_id,
+ )
+ response = self._run_and_poll(
+ file_execution_id=file_execution_id,
+ image_name=image_name,
+ image_tag=image_tag,
+ settings=settings,
+ retry_count=retry_count,
+ )
+ self._update_stage_status_for_tool_execution(file_execution_id, response)
+ self._update_stage_status(
+ status=FileExecutionStageStatus.IN_PROGRESS,
+ stage=FileExecutionStage.FINALIZATION,
+ file_execution_id=file_execution_id,
+ )
+ else:
+ logger.warning(
+ f"File execution data stage {file_execution_data.stage_status.stage} is after tool execution for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
+ )
+ # Assuming the tool execution is successful
+ response = self._create_run_response(
+ status=RunnerContainerRunStatus.SUCCESS,
+ )
+ logger.info(
+ f"Tool execution response: {response} for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
+ )
+ return response
+
+ def _run_and_poll(
+ self,
+ file_execution_id: str,
+ image_name: str,
+ image_tag: str,
+ settings: dict[str, Any],
+ retry_count: int | None = None,
+ ) -> RunnerContainerRunResponse:
+ logger.info(
+ f"Calling runner to run tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
+ )
+ response = self.run_tool_container(
+ file_execution_id=file_execution_id,
+ image_name=image_name,
+ image_tag=image_tag,
+ settings=settings,
+ retry_count=retry_count,
+ )
+ logger.info(
+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"
+ )
+
+ if response.status == RunnerContainerRunStatus.RUNNING:
+ logger.info(
+ f"Polling tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
+ )
+ return self.poll_tool_status(file_execution_id)
+
+ logger.info(
+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"
+ )
+ return response
+
+ def cleanup_tool_container(
+ self,
+ container_name: str,
+ file_execution_id: str,
+ ) -> None:
+ url = f"{self.base_url}{UnstractRunner.CLEANUP_TOOL_CONTAINER_API_ENDPOINT}"
+ headers = {
+ "X-Request-ID": file_execution_id,
+ }
+ data = self.cleanup_tool_container_request_data(container_name=container_name)
+ response = requests.post(url, headers=headers, json=data)
+ if response.status_code != 200:
+ logger.error(
+ f"Error while calling tool {container_name} reason: {response.reason}"
+ )
+ return response.json()
+
+ def _update_stage_status(
+ self,
+ status: FileExecutionStageStatus,
+ stage: FileExecutionStage,
+ file_execution_id: str,
+ ) -> None:
+ file_execution_tracker = FileExecutionStatusTracker()
+
+ file_execution_data = file_execution_tracker.get_data(
+ execution_id=self.execution_id,
+ file_execution_id=file_execution_id,
+ )
+ if not file_execution_data:
+ logger.warning(
+ f"File execution data not found for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
+ )
+ return
+ if file_execution_data.stage_status.stage != stage:
+ logger.warning(
+ f"File execution data stage {file_execution_data.stage_status.stage} is not {stage} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id}"
+ )
+ return
+
+ stage_status = file_execution_data.stage_status
+ stage_status.status = status
+
+ file_execution_tracker.update_stage_status(
+ execution_id=self.execution_id,
+ file_execution_id=file_execution_id,
+ stage_status=stage_status,
+ )
+
+ def _update_stage_status_for_tool_execution(
+ self,
+ file_execution_id: str,
+ response: RunnerContainerRunResponse,
+ ) -> None:
+ stage_status = (
+ FileExecutionStageStatus.SUCCESS
+ if response.status == RunnerContainerRunStatus.SUCCESS
+ else FileExecutionStageStatus.FAILED
+ )
+ self._update_stage_status(
+ status=stage_status,
+ stage=FileExecutionStage.TOOL_EXECUTION,
+ file_execution_id=file_execution_id,
+ )
+
+ def run_tool_container(
+ self,
+ file_execution_id: str,
+ image_name: str,
+ image_tag: str,
+ settings: dict[str, Any],
+ retry_count: int | None = None,
+ ) -> RunnerContainerRunResponse | None:
"""Calling unstract runner to run the required tool.
Args:
@@ -121,7 +417,7 @@ class ToolSandboxHelper:
error_message = response.text
logger.error(f"Error from runner: {error_message}")
raise ToolSanboxError(error_message) from e
- return response.json()
+ return RunnerContainerRunResponse.from_dict(response.json())
def create_tool_request_data(
self,
@@ -150,3 +446,94 @@ class ToolSandboxHelper:
"messaging_channel": self.messaging_channel,
}
return data
+
+ def _create_run_response(
+ self,
+ status: RunnerContainerRunStatus = RunnerContainerRunStatus.SUCCESS,
+ error: str | None = None,
+ ) -> RunnerContainerRunResponse:
+ return RunnerContainerRunResponse(
+ type="RESULT",
+ result=None,
+ error=error,
+ status=status,
+ )
+
+ def cleanup_tool_container_request_data(self, container_name: str) -> dict[str, Any]:
+ """Creating cleanup tool container request data."""
+ data = {"container_name": container_name}
+ return data
+
+ def _check_tool_run_status(
+ self,
+ container_name: str,
+ ) -> dict[str, Any] | None:
+ params = self._create_tool_run_status_check_request_data(container_name)
+ response = requests.get(
+ url=f"{self.base_url}{UnstractRunner.RUN_STATUS_API_ENDPOINT}",
+ params=params,
+ )
+ result: dict[str, Any] | None = None
+ if response.status_code == 200:
+ result = response.json()
+ else:
+ logger.error(
+ f"Error while calling tool {container_name} status code: {response.status_code} reason: {response.reason}"
+ )
+ return result
+
+ def _create_tool_run_status_check_request_data(
+ self,
+ container_name: str,
+ ) -> dict[str, Any]:
+ params = {"container_name": container_name}
+ return params
+
+ def _handle_tool_execution_status(
+ self, execution_id: str, file_execution_id: str, container_name: str
+ ) -> str | None:
+ """Get the tool execution status data from the tool execution tracker."""
+ error = None
+ tool_execution_data = ToolExecutionData(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ )
+ tool_execution_tracker = ToolExecutionTracker()
+ try:
+ tool_execution_status_data = tool_execution_tracker.get_status(
+ tool_execution_data
+ )
+ if not tool_execution_status_data:
+ logger.warning(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - failed to fetch execution status"
+ )
+ return error
+ status = tool_execution_status_data.status
+ error = tool_execution_status_data.error
+ if status == ToolExecutionStatus.FAILED:
+ logger.error(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - tool run failed. Error: {error}"
+ )
+ elif status == ToolExecutionStatus.SUCCESS:
+ logger.info(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - tool execution completed successfully"
+ )
+ else:
+ logger.warning(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - unexpected tool status: {status}"
+ )
+ return error
+ except Exception as error:
+ logger.error(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - failed to fetch execution status. Error: {error}",
+ exc_info=True,
+ )
+ return str(error)
+ finally:
+ # Delete the status from cache since it is no longer needed
+ tool_execution_tracker.delete_status(tool_execution_data)
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
index 8738e2807..2eee55f61 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
@@ -1,6 +1,7 @@
from typing import Any
from unstract.tool_sandbox.constants import ToolCommandKey, UnstractRunner
+from unstract.tool_sandbox.dto import RunnerContainerRunResponse
from unstract.tool_sandbox.helper import ToolSandboxHelper
@@ -95,7 +96,7 @@ class ToolSandbox:
def run_tool(
self, file_execution_id: str, retry_count: int | None = None
- ) -> dict[str, Any] | None:
+ ) -> RunnerContainerRunResponse | None:
return self.helper.call_tool_handler( # type: ignore
file_execution_id,
self.image_name,
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index feffce305..ef05ffd77 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -7,6 +7,7 @@ from redis import Redis
from unstract.core.pubsub_helper import LogPublisher
from unstract.tool_registry import ToolRegistry
from unstract.tool_sandbox import ToolSandbox
+from unstract.tool_sandbox.dto import RunnerContainerRunResponse
from unstract.workflow_execution.constants import ToolExecution
from unstract.workflow_execution.constants import ToolRuntimeVariable as ToolRV
from unstract.workflow_execution.dto import ToolInstance, WorkflowDto
@@ -177,7 +178,7 @@ class ToolsUtils:
self,
file_execution_id: str,
tool_sandbox: ToolSandbox,
- ) -> dict[str, Any] | None:
+ ) -> RunnerContainerRunResponse | None:
return self.run_tool_with_retry(file_execution_id, tool_sandbox)
def run_tool_with_retry(
@@ -185,7 +186,7 @@ class ToolsUtils:
file_execution_id: str,
tool_sandbox: ToolSandbox,
max_retries: int = ToolExecution.MAXIMUM_RETRY,
- ) -> dict[str, Any] | None:
+ ) -> RunnerContainerRunResponse | None:
for retry_count in range(max_retries):
try:
response = tool_sandbox.run_tool(file_execution_id, retry_count)
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index acd7a64fe..c0a2d0715 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -212,8 +212,8 @@ class WorkflowExecutionService:
result = self.tool_utils.run_tool(
file_execution_id=self.file_execution_id, tool_sandbox=sandbox
)
- if result and result.get("error"):
- raise ToolOutputNotFoundException(result.get("error"))
+ if result and result.error:
+ raise ToolOutputNotFoundException(result.error)
if not self.validate_execution_result(step + 1):
raise ToolOutputNotFoundException(
f"Error running tool '{tool_uid}' for run "