Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

error propagation from sidecar using cache (#1302)

ali20c4474775aa
diff --git a/runner/sample.env b/runner/sample.env
index c13680394..5290791ac 100644
--- a/runner/sample.env
+++ b/runner/sample.env
@@ -49,4 +49,5 @@ FLASK_APP=unstract.runner:app
# Tool Sidecar
TOOL_SIDECAR_ENABLED=False
TOOL_SIDECAR_IMAGE_NAME="unstract/tool-sidecar"
-TOOL_SIDECAR_IMAGE_TAG="0.0.1"
+TOOL_SIDECAR_IMAGE_TAG="0.2.0"
+TOOL_EXECUTION_CACHE_TTL_IN_SECOND=86400 # 24 Hours
diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.py
index 0d253096a..1559694ca 100644
--- a/runner/src/unstract/runner/runner.py
+++ b/runner/src/unstract/runner/runner.py
@@ -9,6 +9,11 @@ from flask import Flask
from unstract.core.constants import LogFieldName
from unstract.core.pubsub_helper import LogPublisher
+from unstract.core.tool_execution_status import (
+ ToolExecutionData,
+ ToolExecutionStatus,
+ ToolExecutionTracker,
+)
from unstract.runner.clients.helper import ContainerClientHelper
from unstract.runner.clients.interface import (
ContainerClientInterface,
@@ -307,6 +312,54 @@ class UnstractRunner:
shell_script = f"{mkdir_cmd} && {run_tool_fn}; {execute_cmd}"
return shell_script
+ def _handle_tool_execution_status(
+ self, execution_id: str, file_execution_id: str, container_name: str
+ ):
+ """Get the tool execution status data from the tool execution tracker."""
+ try:
+ tool_execution_data = ToolExecutionData(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ )
+ tool_execution_tracker = ToolExecutionTracker()
+ tool_execution_status = tool_execution_tracker.get_status(tool_execution_data)
+ if not tool_execution_status:
+ self.logger.warning(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - failed to fetch execution status"
+ )
+ return
+ status = tool_execution_status.status
+ error = tool_execution_status.error
+ if status == ToolExecutionStatus.FAILED:
+ self.logger.error(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - tool run failed. Error: {error}"
+ )
+ raise ToolRunException(error)
+ elif status == ToolExecutionStatus.SUCCESS:
+ self.logger.info(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - tool execution completed successfully"
+ )
+ else:
+ self.logger.warning(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - unexpected tool status: {status}"
+ )
+ except ToolRunException as e:
+ # Tool execution failed; propagate the exception to be handled by the caller
+ raise e
+ except Exception as e:
+ self.logger.error(
+ f"Execution ID: {execution_id}, docker "
+ f"container: {container_name} - failed to fetch execution status. Error: {e}",
+ exc_info=True,
+ )
+ finally:
+ # Delete the status from cache since it is no longer needed
+ tool_execution_tracker.delete_status(tool_execution_data)
+
def run_container(
self,
organization_id: str,
@@ -412,6 +465,12 @@ class UnstractRunner:
f"Execution ID: {execution_id}, docker "
f"container: {container_name} completed execution"
)
+ self._handle_tool_execution_status(
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
+ container_name=container_name,
+ )
+
else:
container: ContainerInterface = self.client.run_container(
container_config
diff --git a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py b/tool-sidecar/src/unstract/tool_sidecar/log_processor.py
index 6417eddd6..d00ccd06e 100644
--- a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py
+++ b/tool-sidecar/src/unstract/tool_sidecar/log_processor.py
@@ -13,6 +13,11 @@ from typing import Any
from unstract.core.constants import LogFieldName
from unstract.core.pubsub_helper import LogPublisher
+from unstract.core.tool_execution_status import (
+ ToolExecutionData,
+ ToolExecutionStatus,
+ ToolExecutionTracker,
+)
from .constants import Env, LogLevel, LogType
from .dto import LogLineDTO
@@ -60,6 +65,26 @@ class LogProcessor:
self.file_execution_id = file_execution_id
self.messaging_channel = messaging_channel
self.container_name = container_name
+ self.tool_execution_tracker = ToolExecutionTracker()
+ self._update_tool_execution_status(status=ToolExecutionStatus.RUNNING)
+
+ def _update_tool_execution_status(
+ self, status: ToolExecutionStatus, error: str | None = None
+ ) -> None:
+ try:
+ tool_execution_data = ToolExecutionData(
+ execution_id=self.execution_id,
+ tool_instance_id=self.tool_instance_id,
+ file_execution_id=self.file_execution_id,
+ organization_id=self.organization_id,
+ status=status,
+ error=error,
+ )
+ self.tool_execution_tracker.update_status(
+ tool_execution_data=tool_execution_data
+ )
+ except Exception as e:
+ logger.error(f"Failed to update tool execution status: {e}", exc_info=True)
def wait_for_log_file(self, timeout: int = 300) -> bool:
"""Wait for the log file to be created by the tool container.
@@ -112,10 +137,14 @@ class LogProcessor:
if log_level == LogLevel.ERROR:
logger.error(f"[Tool: {self.container_name}] {log_dict.get('log')}")
log_process_status.error = log_dict.get("log")
+ self._update_tool_execution_status(
+ status=ToolExecutionStatus.FAILED, error=log_dict.get("log")
+ )
else:
logger.info(f"[Tool: {self.container_name}] {log_dict.get('log')}")
elif log_type == LogType.RESULT:
logger.info(f"[Tool {self.container_name}] Completed running")
+ self._update_tool_execution_status(status=ToolExecutionStatus.SUCCESS)
return LogLineDTO(with_result=True)
elif log_type == LogType.UPDATE:
logger.info(f"[Tool: {self.container_name}] Pushing UI updates")
diff --git a/unstract/core/src/unstract/core/exceptions.py b/unstract/core/src/unstract/core/exceptions.py
new file mode 100644
index 000000000..19de7768b
--- /dev/null
+++ b/unstract/core/src/unstract/core/exceptions.py
@@ -0,0 +1,12 @@
+class ToolExecutionStatusException(Exception):
+ """Raised when an invalid tool execution status is provided."""
+
+ def __init__(self, message: str = "Invalid tool execution status"):
+ super().__init__(message)
+
+
+class ToolExecutionValueException(Exception):
+ """Raised when required execution identifiers are missing."""
+
+ def __init__(self, message: str = "Execution ID and file execution ID are required"):
+ super().__init__(message)
diff --git a/unstract/core/src/unstract/core/tool_execution_status.py b/unstract/core/src/unstract/core/tool_execution_status.py
new file mode 100644
index 000000000..f923b5300
--- /dev/null
+++ b/unstract/core/src/unstract/core/tool_execution_status.py
@@ -0,0 +1,171 @@
+import os
+from dataclasses import dataclass
+from enum import Enum
+
+import redis
+
+from unstract.core.exceptions import (
+ ToolExecutionStatusException,
+ ToolExecutionValueException,
+)
+
+
+class ToolExecutionStatus(Enum):
+ RUNNING = "RUNNING"
+ SUCCESS = "SUCCESS"
+ FAILED = "FAILED"
+
+
+class ToolExecutionField:
+ EXECUTION_ID = "execution_id"
+ FILE_EXECUTION_ID = "file_execution_id"
+ TOOL_INSTANCE_ID = "tool_instance_id"
+ ORGANIZATION_ID = "organization_id"
+ STATUS = "status"
+ ERROR = "error"
+
+
+@dataclass
+class ToolExecutionData:
+ execution_id: str
+ file_execution_id: str | None = None
+ tool_instance_id: str | None = None
+ organization_id: str | None = None
+ status: ToolExecutionStatus | None = None
+ error: str | None = None
+
+ def __post_init__(self):
+ self.validate()
+
+ def validate(self) -> None:
+ if not self.execution_id or not self.file_execution_id:
+ raise ToolExecutionValueException()
+
+
+class ToolExecutionTracker:
+ """Tool execution tracker.
+
+ This class is used to track the status of a tool execution.
+ """
+
+ CACHE_TTL_IN_SECOND = int(
+ os.environ.get("TOOL_EXECUTION_CACHE_TTL_IN_SECOND", 60 * 60 * 24)
+ )
+
+ def __init__(self):
+ self.redis_client = redis.Redis(
+ host=os.environ.get("REDIS_HOST"),
+ port=int(os.environ.get("REDIS_PORT", 6379)),
+ username=os.environ.get("REDIS_USER"),
+ password=os.environ.get("REDIS_PASSWORD"),
+ decode_responses=True, # ensures hgetall returns str instead of bytes
+ )
+
+ def _resolve_field(
+ self,
+ field_name: str,
+ new_data: ToolExecutionData,
+ existing_data: ToolExecutionData | None,
+ ) -> str:
+ """Resolves the value of a field based on the new data and existing data.
+
+ Args:
+ field_name (str): Name of the field
+ new_data (ToolExecutionData): New data
+ existing_data (ToolExecutionData | None): Existing data
+
+ Returns:
+ str: Value of the field
+ """
+ new_value = getattr(new_data, field_name)
+ if not new_value:
+ if not existing_data:
+ return ""
+ new_value = getattr(existing_data, field_name)
+ return new_value or ""
+
+ def get_cache_key(self, tool_execution_data: ToolExecutionData) -> str:
+ return f"tool_execution:{tool_execution_data.execution_id}:{tool_execution_data.file_execution_id}"
+
+ def update_status(self, tool_execution_data: ToolExecutionData) -> None:
+ """Update the status of a tool execution.
+
+ Args:
+ tool_execution_data (ToolExecutionData): Status of the tool execution
+ """
+ tool_execution_data.validate()
+
+ try:
+ status = ToolExecutionStatus(tool_execution_data.status)
+ except ValueError:
+ raise ToolExecutionStatusException()
+
+ key = self.get_cache_key(tool_execution_data)
+ with self.redis_client.pipeline() as pipe:
+ existing_data = self.get_status(tool_execution_data)
+
+ tool_instance_id = self._resolve_field(
+ ToolExecutionField.TOOL_INSTANCE_ID,
+ tool_execution_data,
+ existing_data,
+ )
+ organization_id = self._resolve_field(
+ ToolExecutionField.ORGANIZATION_ID,
+ tool_execution_data,
+ existing_data,
+ )
+ error = self._resolve_field(
+ ToolExecutionField.ERROR,
+ tool_execution_data,
+ existing_data,
+ )
+ pipe.hset(
+ name=key,
+ mapping={
+ ToolExecutionField.EXECUTION_ID: tool_execution_data.execution_id,
+ ToolExecutionField.FILE_EXECUTION_ID: tool_execution_data.file_execution_id,
+ ToolExecutionField.TOOL_INSTANCE_ID: tool_instance_id,
+ ToolExecutionField.ORGANIZATION_ID: organization_id,
+ ToolExecutionField.STATUS: status.value,
+ ToolExecutionField.ERROR: error,
+ },
+ )
+ pipe.expire(key, self.CACHE_TTL_IN_SECOND)
+ pipe.execute()
+
+ def get_status(
+ self, tool_execution_data: ToolExecutionData
+ ) -> ToolExecutionData | None:
+ """Get the status of a tool execution.
+
+ Args:
+ tool_execution_data (ToolExecutionData): Status of the tool execution
+
+ Returns:
+ ToolExecutionData | None: Status of the tool execution
+ """
+ tool_execution_data.validate()
+
+ data = self.redis_client.hgetall(self.get_cache_key(tool_execution_data))
+ if not data:
+ return None
+
+ return ToolExecutionData(
+ tool_instance_id=data.get(ToolExecutionField.TOOL_INSTANCE_ID) or None,
+ execution_id=data.get(ToolExecutionField.EXECUTION_ID) or None,
+ organization_id=data.get(ToolExecutionField.ORGANIZATION_ID) or None,
+ file_execution_id=data.get(ToolExecutionField.FILE_EXECUTION_ID) or None,
+ status=ToolExecutionStatus(data[ToolExecutionField.STATUS])
+ if data.get(ToolExecutionField.STATUS)
+ else None,
+ error=data.get(ToolExecutionField.ERROR) or None,
+ )
+
+ def delete_status(self, tool_execution_data: ToolExecutionData) -> None:
+ """Delete the status of a tool execution.
+
+ Args:
+ tool_execution_data (ToolExecutionData): Status of the tool execution
+ """
+ tool_execution_data.validate()
+ self.redis_client.delete(self.get_cache_key(tool_execution_data))