LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
error propagation from sidecar using cache (#1302)
ali20c4474775aa
diff --git a/runner/sample.env b/runner/sample.envindex c13680394..5290791ac 100644--- a/runner/sample.env+++ b/runner/sample.env@@ -49,4 +49,5 @@ FLASK_APP=unstract.runner:app# Tool SidecarTOOL_SIDECAR_ENABLED=FalseTOOL_SIDECAR_IMAGE_NAME="unstract/tool-sidecar"-TOOL_SIDECAR_IMAGE_TAG="0.0.1"+TOOL_SIDECAR_IMAGE_TAG="0.2.0"+TOOL_EXECUTION_CACHE_TTL_IN_SECOND=86400 # 24 Hoursdiff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.pyindex 0d253096a..1559694ca 100644--- a/runner/src/unstract/runner/runner.py+++ b/runner/src/unstract/runner/runner.py@@ -9,6 +9,11 @@ from flask import Flaskfrom unstract.core.constants import LogFieldNamefrom unstract.core.pubsub_helper import LogPublisher+from unstract.core.tool_execution_status import (+ ToolExecutionData,+ ToolExecutionStatus,+ ToolExecutionTracker,+)from unstract.runner.clients.helper import ContainerClientHelperfrom unstract.runner.clients.interface import (ContainerClientInterface,@@ -307,6 +312,54 @@ class UnstractRunner:shell_script = f"{mkdir_cmd} && {run_tool_fn}; {execute_cmd}"return shell_script+ def _handle_tool_execution_status(+ self, execution_id: str, file_execution_id: str, container_name: str+ ):+ """Get the tool execution status data from the tool execution tracker."""+ try:+ tool_execution_data = ToolExecutionData(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ )+ tool_execution_tracker = ToolExecutionTracker()+ tool_execution_status = tool_execution_tracker.get_status(tool_execution_data)+ if not tool_execution_status:+ self.logger.warning(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - failed to fetch execution status"+ )+ return+ status = tool_execution_status.status+ error = tool_execution_status.error+ if status == ToolExecutionStatus.FAILED:+ self.logger.error(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - tool run failed. Error: {error}"+ )+ raise ToolRunException(error)+ elif status == ToolExecutionStatus.SUCCESS:+ self.logger.info(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - tool execution completed successfully"+ )+ else:+ self.logger.warning(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - unexpected tool status: {status}"+ )+ except ToolRunException as e:+ # Tool execution failed; propagate the exception to be handled by the caller+ raise e+ except Exception as e:+ self.logger.error(+ f"Execution ID: {execution_id}, docker "+ f"container: {container_name} - failed to fetch execution status. Error: {e}",+ exc_info=True,+ )+ finally:+ # Delete the status from cache since it is no longer needed+ tool_execution_tracker.delete_status(tool_execution_data)+def run_container(self,organization_id: str,@@ -412,6 +465,12 @@ class UnstractRunner:f"Execution ID: {execution_id}, docker "f"container: {container_name} completed execution")+ self._handle_tool_execution_status(+ execution_id=execution_id,+ file_execution_id=file_execution_id,+ container_name=container_name,+ )+else:container: ContainerInterface = self.client.run_container(container_configdiff --git a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py b/tool-sidecar/src/unstract/tool_sidecar/log_processor.pyindex 6417eddd6..d00ccd06e 100644--- a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py+++ b/tool-sidecar/src/unstract/tool_sidecar/log_processor.py@@ -13,6 +13,11 @@ from typing import Anyfrom unstract.core.constants import LogFieldNamefrom unstract.core.pubsub_helper import LogPublisher+from unstract.core.tool_execution_status import (+ ToolExecutionData,+ ToolExecutionStatus,+ ToolExecutionTracker,+)from .constants import Env, LogLevel, LogTypefrom .dto import LogLineDTO@@ -60,6 +65,26 @@ class LogProcessor:self.file_execution_id = file_execution_idself.messaging_channel = messaging_channelself.container_name = container_name+ self.tool_execution_tracker = ToolExecutionTracker()+ self._update_tool_execution_status(status=ToolExecutionStatus.RUNNING)++ def _update_tool_execution_status(+ self, status: ToolExecutionStatus, error: str | None = None+ ) -> None:+ try:+ tool_execution_data = ToolExecutionData(+ execution_id=self.execution_id,+ tool_instance_id=self.tool_instance_id,+ file_execution_id=self.file_execution_id,+ organization_id=self.organization_id,+ status=status,+ error=error,+ )+ self.tool_execution_tracker.update_status(+ tool_execution_data=tool_execution_data+ )+ except Exception as e:+ logger.error(f"Failed to update tool execution status: {e}", exc_info=True)def wait_for_log_file(self, timeout: int = 300) -> bool:"""Wait for the log file to be created by the tool container.@@ -112,10 +137,14 @@ class LogProcessor:if log_level == LogLevel.ERROR:logger.error(f"[Tool: {self.container_name}] {log_dict.get('log')}")log_process_status.error = log_dict.get("log")+ self._update_tool_execution_status(+ status=ToolExecutionStatus.FAILED, error=log_dict.get("log")+ )else:logger.info(f"[Tool: {self.container_name}] {log_dict.get('log')}")elif log_type == LogType.RESULT:logger.info(f"[Tool {self.container_name}] Completed running")+ self._update_tool_execution_status(status=ToolExecutionStatus.SUCCESS)return LogLineDTO(with_result=True)elif log_type == LogType.UPDATE:logger.info(f"[Tool: {self.container_name}] Pushing UI updates")diff --git a/unstract/core/src/unstract/core/exceptions.py b/unstract/core/src/unstract/core/exceptions.pynew file mode 100644index 000000000..19de7768b--- /dev/null+++ b/unstract/core/src/unstract/core/exceptions.py@@ -0,0 +1,12 @@+class ToolExecutionStatusException(Exception):+ """Raised when an invalid tool execution status is provided."""++ def __init__(self, message: str = "Invalid tool execution status"):+ super().__init__(message)+++class ToolExecutionValueException(Exception):+ """Raised when required execution identifiers are missing."""++ def __init__(self, message: str = "Execution ID and file execution ID are required"):+ super().__init__(message)diff --git a/unstract/core/src/unstract/core/tool_execution_status.py b/unstract/core/src/unstract/core/tool_execution_status.pynew file mode 100644index 000000000..f923b5300--- /dev/null+++ b/unstract/core/src/unstract/core/tool_execution_status.py@@ -0,0 +1,171 @@+import os+from dataclasses import dataclass+from enum import Enum++import redis++from unstract.core.exceptions import (+ ToolExecutionStatusException,+ ToolExecutionValueException,+)+++class ToolExecutionStatus(Enum):+ RUNNING = "RUNNING"+ SUCCESS = "SUCCESS"+ FAILED = "FAILED"+++class ToolExecutionField:+ EXECUTION_ID = "execution_id"+ FILE_EXECUTION_ID = "file_execution_id"+ TOOL_INSTANCE_ID = "tool_instance_id"+ ORGANIZATION_ID = "organization_id"+ STATUS = "status"+ ERROR = "error"+++@dataclass+class ToolExecutionData:+ execution_id: str+ file_execution_id: str | None = None+ tool_instance_id: str | None = None+ organization_id: str | None = None+ status: ToolExecutionStatus | None = None+ error: str | None = None++ def __post_init__(self):+ self.validate()++ def validate(self) -> None:+ if not self.execution_id or not self.file_execution_id:+ raise ToolExecutionValueException()+++class ToolExecutionTracker:+ """Tool execution tracker.++ This class is used to track the status of a tool execution.+ """++ CACHE_TTL_IN_SECOND = int(+ os.environ.get("TOOL_EXECUTION_CACHE_TTL_IN_SECOND", 60 * 60 * 24)+ )++ def __init__(self):+ self.redis_client = redis.Redis(+ host=os.environ.get("REDIS_HOST"),+ port=int(os.environ.get("REDIS_PORT", 6379)),+ username=os.environ.get("REDIS_USER"),+ password=os.environ.get("REDIS_PASSWORD"),+ decode_responses=True, # ensures hgetall returns str instead of bytes+ )++ def _resolve_field(+ self,+ field_name: str,+ new_data: ToolExecutionData,+ existing_data: ToolExecutionData | None,+ ) -> str:+ """Resolves the value of a field based on the new data and existing data.++ Args:+ field_name (str): Name of the field+ new_data (ToolExecutionData): New data+ existing_data (ToolExecutionData | None): Existing data++ Returns:+ str: Value of the field+ """+ new_value = getattr(new_data, field_name)+ if not new_value:+ if not existing_data:+ return ""+ new_value = getattr(existing_data, field_name)+ return new_value or ""++ def get_cache_key(self, tool_execution_data: ToolExecutionData) -> str:+ return f"tool_execution:{tool_execution_data.execution_id}:{tool_execution_data.file_execution_id}"++ def update_status(self, tool_execution_data: ToolExecutionData) -> None:+ """Update the status of a tool execution.++ Args:+ tool_execution_data (ToolExecutionData): Status of the tool execution+ """+ tool_execution_data.validate()++ try:+ status = ToolExecutionStatus(tool_execution_data.status)+ except ValueError:+ raise ToolExecutionStatusException()++ key = self.get_cache_key(tool_execution_data)+ with self.redis_client.pipeline() as pipe:+ existing_data = self.get_status(tool_execution_data)++ tool_instance_id = self._resolve_field(+ ToolExecutionField.TOOL_INSTANCE_ID,+ tool_execution_data,+ existing_data,+ )+ organization_id = self._resolve_field(+ ToolExecutionField.ORGANIZATION_ID,+ tool_execution_data,+ existing_data,+ )+ error = self._resolve_field(+ ToolExecutionField.ERROR,+ tool_execution_data,+ existing_data,+ )+ pipe.hset(+ name=key,+ mapping={+ ToolExecutionField.EXECUTION_ID: tool_execution_data.execution_id,+ ToolExecutionField.FILE_EXECUTION_ID: tool_execution_data.file_execution_id,+ ToolExecutionField.TOOL_INSTANCE_ID: tool_instance_id,+ ToolExecutionField.ORGANIZATION_ID: organization_id,+ ToolExecutionField.STATUS: status.value,+ ToolExecutionField.ERROR: error,+ },+ )+ pipe.expire(key, self.CACHE_TTL_IN_SECOND)+ pipe.execute()++ def get_status(+ self, tool_execution_data: ToolExecutionData+ ) -> ToolExecutionData | None:+ """Get the status of a tool execution.++ Args:+ tool_execution_data (ToolExecutionData): Status of the tool execution++ Returns:+ ToolExecutionData | None: Status of the tool execution+ """+ tool_execution_data.validate()++ data = self.redis_client.hgetall(self.get_cache_key(tool_execution_data))+ if not data:+ return None++ return ToolExecutionData(+ tool_instance_id=data.get(ToolExecutionField.TOOL_INSTANCE_ID) or None,+ execution_id=data.get(ToolExecutionField.EXECUTION_ID) or None,+ organization_id=data.get(ToolExecutionField.ORGANIZATION_ID) or None,+ file_execution_id=data.get(ToolExecutionField.FILE_EXECUTION_ID) or None,+ status=ToolExecutionStatus(data[ToolExecutionField.STATUS])+ if data.get(ToolExecutionField.STATUS)+ else None,+ error=data.get(ToolExecutionField.ERROR) or None,+ )++ def delete_status(self, tool_execution_data: ToolExecutionData) -> None:+ """Delete the status of a tool execution.++ Args:+ tool_execution_data (ToolExecutionData): Status of the tool execution+ """+ tool_execution_data.validate()+ self.redis_client.delete(self.get_cache_key(tool_execution_data))