LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3042 [FIX] Handle SoftTimeLimitExceeded with container cleanup and user-friendly error message (#1699)
alieaecc096182b
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.pyindex 003a28913..014fa4899 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py@@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedeltafrom typing import Anyimport requests+from celery.exceptions import SoftTimeLimitExceededfrom requests import Responsefrom requests.exceptions import ConnectionError, RequestException@@ -365,30 +366,68 @@ class ToolSandboxHelper:settings: dict[str, Any],retry_count: int | None = None,) -> RunnerContainerRunResponse:- logger.info(- f"Calling runner to run tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"- )- response = self.run_tool_container(- file_execution_id=file_execution_id,- image_name=image_name,- image_tag=image_tag,- settings=settings,- retry_count=retry_count,- )- logger.info(- f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"- )+ """Run container and poll for completion with guaranteed cleanup.- if response.status == RunnerContainerRunStatus.RUNNING:+ This method ensures container cleanup happens even if exceptions occur+ (including SoftTimeLimitExceeded from Celery timeouts).+ """+ cleanup_performed = False++ try:logger.info(- f"Polling tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"+ f"Calling runner to run tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"+ )+ response = self.run_tool_container(+ file_execution_id=file_execution_id,+ image_name=image_name,+ image_tag=image_tag,+ settings=settings,+ retry_count=retry_count,+ )+ logger.info(+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}")- return self.poll_tool_status(file_execution_id)- logger.info(- f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"- )- return response+ if response.status == RunnerContainerRunStatus.RUNNING:+ logger.info(+ f"Polling tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"+ )+ # poll_tool_status handles its own cleanup on normal completion+ poll_response = self.poll_tool_status(file_execution_id)+ cleanup_performed = True+ return poll_response++ logger.info(+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"+ )+ return response++ except SoftTimeLimitExceeded:+ logger.exception(+ f"SoftTimeLimitExceeded during tool execution for "+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}. "+ f"Attempting container cleanup before re-raising."+ )+ raise++ except Exception:+ logger.exception(+ f"Exception during tool execution for "+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}. "+ f"Attempting container cleanup before re-raising."+ )+ raise++ finally:+ if not cleanup_performed:+ container_name = self._get_container_name_for_cleanup(+ file_execution_id=file_execution_id,+ )+ self._safe_cleanup_container(+ container_name=container_name,+ file_execution_id=file_execution_id,+ reason="exception_cleanup",+ )def cleanup_tool_container(self,@@ -411,6 +450,76 @@ class ToolSandboxHelper:)return response.json()+ def _get_container_name_for_cleanup(+ self,+ file_execution_id: str,+ ) -> str | None:+ """Retrieve container name for cleanup from Redis tracker.++ Args:+ file_execution_id: The file execution ID++ Returns:+ Container name or None if not available+ """+ try:+ file_execution_tracker = FileExecutionStatusTracker()+ file_execution_data = file_execution_tracker.get_data(+ execution_id=self.execution_id,+ file_execution_id=file_execution_id,+ )+ if file_execution_data and file_execution_data.tool_container_name:+ return file_execution_data.tool_container_name+ except Exception as e:+ logger.warning(+ f"Failed to retrieve container name from tracker for "+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}: {e}"+ )++ return None++ def _safe_cleanup_container(+ self,+ container_name: str | None,+ file_execution_id: str,+ reason: str = "cleanup",+ ) -> None:+ """Safely attempt container cleanup, handling all failure cases gracefully.++ Args:+ container_name: Name of container to clean up (can be None)+ file_execution_id: File execution ID for logging+ reason: Reason for cleanup (for logging)+ """+ if not container_name:+ logger.warning(+ f"Cannot cleanup container - name not available for "+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}, "+ f"reason={reason}"+ )+ return++ try:+ logger.info(+ f"Attempting container cleanup for container={container_name}, "+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}, "+ f"reason={reason}"+ )+ self.cleanup_tool_container(+ container_name=container_name,+ file_execution_id=file_execution_id,+ )+ logger.info(+ f"Container cleanup completed for container={container_name}, "+ f"reason={reason}"+ )+ except Exception as cleanup_error:+ logger.error(+ f"Container cleanup failed for container={container_name}, "+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}, "+ f"reason={reason}, error={cleanup_error}"+ )+def _update_stage_status(self,status: FileExecutionStageStatus,diff --git a/workers/shared/utils/error_utils.py b/workers/shared/utils/error_utils.pynew file mode 100644index 000000000..f7422cb74--- /dev/null+++ b/workers/shared/utils/error_utils.py@@ -0,0 +1,21 @@+"""Error utility functions for user-friendly error messages."""++from celery.exceptions import SoftTimeLimitExceeded+++def get_user_friendly_error_message(exception: Exception) -> str:+ """Convert exception to user-friendly error message.++ Currently handles SoftTimeLimitExceeded specifically.+ Other exceptions are passed through unchanged.++ Args:+ exception: The exception to convert.++ Returns:+ A user-friendly error message string.+ """+ if isinstance(exception, SoftTimeLimitExceeded):+ return "Processing timed out. Couldn't complete processing within the time limit."++ return str(exception)diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex 3b7e7a74b..c6be2830e 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -52,6 +52,7 @@ from unstract.workflow_execution.workflow_execution import WorkflowExecutionServfrom ...api.internal_client import InternalAPIClientfrom ...infrastructure.logging import WorkerLogger+from ...utils.error_utils import get_user_friendly_error_messagefrom ..destination_connector import (DestinationConfig,WorkerDestinationConnector,@@ -854,7 +855,7 @@ class WorkerWorkflowExecutionService:logger.error(f"Tool execution failed for file {file_name}: {str(e)}", exc_info=True)- self._last_execution_error = str(e)+ self._last_execution_error = get_user_friendly_error_message(e)return Falsedef _compile_workflow(