Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3042 [FIX] Handle SoftTimeLimitExceeded with container cleanup and user-friendly error message (#1699)

alieaecc096182b
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
index 003a28913..014fa4899 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
@@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any
import requests
+from celery.exceptions import SoftTimeLimitExceeded
from requests import Response
from requests.exceptions import ConnectionError, RequestException
@@ -365,30 +366,68 @@ class ToolSandboxHelper:
settings: dict[str, Any],
retry_count: int | None = None,
) -> RunnerContainerRunResponse:
- logger.info(
- f"Calling runner to run tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
- )
- response = self.run_tool_container(
- file_execution_id=file_execution_id,
- image_name=image_name,
- image_tag=image_tag,
- settings=settings,
- retry_count=retry_count,
- )
- logger.info(
- f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"
- )
+ """Run container and poll for completion with guaranteed cleanup.
- if response.status == RunnerContainerRunStatus.RUNNING:
+ This method ensures container cleanup happens even if exceptions occur
+ (including SoftTimeLimitExceeded from Celery timeouts).
+ """
+ cleanup_performed = False
+
+ try:
logger.info(
- f"Polling tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
+ f"Calling runner to run tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
+ )
+ response = self.run_tool_container(
+ file_execution_id=file_execution_id,
+ image_name=image_name,
+ image_tag=image_tag,
+ settings=settings,
+ retry_count=retry_count,
+ )
+ logger.info(
+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"
)
- return self.poll_tool_status(file_execution_id)
- logger.info(
- f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"
- )
- return response
+ if response.status == RunnerContainerRunStatus.RUNNING:
+ logger.info(
+ f"Polling tool container for execution_id={self.execution_id}, file_execution_id={file_execution_id}"
+ )
+ # poll_tool_status handles its own cleanup on normal completion
+ poll_response = self.poll_tool_status(file_execution_id)
+ cleanup_performed = True
+ return poll_response
+
+ logger.info(
+ f"Tool container run for execution_id={self.execution_id}, file_execution_id={file_execution_id} completed with status {response.status}"
+ )
+ return response
+
+ except SoftTimeLimitExceeded:
+ logger.exception(
+ f"SoftTimeLimitExceeded during tool execution for "
+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}. "
+ f"Attempting container cleanup before re-raising."
+ )
+ raise
+
+ except Exception:
+ logger.exception(
+ f"Exception during tool execution for "
+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}. "
+ f"Attempting container cleanup before re-raising."
+ )
+ raise
+
+ finally:
+ if not cleanup_performed:
+ container_name = self._get_container_name_for_cleanup(
+ file_execution_id=file_execution_id,
+ )
+ self._safe_cleanup_container(
+ container_name=container_name,
+ file_execution_id=file_execution_id,
+ reason="exception_cleanup",
+ )
def cleanup_tool_container(
self,
@@ -411,6 +450,76 @@ class ToolSandboxHelper:
)
return response.json()
+ def _get_container_name_for_cleanup(
+ self,
+ file_execution_id: str,
+ ) -> str | None:
+ """Retrieve container name for cleanup from Redis tracker.
+
+ Args:
+ file_execution_id: The file execution ID
+
+ Returns:
+ Container name or None if not available
+ """
+ try:
+ file_execution_tracker = FileExecutionStatusTracker()
+ file_execution_data = file_execution_tracker.get_data(
+ execution_id=self.execution_id,
+ file_execution_id=file_execution_id,
+ )
+ if file_execution_data and file_execution_data.tool_container_name:
+ return file_execution_data.tool_container_name
+ except Exception as e:
+ logger.warning(
+ f"Failed to retrieve container name from tracker for "
+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}: {e}"
+ )
+
+ return None
+
+ def _safe_cleanup_container(
+ self,
+ container_name: str | None,
+ file_execution_id: str,
+ reason: str = "cleanup",
+ ) -> None:
+ """Safely attempt container cleanup, handling all failure cases gracefully.
+
+ Args:
+ container_name: Name of container to clean up (can be None)
+ file_execution_id: File execution ID for logging
+ reason: Reason for cleanup (for logging)
+ """
+ if not container_name:
+ logger.warning(
+ f"Cannot cleanup container - name not available for "
+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}, "
+ f"reason={reason}"
+ )
+ return
+
+ try:
+ logger.info(
+ f"Attempting container cleanup for container={container_name}, "
+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}, "
+ f"reason={reason}"
+ )
+ self.cleanup_tool_container(
+ container_name=container_name,
+ file_execution_id=file_execution_id,
+ )
+ logger.info(
+ f"Container cleanup completed for container={container_name}, "
+ f"reason={reason}"
+ )
+ except Exception as cleanup_error:
+ logger.error(
+ f"Container cleanup failed for container={container_name}, "
+ f"execution_id={self.execution_id}, file_execution_id={file_execution_id}, "
+ f"reason={reason}, error={cleanup_error}"
+ )
+
def _update_stage_status(
self,
status: FileExecutionStageStatus,
diff --git a/workers/shared/utils/error_utils.py b/workers/shared/utils/error_utils.py
new file mode 100644
index 000000000..f7422cb74
--- /dev/null
+++ b/workers/shared/utils/error_utils.py
@@ -0,0 +1,21 @@
+"""Error utility functions for user-friendly error messages."""
+
+from celery.exceptions import SoftTimeLimitExceeded
+
+
+def get_user_friendly_error_message(exception: Exception) -> str:
+ """Convert exception to user-friendly error message.
+
+ Currently handles SoftTimeLimitExceeded specifically.
+ Other exceptions are passed through unchanged.
+
+ Args:
+ exception: The exception to convert.
+
+ Returns:
+ A user-friendly error message string.
+ """
+ if isinstance(exception, SoftTimeLimitExceeded):
+ return "Processing timed out. Couldn't complete processing within the time limit."
+
+ return str(exception)
diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py
index 3b7e7a74b..c6be2830e 100644
--- a/workers/shared/workflow/execution/service.py
+++ b/workers/shared/workflow/execution/service.py
@@ -52,6 +52,7 @@ from unstract.workflow_execution.workflow_execution import WorkflowExecutionServ
from ...api.internal_client import InternalAPIClient
from ...infrastructure.logging import WorkerLogger
+from ...utils.error_utils import get_user_friendly_error_message
from ..destination_connector import (
DestinationConfig,
WorkerDestinationConnector,
@@ -854,7 +855,7 @@ class WorkerWorkflowExecutionService:
logger.error(
f"Tool execution failed for file {file_name}: {str(e)}", exc_info=True
)
- self._last_execution_error = str(e)
+ self._last_execution_error = get_user_friendly_error_message(e)
return False
def _compile_workflow(