LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2901 [FIX] Container startup race condition with polling grace period (#1602)
ali83d17ea4f533
diff --git a/backend/sample.env b/backend/sample.envindex 7553bc280..af73870e1 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -31,6 +31,17 @@ REDIS_PORT=6379REDIS_PASSWORD=""REDIS_USER=default+# Redis Retry Configuration+# Controls automatic retry behavior for transient Redis connection failures+# These settings apply to all Redis read operations in execution status trackers+# Max retry attempts after initial attempt before giving up (default: 4, total attempts: 5)+REDIS_RETRY_MAX_ATTEMPTS=4+# Exponential backoff multiplier in seconds (default: 0.5)+# With defaults (4 retries after initial = 5 total attempts):+# Retry delays: 0.5s, 1s, 2s, 4s = 5 total attempts over 7.5s+# Handles: network blips, Redis restarts (3-5s), connection pool exhaustion+REDIS_RETRY_BACKOFF_FACTOR=0.5+# Connector OAuthSOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND=3600GOOGLE_OAUTH2_KEY=diff --git a/unstract/core/src/unstract/core/file_execution_tracker.py b/unstract/core/src/unstract/core/file_execution_tracker.pyindex d447c7648..43c5fabe3 100644--- a/unstract/core/src/unstract/core/file_execution_tracker.py+++ b/unstract/core/src/unstract/core/file_execution_tracker.py@@ -12,6 +12,7 @@ from unstract.core.exceptions import (FileExecutionTrackerNotFound,FileExecutionTrackerValueException,)+from unstract.core.utilities import retry_on_redis_errorlogger = logging.getLogger(__name__)@@ -286,11 +287,16 @@ class FileExecutionStatusTracker:pipe.expire(key, self.CACHE_TTL_IN_SECOND)pipe.execute()+ @retry_on_redis_error(retry_logger=logger)def get_data(self, execution_id: str, file_execution_id: str) -> FileExecutionData | None:"""Get the status of a file execution.+ This method includes automatic retry logic for transient Redis connection errors.+ Retry behavior is configurable via REDIS_RETRY_MAX_ATTEMPTS and+ REDIS_RETRY_BACKOFF_FACTOR environment variables (defaults: 5 retries, 0.5s backoff).+Args:execution_id (str): Execution id of the file executionfile_execution_id (str): File execution id of the file execution@@ -304,8 +310,10 @@ class FileExecutionStatusTracker:if not data:return None- stage_status_dict = json.loads(data.get(FileExecutionField.STAGE_STATUS, {}))- status_history_list = json.loads(data.get(FileExecutionField.STATUS_HISTORY, []))+ stage_status_raw = data.get(FileExecutionField.STAGE_STATUS) or "{}"+ status_history_raw = data.get(FileExecutionField.STATUS_HISTORY) or "[]"+ stage_status_dict = json.loads(stage_status_raw)+ status_history_list = json.loads(status_history_raw)return FileExecutionData(execution_id=execution_id,organization_id=data.get(FileExecutionField.ORGANIZATION_ID),diff --git a/unstract/core/src/unstract/core/tool_execution_status.py b/unstract/core/src/unstract/core/tool_execution_status.pyindex 2fadba21e..ca56180bc 100644--- a/unstract/core/src/unstract/core/tool_execution_status.py+++ b/unstract/core/src/unstract/core/tool_execution_status.py@@ -9,6 +9,7 @@ from unstract.core.exceptions import (ToolExecutionStatusException,ToolExecutionValueException,)+from unstract.core.utilities import retry_on_redis_errorlogger = logging.getLogger(__name__)@@ -142,11 +143,16 @@ class ToolExecutionTracker:pipe.expire(key, self.CACHE_TTL_IN_SECOND)pipe.execute()+ @retry_on_redis_error(retry_logger=logger)def get_status(self, tool_execution_data: ToolExecutionData) -> ToolExecutionData | None:"""Get the status of a tool execution.+ This method includes automatic retry logic for transient Redis connection errors.+ Retry behavior is configurable via REDIS_RETRY_MAX_ATTEMPTS and+ REDIS_RETRY_BACKOFF_FACTOR environment variables (defaults: 5 retries, 0.5s backoff).+Args:tool_execution_data (ToolExecutionData): Status of the tool executiondiff --git a/unstract/core/src/unstract/core/utilities.py b/unstract/core/src/unstract/core/utilities.pyindex 6b52e7e73..6bc7af6ce 100644--- a/unstract/core/src/unstract/core/utilities.py+++ b/unstract/core/src/unstract/core/utilities.py@@ -1,7 +1,12 @@+import functoolsimport loggingimport os+import timeimport uuid+from redis.exceptions import ConnectionError as RedisConnectionError+from redis.exceptions import TimeoutError as RedisTimeoutError+logger = logging.getLogger()@@ -70,3 +75,146 @@ def redact_sensitive_string(str_to_redact: str, reveal_length: int = 4) -> str:revealed_part = str_to_redact[:reveal_length]redacted_part = "x" * redacted_lengthreturn revealed_part + redacted_part+++def _safe_get_env_int(key: str, default: int, validation_logger=None) -> int:+ """Safely get integer from environment variable with validation.++ Args:+ key (str): Environment variable name+ default (int): Default value if invalid or missing+ validation_logger: Logger for warnings (uses module logger if None)++ Returns:+ int: Validated integer value or default++ Raises:+ No exceptions - always returns a valid integer (default on error)+ """+ log = validation_logger or logger+ try:+ value = int(os.getenv(key, str(default)))+ if value < 0:+ raise ValueError(f"{key} must be non-negative, got {value}")+ return value+ except (ValueError, TypeError) as e:+ env_value = os.getenv(key)+ log.warning(+ f"Invalid {key} environment variable '{env_value}': {e}. "+ f"Using default value: {default}"+ )+ return default+++def _safe_get_env_float(key: str, default: float, validation_logger=None) -> float:+ """Safely get float from environment variable with validation.++ Args:+ key (str): Environment variable name+ default (float): Default value if invalid or missing+ validation_logger: Logger for warnings (uses module logger if None)++ Returns:+ float: Validated float value or default++ Raises:+ No exceptions - always returns a valid float (default on error)+ """+ log = validation_logger or logger+ try:+ value = float(os.getenv(key, str(default)))+ if value < 0:+ raise ValueError(f"{key} must be non-negative, got {value}")+ return value+ except (ValueError, TypeError) as e:+ env_value = os.getenv(key)+ log.warning(+ f"Invalid {key} environment variable '{env_value}': {e}. "+ f"Using default value: {default}"+ )+ return default+++def retry_on_redis_error(+ max_retries: int | None = None, backoff_factor: float | None = None, retry_logger=None+):+ """Decorator to retry Redis operations on transient connection errors.++ This decorator automatically retries Redis operations that fail due to transient+ connection issues like network timeouts or temporary connection loss.++ Args:+ max_retries (int | None): Number of retries after initial attempt. If None, reads from+ REDIS_RETRY_MAX_ATTEMPTS env var (default: 4, total attempts: 5)+ backoff_factor (float | None): Exponential backoff multiplier in seconds. If None,+ reads from REDIS_RETRY_BACKOFF_FACTOR env var (default: 0.5)+ With defaults (4 retries after initial, 0.5s backoff = 5 total attempts):+ - Initial attempt (no delay)+ - 1st retry: 0.5s delay (cumulative: 0.5s)+ - 2nd retry: 1.0s delay (cumulative: 1.5s)+ - 3rd retry: 2.0s delay (cumulative: 3.5s)+ - 4th retry: 4.0s delay (cumulative: 7.5s)+ Total: 5 attempts over 7.5 seconds+ retry_logger: Logger instance for warnings (optional, uses module logger if None)++ Environment Variables:+ REDIS_RETRY_MAX_ATTEMPTS (int): Retries after initial attempt (default: 4, total attempts: 5)+ REDIS_RETRY_BACKOFF_FACTOR (float): Exponential backoff multiplier (default: 0.5)++ Retries on:+ - redis.exceptions.ConnectionError: Connection failures+ - redis.exceptions.TimeoutError: Request timeouts++ Does NOT retry on:+ - redis.exceptions.DataError: Invalid data type+ - redis.exceptions.ResponseError: Redis command errors+ (These are application logic errors, not transient failures)++ Example:+ # Use env var defaults (5 retries, 0.5s backoff)+ @retry_on_redis_error(retry_logger=logger)+ def get_data_from_redis(self, key):+ return self.redis_client.get(key)++ # Override defaults+ @retry_on_redis_error(max_retries=3, backoff_factor=0.2, retry_logger=logger)+ def get_critical_data(self, key):+ return self.redis_client.get(key)++ Returns:+ Decorated function with automatic retry logic+ """+ # Read from env vars if not explicitly provided, with validation+ if max_retries is None:+ max_retries = _safe_get_env_int("REDIS_RETRY_MAX_ATTEMPTS", 4, retry_logger)+ if backoff_factor is None:+ backoff_factor = _safe_get_env_float(+ "REDIS_RETRY_BACKOFF_FACTOR", 0.5, retry_logger+ )++ def decorator(func):+ @functools.wraps(func)+ def wrapper(*args, **kwargs):+ log = retry_logger or logger+ for attempt in range(max_retries + 1): # Initial attempt + N retries+ try:+ return func(*args, **kwargs)+ except (RedisConnectionError, RedisTimeoutError) as e:+ if attempt == max_retries: # Last retry exhausted+ # Final attempt failed, re-raise the exception+ log.exception(+ f"Redis operation {func.__name__} failed after {max_retries + 1} total attempts"+ )+ raise+ # Calculate exponential backoff delay+ delay = backoff_factor * (2**attempt)+ log.warning(+ f"Redis transient error in {func.__name__}, "+ f"retry {attempt + 1}/{max_retries} after {delay:.1f}s: {e}"+ )+ time.sleep(delay)+ return None++ return wrapper++ return decoratordiff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.pyindex 091696491..c259a1858 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py@@ -23,7 +23,7 @@ from unstract.core.tool_execution_status import (ToolExecutionStatus,ToolExecutionTracker,)-from unstract.core.utilities import UnstractUtils+from unstract.core.utilities import UnstractUtils, _safe_get_env_intfrom unstract.tool_sandbox.constants import UnstractRunnerfrom unstract.tool_sandbox.dto import (RunnerContainerRunResponse,@@ -32,6 +32,10 @@ from unstract.tool_sandbox.dto import (logger = logging.getLogger(__name__)+# Polling grace period to tolerate NOT_FOUND status during container startup+# During this period, NOT_FOUND status doesn't immediately mean container failed+POLL_NOT_FOUND_GRACE_PERIOD = _safe_get_env_int("POLL_NOT_FOUND_GRACE_PERIOD", 40, logger)+COMPLETED_FINAL_STATUSES = {ContainerStatus.EXITED.value,ContainerStatus.DEAD.value,@@ -132,12 +136,61 @@ class ToolSandboxHelper:end_time = start_time + timedelta(seconds=max_wait_seconds)response: RunnerContainerRunResponse | None = None+ # Track when NOT_FOUND status first seen for grace period handling+ not_found_first_seen: datetime | None = None+while datetime.now(UTC) < end_time:status = self._check_tool_run_status(file_execution_data.tool_container_name)elapsed = (datetime.now(UTC) - start_time).total_seconds()logger.info(f"Tool status {status} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id} - elapsed: {elapsed:.2f}s")++ # Special handling for NOT_FOUND status to avoid race conditions during container startup+ if status and status.get("status") == ContainerStatus.NOT_FOUND.value:+ if not_found_first_seen is None:+ not_found_first_seen = datetime.now(UTC)+ logger.info(+ f"Container NOT_FOUND detected - starting {POLL_NOT_FOUND_GRACE_PERIOD}s polling grace period "+ f"for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"+ )++ not_found_duration = (+ datetime.now(UTC) - not_found_first_seen+ ).total_seconds()++ if not_found_duration < POLL_NOT_FOUND_GRACE_PERIOD:+ logger.debug(+ f"Within polling grace period ({not_found_duration:.1f}s/{POLL_NOT_FOUND_GRACE_PERIOD}s) - "+ f"continuing poll for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"+ )+ time.sleep(interval_seconds)+ continue # Don't treat as completed yet, continue polling+ else:+ logger.warning(+ f"Polling grace period exceeded ({not_found_duration:.1f}s) - "+ f"treating NOT_FOUND as completed for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"+ )+ # Fall through to existing completion handling below+ elif not_found_first_seen is not None:+ # Grace period was active, check if status actually changed+ if status is None:+ # Transient fetch failure - don't reset grace period+ logger.warning(+ f"Transient status fetch failure during grace period for "+ f"execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"+ )+ time.sleep(interval_seconds)+ continue+ else:+ # Status changed from NOT_FOUND to something else+ new_status = status.get("status", "UNKNOWN")+ logger.info(+ f"Status changed from NOT_FOUND to {new_status} - "+ f"resetting grace period tracking for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"+ )+ not_found_first_seen = None+if status and status.get("status") in COMPLETED_FINAL_STATUSES:error = self._handle_tool_execution_status(execution_id=self.execution_id,@@ -524,32 +577,36 @@ class ToolSandboxHelper:tool_execution_data)if not tool_execution_status_data:- logger.warning(- f"Execution ID: {execution_id}, docker "- f"container: {container_name} - failed to fetch execution status"+ error_msg = (+ f"Container {container_name} failed to write execution status data. "+ f"Container may have crashed during startup or tool execution failed before writing status."+ )+ logger.error(+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker container: {container_name} - "+ f"failed to fetch execution status. {error_msg}")- return error+ return error_msgstatus = tool_execution_status_data.statuserror = tool_execution_status_data.errorif status == ToolExecutionStatus.FAILED:logger.error(- f"Execution ID: {execution_id}, docker "+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "f"container: {container_name} - tool run failed. Error: {error}")elif status == ToolExecutionStatus.SUCCESS:logger.info(- f"Execution ID: {execution_id}, docker "+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "f"container: {container_name} - tool execution completed successfully")else:logger.warning(- f"Execution ID: {execution_id}, docker "+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "f"container: {container_name} - unexpected tool status: {status}")return errorexcept Exception as error:logger.error(- f"Execution ID: {execution_id}, docker "+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "f"container: {container_name} - failed to fetch execution status. Error: {error}",exc_info=True,)diff --git a/workers/sample.env b/workers/sample.envindex 48148e154..3f281817d 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -266,6 +266,21 @@ INSTANT_WF_POLLING_TIMEOUT=300# Active File Execution cache in secondsACTIVE_FILE_CACHE_TTL=300+# Polling Grace Period for NOT_FOUND Status+# How long the status poller tolerates NOT_FOUND before treating it as failure+POLL_NOT_FOUND_GRACE_PERIOD=40++# Redis Retry Configuration+# Controls automatic retry behavior for transient Redis connection failures+# These settings apply to all Redis read operations in execution status trackers+# Max retry attempts after initial attempt before giving up (default: 4, total attempts: 5)+REDIS_RETRY_MAX_ATTEMPTS=4+# Exponential backoff multiplier in seconds (default: 0.5)+# With defaults (4 retries after initial = 5 total attempts):+# Retry delays: 0.5s, 1s, 2s, 4s = 5 total attempts over 7.5s+# Handles: network blips, Redis restarts (3-5s), connection pool exhaustion+REDIS_RETRY_BACKOFF_FACTOR=0.5+# =============================================================================# Development Settings# =============================================================================@@ -323,6 +338,20 @@ CSRF_COOKIE_SECURE=falseSENTRY_DSN=SENTRY_ENVIRONMENT=development+# Google Drive+GDRIVE_GOOGLE_SERVICE_ACCOUNT=+GDRIVE_GOOGLE_PROJECT_ID=+GOOGLE_STORAGE_ACCESS_KEY_ID=+GOOGLE_STORAGE_SECRET_ACCESS_KEY=+GOOGLE_STORAGE_BASE_URL=https://storage.googleapis.com++# Connector OAuth+SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND=3600+GOOGLE_OAUTH2_KEY=+GOOGLE_OAUTH2_SECRET=+++# =============================================================================# Local Development Overrides# =============================================================================