Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2901 [FIX] Container startup race condition with polling grace period (#1602)

ali83d17ea4f533
diff --git a/backend/sample.env b/backend/sample.env
index 7553bc280..af73870e1 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -31,6 +31,17 @@ REDIS_PORT=6379
REDIS_PASSWORD=""
REDIS_USER=default
+# Redis Retry Configuration
+# Controls automatic retry behavior for transient Redis connection failures
+# These settings apply to all Redis read operations in execution status trackers
+# Max retry attempts after initial attempt before giving up (default: 4, total attempts: 5)
+REDIS_RETRY_MAX_ATTEMPTS=4
+# Exponential backoff multiplier in seconds (default: 0.5)
+# With defaults (4 retries after initial = 5 total attempts):
+# Retry delays: 0.5s, 1s, 2s, 4s = 5 total attempts over 7.5s
+# Handles: network blips, Redis restarts (3-5s), connection pool exhaustion
+REDIS_RETRY_BACKOFF_FACTOR=0.5
+
# Connector OAuth
SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND=3600
GOOGLE_OAUTH2_KEY=
diff --git a/unstract/core/src/unstract/core/file_execution_tracker.py b/unstract/core/src/unstract/core/file_execution_tracker.py
index d447c7648..43c5fabe3 100644
--- a/unstract/core/src/unstract/core/file_execution_tracker.py
+++ b/unstract/core/src/unstract/core/file_execution_tracker.py
@@ -12,6 +12,7 @@ from unstract.core.exceptions import (
FileExecutionTrackerNotFound,
FileExecutionTrackerValueException,
)
+from unstract.core.utilities import retry_on_redis_error
logger = logging.getLogger(__name__)
@@ -286,11 +287,16 @@ class FileExecutionStatusTracker:
pipe.expire(key, self.CACHE_TTL_IN_SECOND)
pipe.execute()
+ @retry_on_redis_error(retry_logger=logger)
def get_data(
self, execution_id: str, file_execution_id: str
) -> FileExecutionData | None:
"""Get the status of a file execution.
+ This method includes automatic retry logic for transient Redis connection errors.
+ Retry behavior is configurable via REDIS_RETRY_MAX_ATTEMPTS and
+ REDIS_RETRY_BACKOFF_FACTOR environment variables (defaults: 5 retries, 0.5s backoff).
+
Args:
execution_id (str): Execution id of the file execution
file_execution_id (str): File execution id of the file execution
@@ -304,8 +310,10 @@ class FileExecutionStatusTracker:
if not data:
return None
- stage_status_dict = json.loads(data.get(FileExecutionField.STAGE_STATUS, {}))
- status_history_list = json.loads(data.get(FileExecutionField.STATUS_HISTORY, []))
+ stage_status_raw = data.get(FileExecutionField.STAGE_STATUS) or "{}"
+ status_history_raw = data.get(FileExecutionField.STATUS_HISTORY) or "[]"
+ stage_status_dict = json.loads(stage_status_raw)
+ status_history_list = json.loads(status_history_raw)
return FileExecutionData(
execution_id=execution_id,
organization_id=data.get(FileExecutionField.ORGANIZATION_ID),
diff --git a/unstract/core/src/unstract/core/tool_execution_status.py b/unstract/core/src/unstract/core/tool_execution_status.py
index 2fadba21e..ca56180bc 100644
--- a/unstract/core/src/unstract/core/tool_execution_status.py
+++ b/unstract/core/src/unstract/core/tool_execution_status.py
@@ -9,6 +9,7 @@ from unstract.core.exceptions import (
ToolExecutionStatusException,
ToolExecutionValueException,
)
+from unstract.core.utilities import retry_on_redis_error
logger = logging.getLogger(__name__)
@@ -142,11 +143,16 @@ class ToolExecutionTracker:
pipe.expire(key, self.CACHE_TTL_IN_SECOND)
pipe.execute()
+ @retry_on_redis_error(retry_logger=logger)
def get_status(
self, tool_execution_data: ToolExecutionData
) -> ToolExecutionData | None:
"""Get the status of a tool execution.
+ This method includes automatic retry logic for transient Redis connection errors.
+ Retry behavior is configurable via REDIS_RETRY_MAX_ATTEMPTS and
+ REDIS_RETRY_BACKOFF_FACTOR environment variables (defaults: 5 retries, 0.5s backoff).
+
Args:
tool_execution_data (ToolExecutionData): Status of the tool execution
diff --git a/unstract/core/src/unstract/core/utilities.py b/unstract/core/src/unstract/core/utilities.py
index 6b52e7e73..6bc7af6ce 100644
--- a/unstract/core/src/unstract/core/utilities.py
+++ b/unstract/core/src/unstract/core/utilities.py
@@ -1,7 +1,12 @@
+import functools
import logging
import os
+import time
import uuid
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import TimeoutError as RedisTimeoutError
+
logger = logging.getLogger()
@@ -70,3 +75,146 @@ def redact_sensitive_string(str_to_redact: str, reveal_length: int = 4) -> str:
revealed_part = str_to_redact[:reveal_length]
redacted_part = "x" * redacted_length
return revealed_part + redacted_part
+
+
+def _safe_get_env_int(key: str, default: int, validation_logger=None) -> int:
+ """Safely get integer from environment variable with validation.
+
+ Args:
+ key (str): Environment variable name
+ default (int): Default value if invalid or missing
+ validation_logger: Logger for warnings (uses module logger if None)
+
+ Returns:
+ int: Validated integer value or default
+
+ Raises:
+ No exceptions - always returns a valid integer (default on error)
+ """
+ log = validation_logger or logger
+ try:
+ value = int(os.getenv(key, str(default)))
+ if value < 0:
+ raise ValueError(f"{key} must be non-negative, got {value}")
+ return value
+ except (ValueError, TypeError) as e:
+ env_value = os.getenv(key)
+ log.warning(
+ f"Invalid {key} environment variable '{env_value}': {e}. "
+ f"Using default value: {default}"
+ )
+ return default
+
+
+def _safe_get_env_float(key: str, default: float, validation_logger=None) -> float:
+ """Safely get float from environment variable with validation.
+
+ Args:
+ key (str): Environment variable name
+ default (float): Default value if invalid or missing
+ validation_logger: Logger for warnings (uses module logger if None)
+
+ Returns:
+ float: Validated float value or default
+
+ Raises:
+ No exceptions - always returns a valid float (default on error)
+ """
+ log = validation_logger or logger
+ try:
+ value = float(os.getenv(key, str(default)))
+ if value < 0:
+ raise ValueError(f"{key} must be non-negative, got {value}")
+ return value
+ except (ValueError, TypeError) as e:
+ env_value = os.getenv(key)
+ log.warning(
+ f"Invalid {key} environment variable '{env_value}': {e}. "
+ f"Using default value: {default}"
+ )
+ return default
+
+
+def retry_on_redis_error(
+ max_retries: int | None = None, backoff_factor: float | None = None, retry_logger=None
+):
+ """Decorator to retry Redis operations on transient connection errors.
+
+ This decorator automatically retries Redis operations that fail due to transient
+ connection issues like network timeouts or temporary connection loss.
+
+ Args:
+ max_retries (int | None): Number of retries after initial attempt. If None, reads from
+ REDIS_RETRY_MAX_ATTEMPTS env var (default: 4, total attempts: 5)
+ backoff_factor (float | None): Exponential backoff multiplier in seconds. If None,
+ reads from REDIS_RETRY_BACKOFF_FACTOR env var (default: 0.5)
+ With defaults (4 retries after initial, 0.5s backoff = 5 total attempts):
+ - Initial attempt (no delay)
+ - 1st retry: 0.5s delay (cumulative: 0.5s)
+ - 2nd retry: 1.0s delay (cumulative: 1.5s)
+ - 3rd retry: 2.0s delay (cumulative: 3.5s)
+ - 4th retry: 4.0s delay (cumulative: 7.5s)
+ Total: 5 attempts over 7.5 seconds
+ retry_logger: Logger instance for warnings (optional, uses module logger if None)
+
+ Environment Variables:
+ REDIS_RETRY_MAX_ATTEMPTS (int): Retries after initial attempt (default: 4, total attempts: 5)
+ REDIS_RETRY_BACKOFF_FACTOR (float): Exponential backoff multiplier (default: 0.5)
+
+ Retries on:
+ - redis.exceptions.ConnectionError: Connection failures
+ - redis.exceptions.TimeoutError: Request timeouts
+
+ Does NOT retry on:
+ - redis.exceptions.DataError: Invalid data type
+ - redis.exceptions.ResponseError: Redis command errors
+ (These are application logic errors, not transient failures)
+
+ Example:
+ # Use env var defaults (5 retries, 0.5s backoff)
+ @retry_on_redis_error(retry_logger=logger)
+ def get_data_from_redis(self, key):
+ return self.redis_client.get(key)
+
+ # Override defaults
+ @retry_on_redis_error(max_retries=3, backoff_factor=0.2, retry_logger=logger)
+ def get_critical_data(self, key):
+ return self.redis_client.get(key)
+
+ Returns:
+ Decorated function with automatic retry logic
+ """
+ # Read from env vars if not explicitly provided, with validation
+ if max_retries is None:
+ max_retries = _safe_get_env_int("REDIS_RETRY_MAX_ATTEMPTS", 4, retry_logger)
+ if backoff_factor is None:
+ backoff_factor = _safe_get_env_float(
+ "REDIS_RETRY_BACKOFF_FACTOR", 0.5, retry_logger
+ )
+
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ log = retry_logger or logger
+ for attempt in range(max_retries + 1): # Initial attempt + N retries
+ try:
+ return func(*args, **kwargs)
+ except (RedisConnectionError, RedisTimeoutError) as e:
+ if attempt == max_retries: # Last retry exhausted
+ # Final attempt failed, re-raise the exception
+ log.exception(
+ f"Redis operation {func.__name__} failed after {max_retries + 1} total attempts"
+ )
+ raise
+ # Calculate exponential backoff delay
+ delay = backoff_factor * (2**attempt)
+ log.warning(
+ f"Redis transient error in {func.__name__}, "
+ f"retry {attempt + 1}/{max_retries} after {delay:.1f}s: {e}"
+ )
+ time.sleep(delay)
+ return None
+
+ return wrapper
+
+ return decorator
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
index 091696491..c259a1858 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
@@ -23,7 +23,7 @@ from unstract.core.tool_execution_status import (
ToolExecutionStatus,
ToolExecutionTracker,
)
-from unstract.core.utilities import UnstractUtils
+from unstract.core.utilities import UnstractUtils, _safe_get_env_int
from unstract.tool_sandbox.constants import UnstractRunner
from unstract.tool_sandbox.dto import (
RunnerContainerRunResponse,
@@ -32,6 +32,10 @@ from unstract.tool_sandbox.dto import (
logger = logging.getLogger(__name__)
+# Polling grace period to tolerate NOT_FOUND status during container startup
+# During this period, NOT_FOUND status doesn't immediately mean container failed
+POLL_NOT_FOUND_GRACE_PERIOD = _safe_get_env_int("POLL_NOT_FOUND_GRACE_PERIOD", 40, logger)
+
COMPLETED_FINAL_STATUSES = {
ContainerStatus.EXITED.value,
ContainerStatus.DEAD.value,
@@ -132,12 +136,61 @@ class ToolSandboxHelper:
end_time = start_time + timedelta(seconds=max_wait_seconds)
response: RunnerContainerRunResponse | None = None
+ # Track when NOT_FOUND status first seen for grace period handling
+ not_found_first_seen: datetime | None = None
+
while datetime.now(UTC) < end_time:
status = self._check_tool_run_status(file_execution_data.tool_container_name)
elapsed = (datetime.now(UTC) - start_time).total_seconds()
logger.info(
f"Tool status {status} for execution_id: {self.execution_id} and file_execution_id: {file_execution_id} - elapsed: {elapsed:.2f}s"
)
+
+ # Special handling for NOT_FOUND status to avoid race conditions during container startup
+ if status and status.get("status") == ContainerStatus.NOT_FOUND.value:
+ if not_found_first_seen is None:
+ not_found_first_seen = datetime.now(UTC)
+ logger.info(
+ f"Container NOT_FOUND detected - starting {POLL_NOT_FOUND_GRACE_PERIOD}s polling grace period "
+ f"for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
+ )
+
+ not_found_duration = (
+ datetime.now(UTC) - not_found_first_seen
+ ).total_seconds()
+
+ if not_found_duration < POLL_NOT_FOUND_GRACE_PERIOD:
+ logger.debug(
+ f"Within polling grace period ({not_found_duration:.1f}s/{POLL_NOT_FOUND_GRACE_PERIOD}s) - "
+ f"continuing poll for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
+ )
+ time.sleep(interval_seconds)
+ continue # Don't treat as completed yet, continue polling
+ else:
+ logger.warning(
+ f"Polling grace period exceeded ({not_found_duration:.1f}s) - "
+ f"treating NOT_FOUND as completed for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
+ )
+ # Fall through to existing completion handling below
+ elif not_found_first_seen is not None:
+ # Grace period was active, check if status actually changed
+ if status is None:
+ # Transient fetch failure - don't reset grace period
+ logger.warning(
+ f"Transient status fetch failure during grace period for "
+ f"execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
+ )
+ time.sleep(interval_seconds)
+ continue
+ else:
+ # Status changed from NOT_FOUND to something else
+ new_status = status.get("status", "UNKNOWN")
+ logger.info(
+ f"Status changed from NOT_FOUND to {new_status} - "
+ f"resetting grace period tracking for execution_id: {self.execution_id}, file_execution_id: {file_execution_id}"
+ )
+ not_found_first_seen = None
+
if status and status.get("status") in COMPLETED_FINAL_STATUSES:
error = self._handle_tool_execution_status(
execution_id=self.execution_id,
@@ -524,32 +577,36 @@ class ToolSandboxHelper:
tool_execution_data
)
if not tool_execution_status_data:
- logger.warning(
- f"Execution ID: {execution_id}, docker "
- f"container: {container_name} - failed to fetch execution status"
+ error_msg = (
+ f"Container {container_name} failed to write execution status data. "
+ f"Container may have crashed during startup or tool execution failed before writing status."
+ )
+ logger.error(
+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker container: {container_name} - "
+ f"failed to fetch execution status. {error_msg}"
)
- return error
+ return error_msg
status = tool_execution_status_data.status
error = tool_execution_status_data.error
if status == ToolExecutionStatus.FAILED:
logger.error(
- f"Execution ID: {execution_id}, docker "
+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "
f"container: {container_name} - tool run failed. Error: {error}"
)
elif status == ToolExecutionStatus.SUCCESS:
logger.info(
- f"Execution ID: {execution_id}, docker "
+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "
f"container: {container_name} - tool execution completed successfully"
)
else:
logger.warning(
- f"Execution ID: {execution_id}, docker "
+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "
f"container: {container_name} - unexpected tool status: {status}"
)
return error
except Exception as error:
logger.error(
- f"Execution ID: {execution_id}, docker "
+ f"Execution ID: {execution_id}, file execution ID: {file_execution_id}, docker "
f"container: {container_name} - failed to fetch execution status. Error: {error}",
exc_info=True,
)
diff --git a/workers/sample.env b/workers/sample.env
index 48148e154..3f281817d 100644
--- a/workers/sample.env
+++ b/workers/sample.env
@@ -266,6 +266,21 @@ INSTANT_WF_POLLING_TIMEOUT=300
# Active File Execution cache in seconds
ACTIVE_FILE_CACHE_TTL=300
+# Polling Grace Period for NOT_FOUND Status
+# How long the status poller tolerates NOT_FOUND before treating it as failure
+POLL_NOT_FOUND_GRACE_PERIOD=40
+
+# Redis Retry Configuration
+# Controls automatic retry behavior for transient Redis connection failures
+# These settings apply to all Redis read operations in execution status trackers
+# Max retry attempts after initial attempt before giving up (default: 4, total attempts: 5)
+REDIS_RETRY_MAX_ATTEMPTS=4
+# Exponential backoff multiplier in seconds (default: 0.5)
+# With defaults (4 retries after initial = 5 total attempts):
+# Retry delays: 0.5s, 1s, 2s, 4s = 5 total attempts over 7.5s
+# Handles: network blips, Redis restarts (3-5s), connection pool exhaustion
+REDIS_RETRY_BACKOFF_FACTOR=0.5
+
# =============================================================================
# Development Settings
# =============================================================================
@@ -323,6 +338,20 @@ CSRF_COOKIE_SECURE=false
SENTRY_DSN=
SENTRY_ENVIRONMENT=development
+# Google Drive
+GDRIVE_GOOGLE_SERVICE_ACCOUNT=
+GDRIVE_GOOGLE_PROJECT_ID=
+GOOGLE_STORAGE_ACCESS_KEY_ID=
+GOOGLE_STORAGE_SECRET_ACCESS_KEY=
+GOOGLE_STORAGE_BASE_URL=https://storage.googleapis.com
+
+# Connector OAuth
+SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND=3600
+GOOGLE_OAUTH2_KEY=
+GOOGLE_OAUTH2_SECRET=
+
+
+
# =============================================================================
# Local Development Overrides
# =============================================================================