AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Make connection retries abort on shutdown (#13382)
Reinier van der Leerd5545328b320
diff --git a/autogpt_platform/backend/backend/util/process.py b/autogpt_platform/backend/backend/util/process.pyindex beadcfd296..664b7f3c28 100644--- a/autogpt_platform/backend/backend/util/process.py+++ b/autogpt_platform/backend/backend/util/process.py@@ -8,6 +8,7 @@ from typing import Optionalfrom backend.util.logging import configure_loggingfrom backend.util.metrics import sentry_init+from backend.util.retry import stop_retry_loopsfrom backend.util.settings import set_service_namelogger = logging.getLogger(__name__)@@ -93,6 +94,7 @@ class AppProcess(ABC):os.write(sys.stdout.fileno(), (message + "\n").encode())def _self_terminate(self, signum: int, frame):+ stop_retry_loops()if not self._shutting_down:self._shutting_down = Truesys.exit(0)diff --git a/autogpt_platform/backend/backend/util/retry.py b/autogpt_platform/backend/backend/util/retry.pyindex 0003df165a..743dce361e 100644--- a/autogpt_platform/backend/backend/util/retry.py+++ b/autogpt_platform/backend/backend/util/retry.py@@ -1,4 +1,5 @@import asyncio+import inspectimport loggingimport osimport threading@@ -12,6 +13,7 @@ from tenacity import (stop_after_attempt,wait_exponential_jitter,)+from tenacity.stop import stop_basefrom backend.util.settings import get_service_name@@ -20,6 +22,28 @@ logger = logging.getLogger(__name__)# Alert threshold for excessive retriesEXCESSIVE_RETRY_THRESHOLD = 50+# Process-wide shutdown flag. The AppProcess/AppService signal handlers set this+# (via stop_retry_loops) so in-flight retry loops abort promptly instead of+# blocking shutdown for up to max_retry * max_wait seconds if e.g. a backing+# service (Redis/RabbitMQ/RPC peer) is unreachable.+_shutdown_event = threading.Event()++# How often the async interruptible sleep re-checks the shutdown flag.+_SHUTDOWN_POLL_INTERVAL = 0.5+++def stop_retry_loops() -> None:+ """Tell every retry loop in this process to stop ASAP.++ Safe to call from a signal handler: it only sets a ``threading.Event``.+ """+ _shutdown_event.set()+++def is_shutting_down() -> bool:+ return _shutdown_event.is_set()++# Rate limiting for alerts - track last alert time per function+error combination_alert_rate_limiter = {}_rate_limiter_lock = threading.Lock()@@ -141,21 +165,39 @@ def create_retry_decorator(Returns:Configured retry decorator"""- if exclude_exceptions:- return retry(- stop=stop_after_attempt(max_attempts),- wait=wait_exponential_jitter(max=max_wait),- before_sleep=_create_retry_callback(context),- reraise=reraise,- retry=retry_if_not_exception_type(exclude_exceptions),+ stop = stop_after_attempt(max_attempts) | _StopOnShutdown()+ wait = wait_exponential_jitter(max=max_wait)+ before_sleep = _create_retry_callback(context)+ retry_on = (+ retry_if_not_exception_type(exclude_exceptions) if exclude_exceptions else None+ )++ def decorator(func):+ # Pick the interruptible sleep matching the wrapped function so a+ # pending backoff doesn't keep the process alive after a shutdown signal.+ sleep = (+ _interruptible_async_sleep+ if inspect.iscoroutinefunction(func)+ else _interruptible_sleep)- else:+ if retry_on is not None:+ return retry(+ sleep=sleep,+ stop=stop,+ wait=wait,+ before_sleep=before_sleep,+ reraise=reraise,+ retry=retry_on,+ )(func)return retry(- stop=stop_after_attempt(max_attempts),- wait=wait_exponential_jitter(max=max_wait),- before_sleep=_create_retry_callback(context),+ sleep=sleep,+ stop=stop,+ wait=wait,+ before_sleep=before_sleep,reraise=reraise,- )+ )(func)++ return decoratordef _log_prefix(resource_name: str, conn_id: str):@@ -207,16 +249,26 @@ def conn_retry()def decorator(func):- is_coroutine = asyncio.iscoroutinefunction(func)- # Use static retry configuration+ is_coroutine = inspect.iscoroutinefunction(func)+ # Use static retry configuration. _StopOnShutdown + the interruptible+ # sleep let an in-flight retry bail out promptly on SIGINT/SIGTERM+ # instead of blocking shutdown for up to max_retry * max_wait seconds.retry_decorator = retry(- stop=stop_after_attempt(max_retry + 1), # +1 for the initial attempt+ stop=stop_after_attempt(max_retry + 1) # +1 for the initial attempt+ | _StopOnShutdown(),wait=wait_exponential_jitter(max=max_wait),before_sleep=on_retry,reraise=True,+ sleep=_interruptible_async_sleep if is_coroutine else _interruptible_sleep,)wrapped_func = retry_decorator(func)+ def _log_failure(prefix: str, error: Exception):+ if is_shutting_down():+ logger.info(f"{prefix} {action_name} aborted: shutting down")+ else:+ logger.warning(f"{prefix} {action_name} failed after retries: {error}")+@wraps(func)def sync_wrapper(*args, **kwargs):prefix = _log_prefix(resource_name, conn_id)@@ -226,7 +278,7 @@ def conn_retry(logger.info(f"{prefix} {action_name} completed successfully.")return resultexcept Exception as e:- logger.warning(f"{prefix} {action_name} failed after retries: {e}")+ _log_failure(prefix, e)raise@wraps(func)@@ -238,7 +290,7 @@ def conn_retry(logger.info(f"{prefix} {action_name} completed successfully.")return resultexcept Exception as e:- logger.warning(f"{prefix} {action_name} failed after retries: {e}")+ _log_failure(prefix, e)raisereturn async_wrapper if is_coroutine else sync_wrapper@@ -246,18 +298,50 @@ def conn_retry(return decorator+class _StopOnShutdown(stop_base):+ """Tenacity stop condition that fires once a process shutdown is requested."""++ def __call__(self, retry_state) -> bool:+ return _shutdown_event.is_set()+++def _interruptible_sleep(seconds: float) -> None:+ """``time.sleep`` replacement that wakes immediately on shutdown.++ Worker threads don't receive SIGINT/SIGTERM (only the main thread does), so+ a plain ``time.sleep`` here can't be interrupted; waiting on the shutdown+ Event can.+ """+ _shutdown_event.wait(seconds)+++async def _interruptible_async_sleep(seconds: float) -> None:+ """``asyncio.sleep`` replacement that returns early on shutdown.++ Polls the shared shutdown flag so async retry loops on background event+ loops (which nothing cancels on shutdown) don't sit through a full backoff.+ """+ loop = asyncio.get_running_loop()+ end = loop.time() + seconds+ while not _shutdown_event.is_set():+ remaining = end - loop.time()+ if remaining <= 0:+ return+ await asyncio.sleep(min(remaining, _SHUTDOWN_POLL_INTERVAL))++# Preconfigured retry decorator for general functionsfunc_retry = create_retry_decorator(max_attempts=5)def continuous_retry(*, retry_delay: float = 1.0):def decorator(func):- is_coroutine = asyncio.iscoroutinefunction(func)+ is_coroutine = inspect.iscoroutinefunction(func)@wraps(func)def sync_wrapper(*args, **kwargs):counter = 0- while True:+ while not is_shutting_down():try:return func(*args, **kwargs)except Exception as exc:@@ -273,12 +357,13 @@ def continuous_retry(*, retry_delay: float = 1.0):str(exc) or type(exc).__name__,retry_delay,)- time.sleep(retry_delay)+ _interruptible_sleep(retry_delay)+ logger.info("%s stopped retrying: shutting down", func.__name__)@wraps(func)async def async_wrapper(*args, **kwargs):counter = 0- while True:+ while not is_shutting_down():try:return await func(*args, **kwargs)except Exception as exc:@@ -294,7 +379,8 @@ def continuous_retry(*, retry_delay: float = 1.0):str(exc) or type(exc).__name__,retry_delay,)- await asyncio.sleep(retry_delay)+ await _interruptible_async_sleep(retry_delay)+ logger.info("%s stopped retrying: shutting down", func.__name__)return async_wrapper if is_coroutine else sync_wrapperdiff --git a/autogpt_platform/backend/backend/util/retry_test.py b/autogpt_platform/backend/backend/util/retry_test.pyindex 092481caca..16339479b9 100644--- a/autogpt_platform/backend/backend/util/retry_test.py+++ b/autogpt_platform/backend/backend/util/retry_test.py@@ -8,14 +8,28 @@ import pytestfrom backend.util.retry import (ALERT_RATE_LIMIT_SECONDS,_alert_rate_limiter,+ _interruptible_async_sleep,+ _interruptible_sleep,_rate_limiter_lock,_send_critical_retry_alert,+ _shutdown_event,conn_retry,+ continuous_retry,create_retry_decorator,+ is_shutting_down,should_send_alert,+ stop_retry_loops,)[email protected](autouse=True)+def _reset_shutdown_event():+ """Keep the process-wide shutdown flag from leaking between tests."""+ _shutdown_event.clear()+ yield+ _shutdown_event.clear()++def test_conn_retry_sync_function():retry_count = 0@@ -60,6 +74,109 @@ async def test_conn_retry_async_function():assert str(e.value) == "Test error"+def test_conn_retry_sync_aborts_on_shutdown():+ """A shutdown request must stop the retry loop instead of exhausting it."""+ attempts = 0++ @conn_retry("Test", "Test function", max_retry=100, max_wait=30)+ def test_function():+ nonlocal attempts+ attempts += 1+ stop_retry_loops() # signal shutdown mid-flight+ raise ConnectionError("Redis down")++ start = time.monotonic()+ with pytest.raises(ConnectionError):+ test_function()+ # Bails out near-immediately rather than running ~101 attempts / waiting 30s.+ assert attempts < 5+ assert time.monotonic() - start < 5+++async def test_conn_retry_async_aborts_on_shutdown():+ attempts = 0++ @conn_retry("Test", "Test function", max_retry=100, max_wait=30)+ async def test_function():+ nonlocal attempts+ attempts += 1+ stop_retry_loops()+ raise ConnectionError("Redis down")++ start = time.monotonic()+ with pytest.raises(ConnectionError):+ await test_function()+ assert attempts < 5+ assert time.monotonic() - start < 5+++def test_create_retry_decorator_aborts_on_shutdown():+ attempts = 0++ @create_retry_decorator(max_attempts=100, max_wait=30)+ def always_failing_function():+ nonlocal attempts+ attempts += 1+ stop_retry_loops()+ raise ValueError("persistent failure")++ with pytest.raises(ValueError):+ always_failing_function()+ assert attempts < 5+++def test_interruptible_sleep_returns_early_on_shutdown():+ stop_retry_loops()+ assert is_shutting_down()+ start = time.monotonic()+ _interruptible_sleep(30)+ assert time.monotonic() - start < 1+++async def test_interruptible_async_sleep_returns_early_on_shutdown():+ stop_retry_loops()+ start = time.monotonic()+ await _interruptible_async_sleep(30)+ assert time.monotonic() - start < 1+++def test_continuous_retry_sync_stops_on_shutdown():+ """The infinite continuous_retry loop must exit when shutdown is requested."""+ attempts = 0++ @continuous_retry(retry_delay=30)+ def always_failing_function():+ nonlocal attempts+ attempts += 1+ stop_retry_loops()+ raise RuntimeError("never recovers")++ start = time.monotonic()+ # Returns instead of looping forever; no exception propagates.+ assert always_failing_function() is None+ assert attempts < 5+ assert time.monotonic() - start < 5+++async def test_continuous_retry_async_stops_on_shutdown():+ attempts = 0++ @continuous_retry(retry_delay=30)+ async def always_failing_function():+ nonlocal attempts+ attempts += 1+ stop_retry_loops()+ raise RuntimeError("never recovers")++ start = time.monotonic()+ assert await always_failing_function() is None+ assert attempts < 5+ assert time.monotonic() - start < 5++class TestRetryRateLimiting:"""Test the rate limiting functionality for critical retry alerts."""diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 5b218f25ea..cf9488036a 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -39,7 +39,7 @@ from backend.monitoring.instrumentation import instrument_fastapifrom backend.util.json import to_dictfrom backend.util.metrics import sentry_initfrom backend.util.process import AppProcess-from backend.util.retry import conn_retry, create_retry_decorator+from backend.util.retry import conn_retry, create_retry_decorator, stop_retry_loopsfrom backend.util.settings import Config, get_service_namelogger = logging.getLogger(__name__)@@ -367,6 +367,7 @@ class AppService(BaseAppService, ABC):def _self_terminate(self, signum: int, frame):"""Pass SIGTERM to Uvicorn so it can shut down gracefully"""+ stop_retry_loops()signame = signal.Signals(signum).nameif not self._shutting_down:self._shutting_down = True