AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): retry transient redis cluster errors internally + downgrade lock-blip noise (#13076)
Zamil Majdy12cc0f6db590
diff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.pyindex 048f63ea69..9e32ffcbbd 100644--- a/autogpt_platform/backend/backend/data/redis_client.py+++ b/autogpt_platform/backend/backend/data/redis_client.py@@ -7,7 +7,14 @@ from redis import Redisfrom redis.asyncio import Redis as AsyncRedisfrom redis.asyncio.cluster import ClusterNode as AsyncClusterNodefrom redis.asyncio.cluster import RedisCluster as AsyncRedisCluster+from redis.asyncio.retry import Retry as AsyncRetry+from redis.backoff import ExponentialBackofffrom redis.cluster import ClusterNode, RedisCluster+from redis.exceptions import ClusterDownError+from redis.exceptions import ConnectionError as RedisConnectionError+from redis.exceptions import RedisError+from redis.exceptions import TimeoutError as RedisTimeoutError+from redis.retry import Retryfrom backend.util.cache import cachedfrom backend.util.retry import conn_retry@@ -35,6 +42,17 @@ USE_ANNOUNCED_ADDRESS = os.getenv("REDIS_USE_ANNOUNCED_ADDRESS", "").lower() in"yes",)+# Retry transient cluster errors internally so a rotation blip never surfaces+# as a graph-exec 500.+REDIS_RETRY_ATTEMPTS = int(os.getenv("REDIS_RETRY_ATTEMPTS", "5"))++# Errors that should trigger a per-command retry inside the redis-py client.+TRANSIENT_REDIS_ERRORS: tuple[type[RedisError], ...] = (+ RedisConnectionError,+ RedisTimeoutError,+ ClusterDownError,+)+logger = logging.getLogger(__name__)# Aliases so call-sites don't care which class this is.@@ -42,6 +60,25 @@ RedisClient = RedisClusterAsyncRedisClient = AsyncRedisCluster+def _build_retry() -> Retry:+ """Per-command retry with exponential backoff (0.1s → 10s cap)."""+ return Retry(+ backoff=ExponentialBackoff(cap=10, base=0.1),+ retries=REDIS_RETRY_ATTEMPTS,+ supported_errors=TRANSIENT_REDIS_ERRORS,+ )+++def _build_async_retry() -> AsyncRetry:+ """Async sibling of :func:`_build_retry` — redis-py ships separate sync+ and async ``Retry`` classes; passing the wrong one fails type checks."""+ return AsyncRetry(+ backoff=ExponentialBackoff(cap=10, base=0.1),+ retries=REDIS_RETRY_ATTEMPTS,+ supported_errors=TRANSIENT_REDIS_ERRORS,+ )++def _address_remap(addr: tuple[str, int]) -> tuple[str, int]:"""Pin each shard to the seed `HOST`, keep its announced port.@@ -65,6 +102,8 @@ def connect() -> RedisClient:socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,address_remap=_address_remap,+ # Drives both per-command retries and the cluster-level retry counter.+ retry=_build_retry(),)# Close on PING failure so retries don't leak ClusterNodes (AUTOGPT-SERVER-8T1).try:@@ -100,6 +139,10 @@ async def connect_async() -> AsyncRedisClient:socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,address_remap=_address_remap,+ # redis-py 6.x AsyncRedisCluster ignores `retry_on_error` — the cluster+ # retry path uses a hardcoded {Timeout, Connection, ClusterDown} set.+ # Pass `retry` only to match the sync RedisCluster call above.+ retry=_build_async_retry(),)# Close on PING failure so retries don't leak ClusterNodes (AUTOGPT-SERVER-8V6/8V4/8V3).try:diff --git a/autogpt_platform/backend/backend/data/redis_client_test.py b/autogpt_platform/backend/backend/data/redis_client_test.pyindex ef2a6a1e25..a83c1a0729 100644--- a/autogpt_platform/backend/backend/data/redis_client_test.py+++ b/autogpt_platform/backend/backend/data/redis_client_test.py@@ -7,7 +7,12 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom redis.asyncio.cluster import RedisCluster as AsyncRedisCluster+from redis.asyncio.retry import Retry as AsyncRetryfrom redis.cluster import RedisCluster+from redis.exceptions import ClusterDownError+from redis.exceptions import ConnectionError as RedisConnectionError+from redis.exceptions import TimeoutError as RedisTimeoutError+from redis.retry import Retryimport backend.data.redis_client as redis_client@@ -42,6 +47,23 @@ def test_connect_builds_redis_cluster() -> None:client.ping.assert_called_once()+def test_connect_configures_retry_on_transient_errors() -> None:+ """Sync RedisCluster must carry a Retry that fires on transient errors,+ so a shard rotation blip retries internally instead of surfacing as 500."""+ with patch.object(redis_client, "RedisCluster", autospec=True) as mock_cluster:+ mock_cluster.return_value = MagicMock(spec=RedisCluster)+ redis_client.connect()++ kwargs = mock_cluster.call_args.kwargs+ retry = kwargs["retry"]+ assert isinstance(retry, Retry)+ assert retry.get_retries() == redis_client.REDIS_RETRY_ATTEMPTS+ supported = set(retry._supported_errors)+ assert RedisConnectionError in supported+ assert RedisTimeoutError in supported+ assert ClusterDownError in supported++def test_address_remap_pins_host_and_preserves_port() -> None:"""Default remap rewrites announced shard host to the configured seed."""with patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", False):@@ -57,6 +79,32 @@ def test_address_remap_passthrough_when_use_announced_address() -> None:assert redis_client._address_remap(("redis-1", 17001)) == ("redis-1", 17001)+async def test_connect_async_configures_retry() -> None:+ """Async RedisCluster must carry an ``AsyncRetry`` keyed on the same+ transient errors as the sync path. ``retry_on_error`` is intentionally+ omitted — redis-py 6.x ignores it for cluster operations (the cluster+ retry path uses a hardcoded {Timeout, Connection, ClusterDown} set)."""+ with patch.object(redis_client, "AsyncRedisCluster", autospec=True) as mock_cluster:+ fake = MagicMock(spec=AsyncRedisCluster)+ fake.ping = AsyncMock()+ mock_cluster.return_value = fake+ await redis_client.connect_async()++ kwargs = mock_cluster.call_args.kwargs+ retry = kwargs["retry"]+ # redis-py's async cluster uses a separate AsyncRetry class — passing the+ # sync `Retry` would type-fail at construction time.+ assert isinstance(retry, AsyncRetry)+ assert retry.get_retries() == redis_client.REDIS_RETRY_ATTEMPTS+ supported = set(retry._supported_errors)+ assert RedisConnectionError in supported+ assert RedisTimeoutError in supported+ assert ClusterDownError in supported+ # No `retry_on_error` kwarg — it's ineffective on AsyncRedisCluster.+ assert "retry_on_error" not in kwargs++@pytest.mark.asyncioasync def test_connect_async_builds_async_redis_cluster() -> None:with patch.object(redis_client, "AsyncRedisCluster", autospec=True) as mock_cluster:diff --git a/autogpt_platform/backend/backend/executor/cluster_lock.py b/autogpt_platform/backend/backend/executor/cluster_lock.pyindex de9305350a..d38e0d0118 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock.py@@ -6,11 +6,43 @@ import threadingimport timefrom typing import TYPE_CHECKING, Any, cast+from redis.exceptions import ClusterDownError+from redis.exceptions import ConnectionError as RedisConnectionError+from redis.exceptions import TimeoutError as RedisTimeoutError+if TYPE_CHECKING:from backend.data.redis_client import AsyncRedisClient, RedisClientlogger = logging.getLogger(__name__)+# Transient redis errors retried internally by the client; if they still+# surface here, retries are exhausted — log at warning to keep Sentry quiet+# during rotation windows.+_TRANSIENT_LOCK_ERRORS: tuple[type[BaseException], ...] = (+ RedisConnectionError,+ RedisTimeoutError,+ ClusterDownError,+)++# redis-py's reconnect path dereferences `.host` / `.port` on a plain+# ConnectionError, surfacing as AttributeError with one of these signatures.+# Only suppress AttributeError when the message matches — a typo on a+# ClusterLock attribute must still bubble up at ERROR.+_REDIS_RECONNECT_ATTRS = ("'host'", "'port'", "'connection_pool'")+++def _is_transient_redis_error(exc: BaseException) -> bool:+ """True iff ``exc`` is a transient redis-cluster error worth retrying+ or downgrading to WARNING. Generic ``AttributeError`` is only accepted+ when its message matches the redis-py reconnect signature."""+ if isinstance(exc, _TRANSIENT_LOCK_ERRORS):+ return True+ if isinstance(exc, AttributeError):+ msg = str(exc)+ return any(attr in msg for attr in _REDIS_RECONNECT_ATTRS)+ return False++# CAS release: DEL only when the stored owner still matches — guards against# wiping a successor's lock after an external force-release._RELEASE_LUA = (@@ -62,7 +94,13 @@ class ClusterLock:return Noneexcept Exception as e:- logger.error(f"ClusterLock.try_acquire failed for key {self.key}: {e}")+ if _is_transient_redis_error(e):+ logger.warning(+ f"ClusterLock.try_acquire transient redis error for key "+ f"{self.key}: {type(e).__name__}: {e}"+ )+ else:+ logger.error(f"ClusterLock.try_acquire failed for key {self.key}: {e}")return Nonedef refresh(self) -> bool:@@ -119,7 +157,13 @@ class ClusterLock:return Falseexcept Exception as e:- logger.error(f"ClusterLock.refresh failed for key {self.key}: {e}")+ if _is_transient_redis_error(e):+ logger.warning(+ f"ClusterLock.refresh transient redis error for key "+ f"{self.key}: {type(e).__name__}: {e}"+ )+ else:+ logger.error(f"ClusterLock.refresh failed for key {self.key}: {e}")with self._refresh_lock:self._last_refresh = 0return False@@ -188,7 +232,15 @@ class AsyncClusterLock:return Noneexcept Exception as e:- logger.error(f"AsyncClusterLock.try_acquire failed for key {self.key}: {e}")+ if _is_transient_redis_error(e):+ logger.warning(+ f"AsyncClusterLock.try_acquire transient redis error for key "+ f"{self.key}: {type(e).__name__}: {e}"+ )+ else:+ logger.error(+ f"AsyncClusterLock.try_acquire failed for key {self.key}: {e}"+ )return Noneasync def refresh(self) -> bool:@@ -245,7 +297,13 @@ class AsyncClusterLock:return Falseexcept Exception as e:- logger.error(f"AsyncClusterLock.refresh failed for key {self.key}: {e}")+ if _is_transient_redis_error(e):+ logger.warning(+ f"AsyncClusterLock.refresh transient redis error for key "+ f"{self.key}: {type(e).__name__}: {e}"+ )+ else:+ logger.error(f"AsyncClusterLock.refresh failed for key {self.key}: {e}")async with self._refresh_lock:self._last_refresh = 0return Falsediff --git a/autogpt_platform/backend/backend/executor/cluster_lock_test.py b/autogpt_platform/backend/backend/executor/cluster_lock_test.pyindex 137209e410..c9b970018c 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock_test.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock_test.py@@ -10,11 +10,15 @@ import loggingimport timeimport uuidfrom threading import Thread+from unittest.mock import AsyncMock, MagicMockimport pytestimport redis+from redis.exceptions import ClusterDownError+from redis.exceptions import ConnectionError as RedisConnectionError+from redis.exceptions import TimeoutError as RedisTimeoutError-from .cluster_lock import ClusterLock+from .cluster_lock import AsyncClusterLock, ClusterLocklogger = logging.getLogger(__name__)@@ -536,6 +540,150 @@ class TestClusterLockRealWorldScenarios:assert new_lock.try_acquire() == owner_id+class TestClusterLockTransientErrorHandling:+ """Transient redis-cluster errors must degrade gracefully — try_acquire+ returns None, refresh returns False — and must be logged at WARNING, not+ ERROR (so Sentry isn't flooded during a shard rotation)."""++ @pytest.mark.parametrize(+ "exc",+ [+ RedisConnectionError("connection lost"),+ RedisTimeoutError("read timeout"),+ ClusterDownError("CLUSTERDOWN The cluster is down"),+ # redis-py's reconnect path dereferences `.host` on plain+ # ConnectionError, surfacing as AttributeError — must also be+ # treated as transient.+ AttributeError("'ConnectionError' object has no attribute 'host'"),+ ],+ )+ def test_try_acquire_returns_none_on_transient_error(self, exc, caplog):+ fake_redis = MagicMock()+ fake_redis.set.side_effect = exc+ lock_key = f"test_lock:{uuid.uuid4()}"+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)++ with caplog.at_level(logging.WARNING):+ assert lock.try_acquire() is None++ assert lock._last_refresh == 0+ # Must be WARNING, not ERROR — that's the whole point of this branch.+ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]+ assert records, "expected a try_acquire log record"+ assert all(r.levelno == logging.WARNING for r in records)++ @pytest.mark.parametrize(+ "exc",+ [+ RedisConnectionError("connection lost"),+ RedisTimeoutError("read timeout"),+ ClusterDownError("CLUSTERDOWN The cluster is down"),+ AttributeError("'ConnectionError' object has no attribute 'host'"),+ ],+ )+ def test_refresh_returns_false_on_transient_error(self, exc, caplog):+ fake_redis = MagicMock()+ fake_redis.get.side_effect = exc+ lock_key = f"test_lock:{uuid.uuid4()}"+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)+ # Pretend we own the lock so refresh runs the get() path.+ lock._last_refresh = time.time() - 1000++ with caplog.at_level(logging.WARNING):+ assert lock.refresh() is False++ assert lock._last_refresh == 0+ records = [r for r in caplog.records if "refresh" in r.getMessage()]+ assert records, "expected a refresh log record"+ assert all(r.levelno == logging.WARNING for r in records)++ def test_try_acquire_unknown_error_logged_at_error(self, caplog):+ """A non-transient exception must still log at ERROR so we notice."""+ fake_redis = MagicMock()+ fake_redis.set.side_effect = RuntimeError("totally unexpected")+ lock_key = f"test_lock:{uuid.uuid4()}"+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)++ with caplog.at_level(logging.WARNING):+ assert lock.try_acquire() is None++ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]+ assert records, "expected a try_acquire log record"+ assert any(r.levelno == logging.ERROR for r in records)++ def test_try_acquire_unrelated_attribute_error_logs_at_error(self, caplog):+ """A bare ``AttributeError`` whose message doesn't match the redis-py+ reconnect signature must surface at ERROR — a typo on a ClusterLock+ attribute would otherwise be silently swallowed as a 'transient' error."""+ fake_redis = MagicMock()+ fake_redis.set.side_effect = AttributeError(+ "'ClusterLock' object has no attribute 'tymeout'"+ )+ lock_key = f"test_lock:{uuid.uuid4()}"+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)++ with caplog.at_level(logging.WARNING):+ assert lock.try_acquire() is None++ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]+ assert records, "expected a try_acquire log record"+ assert any(r.levelno == logging.ERROR for r in records)+ assert not any(r.levelno == logging.WARNING for r in records)+++class TestAsyncClusterLockTransientErrorHandling:+ """Async sibling of ``TestClusterLockTransientErrorHandling``."""++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "exc",+ [+ RedisConnectionError("connection lost"),+ RedisTimeoutError("read timeout"),+ ClusterDownError("CLUSTERDOWN The cluster is down"),+ AttributeError("'ConnectionError' object has no attribute 'host'"),+ ],+ )+ async def test_try_acquire_returns_none_on_transient_error(self, exc, caplog):+ fake_redis = MagicMock()+ fake_redis.set = AsyncMock(side_effect=exc)+ lock_key = f"test_lock:{uuid.uuid4()}"+ lock = AsyncClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)++ with caplog.at_level(logging.WARNING):+ assert await lock.try_acquire() is None++ assert lock._last_refresh == 0+ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]+ assert records, "expected a try_acquire log record"+ assert all(r.levelno == logging.WARNING for r in records)++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "exc",+ [+ RedisConnectionError("connection lost"),+ RedisTimeoutError("read timeout"),+ ClusterDownError("CLUSTERDOWN The cluster is down"),+ AttributeError("'ConnectionError' object has no attribute 'host'"),+ ],+ )+ async def test_refresh_returns_false_on_transient_error(self, exc, caplog):+ fake_redis = MagicMock()+ fake_redis.get = AsyncMock(side_effect=exc)+ lock_key = f"test_lock:{uuid.uuid4()}"+ lock = AsyncClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)+ lock._last_refresh = time.time() - 1000++ with caplog.at_level(logging.WARNING):+ assert await lock.refresh() is False++ assert lock._last_refresh == 0+ records = [r for r in caplog.records if "refresh" in r.getMessage()]+ assert records, "expected a refresh log record"+ assert all(r.levelno == logging.WARNING for r in records)++if __name__ == "__main__":# Run specific test for quick validationpytest.main([__file__, "-v"])