Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/executor): retry transient redis cluster errors internally + downgrade lock-blip noise (#13076)

Zamil Majdy12cc0f6db590
diff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.py
index 048f63ea69..9e32ffcbbd 100644
--- a/autogpt_platform/backend/backend/data/redis_client.py
+++ b/autogpt_platform/backend/backend/data/redis_client.py
@@ -7,7 +7,14 @@ from redis import Redis
from redis.asyncio import Redis as AsyncRedis
from redis.asyncio.cluster import ClusterNode as AsyncClusterNode
from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster
+from redis.asyncio.retry import Retry as AsyncRetry
+from redis.backoff import ExponentialBackoff
from redis.cluster import ClusterNode, RedisCluster
+from redis.exceptions import ClusterDownError
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import RedisError
+from redis.exceptions import TimeoutError as RedisTimeoutError
+from redis.retry import Retry
from backend.util.cache import cached
from backend.util.retry import conn_retry
@@ -35,6 +42,17 @@ USE_ANNOUNCED_ADDRESS = os.getenv("REDIS_USE_ANNOUNCED_ADDRESS", "").lower() in
"yes",
)
+# Retry transient cluster errors internally so a rotation blip never surfaces
+# as a graph-exec 500.
+REDIS_RETRY_ATTEMPTS = int(os.getenv("REDIS_RETRY_ATTEMPTS", "5"))
+
+# Errors that should trigger a per-command retry inside the redis-py client.
+TRANSIENT_REDIS_ERRORS: tuple[type[RedisError], ...] = (
+ RedisConnectionError,
+ RedisTimeoutError,
+ ClusterDownError,
+)
+
logger = logging.getLogger(__name__)
# Aliases so call-sites don't care which class this is.
@@ -42,6 +60,25 @@ RedisClient = RedisCluster
AsyncRedisClient = AsyncRedisCluster
+def _build_retry() -> Retry:
+ """Per-command retry with exponential backoff (0.1s → 10s cap)."""
+ return Retry(
+ backoff=ExponentialBackoff(cap=10, base=0.1),
+ retries=REDIS_RETRY_ATTEMPTS,
+ supported_errors=TRANSIENT_REDIS_ERRORS,
+ )
+
+
+def _build_async_retry() -> AsyncRetry:
+ """Async sibling of :func:`_build_retry` — redis-py ships separate sync
+ and async ``Retry`` classes; passing the wrong one fails type checks."""
+ return AsyncRetry(
+ backoff=ExponentialBackoff(cap=10, base=0.1),
+ retries=REDIS_RETRY_ATTEMPTS,
+ supported_errors=TRANSIENT_REDIS_ERRORS,
+ )
+
+
def _address_remap(addr: tuple[str, int]) -> tuple[str, int]:
"""Pin each shard to the seed `HOST`, keep its announced port.
@@ -65,6 +102,8 @@ def connect() -> RedisClient:
socket_keepalive=True,
health_check_interval=HEALTH_CHECK_INTERVAL,
address_remap=_address_remap,
+ # Drives both per-command retries and the cluster-level retry counter.
+ retry=_build_retry(),
)
# Close on PING failure so retries don't leak ClusterNodes (AUTOGPT-SERVER-8T1).
try:
@@ -100,6 +139,10 @@ async def connect_async() -> AsyncRedisClient:
socket_keepalive=True,
health_check_interval=HEALTH_CHECK_INTERVAL,
address_remap=_address_remap,
+ # redis-py 6.x AsyncRedisCluster ignores `retry_on_error` — the cluster
+ # retry path uses a hardcoded {Timeout, Connection, ClusterDown} set.
+ # Pass `retry` only to match the sync RedisCluster call above.
+ retry=_build_async_retry(),
)
# Close on PING failure so retries don't leak ClusterNodes (AUTOGPT-SERVER-8V6/8V4/8V3).
try:
diff --git a/autogpt_platform/backend/backend/data/redis_client_test.py b/autogpt_platform/backend/backend/data/redis_client_test.py
index ef2a6a1e25..a83c1a0729 100644
--- a/autogpt_platform/backend/backend/data/redis_client_test.py
+++ b/autogpt_platform/backend/backend/data/redis_client_test.py
@@ -7,7 +7,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster
+from redis.asyncio.retry import Retry as AsyncRetry
from redis.cluster import RedisCluster
+from redis.exceptions import ClusterDownError
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import TimeoutError as RedisTimeoutError
+from redis.retry import Retry
import backend.data.redis_client as redis_client
@@ -42,6 +47,23 @@ def test_connect_builds_redis_cluster() -> None:
client.ping.assert_called_once()
+def test_connect_configures_retry_on_transient_errors() -> None:
+ """Sync RedisCluster must carry a Retry that fires on transient errors,
+ so a shard rotation blip retries internally instead of surfacing as 500."""
+ with patch.object(redis_client, "RedisCluster", autospec=True) as mock_cluster:
+ mock_cluster.return_value = MagicMock(spec=RedisCluster)
+ redis_client.connect()
+
+ kwargs = mock_cluster.call_args.kwargs
+ retry = kwargs["retry"]
+ assert isinstance(retry, Retry)
+ assert retry.get_retries() == redis_client.REDIS_RETRY_ATTEMPTS
+ supported = set(retry._supported_errors)
+ assert RedisConnectionError in supported
+ assert RedisTimeoutError in supported
+ assert ClusterDownError in supported
+
+
def test_address_remap_pins_host_and_preserves_port() -> None:
"""Default remap rewrites announced shard host to the configured seed."""
with patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", False):
@@ -57,6 +79,32 @@ def test_address_remap_passthrough_when_use_announced_address() -> None:
assert redis_client._address_remap(("redis-1", 17001)) == ("redis-1", 17001)
+async def test_connect_async_configures_retry() -> None:
+ """Async RedisCluster must carry an ``AsyncRetry`` keyed on the same
+ transient errors as the sync path. ``retry_on_error`` is intentionally
+ omitted — redis-py 6.x ignores it for cluster operations (the cluster
+ retry path uses a hardcoded {Timeout, Connection, ClusterDown} set)."""
+ with patch.object(redis_client, "AsyncRedisCluster", autospec=True) as mock_cluster:
+ fake = MagicMock(spec=AsyncRedisCluster)
+ fake.ping = AsyncMock()
+ mock_cluster.return_value = fake
+ await redis_client.connect_async()
+
+ kwargs = mock_cluster.call_args.kwargs
+ retry = kwargs["retry"]
+ # redis-py's async cluster uses a separate AsyncRetry class — passing the
+ # sync `Retry` would type-fail at construction time.
+ assert isinstance(retry, AsyncRetry)
+ assert retry.get_retries() == redis_client.REDIS_RETRY_ATTEMPTS
+ supported = set(retry._supported_errors)
+ assert RedisConnectionError in supported
+ assert RedisTimeoutError in supported
+ assert ClusterDownError in supported
+ # No `retry_on_error` kwarg — it's ineffective on AsyncRedisCluster.
+ assert "retry_on_error" not in kwargs
+
+
@pytest.mark.asyncio
async def test_connect_async_builds_async_redis_cluster() -> None:
with patch.object(redis_client, "AsyncRedisCluster", autospec=True) as mock_cluster:
diff --git a/autogpt_platform/backend/backend/executor/cluster_lock.py b/autogpt_platform/backend/backend/executor/cluster_lock.py
index de9305350a..d38e0d0118 100644
--- a/autogpt_platform/backend/backend/executor/cluster_lock.py
+++ b/autogpt_platform/backend/backend/executor/cluster_lock.py
@@ -6,11 +6,43 @@ import threading
import time
from typing import TYPE_CHECKING, Any, cast
+from redis.exceptions import ClusterDownError
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import TimeoutError as RedisTimeoutError
+
if TYPE_CHECKING:
from backend.data.redis_client import AsyncRedisClient, RedisClient
logger = logging.getLogger(__name__)
+# Transient redis errors retried internally by the client; if they still
+# surface here, retries are exhausted — log at warning to keep Sentry quiet
+# during rotation windows.
+_TRANSIENT_LOCK_ERRORS: tuple[type[BaseException], ...] = (
+ RedisConnectionError,
+ RedisTimeoutError,
+ ClusterDownError,
+)
+
+# redis-py's reconnect path dereferences `.host` / `.port` on a plain
+# ConnectionError, surfacing as AttributeError with one of these signatures.
+# Only suppress AttributeError when the message matches — a typo on a
+# ClusterLock attribute must still bubble up at ERROR.
+_REDIS_RECONNECT_ATTRS = ("'host'", "'port'", "'connection_pool'")
+
+
+def _is_transient_redis_error(exc: BaseException) -> bool:
+ """True iff ``exc`` is a transient redis-cluster error worth retrying
+ or downgrading to WARNING. Generic ``AttributeError`` is only accepted
+ when its message matches the redis-py reconnect signature."""
+ if isinstance(exc, _TRANSIENT_LOCK_ERRORS):
+ return True
+ if isinstance(exc, AttributeError):
+ msg = str(exc)
+ return any(attr in msg for attr in _REDIS_RECONNECT_ATTRS)
+ return False
+
+
# CAS release: DEL only when the stored owner still matches — guards against
# wiping a successor's lock after an external force-release.
_RELEASE_LUA = (
@@ -62,7 +94,13 @@ class ClusterLock:
return None
except Exception as e:
- logger.error(f"ClusterLock.try_acquire failed for key {self.key}: {e}")
+ if _is_transient_redis_error(e):
+ logger.warning(
+ f"ClusterLock.try_acquire transient redis error for key "
+ f"{self.key}: {type(e).__name__}: {e}"
+ )
+ else:
+ logger.error(f"ClusterLock.try_acquire failed for key {self.key}: {e}")
return None
def refresh(self) -> bool:
@@ -119,7 +157,13 @@ class ClusterLock:
return False
except Exception as e:
- logger.error(f"ClusterLock.refresh failed for key {self.key}: {e}")
+ if _is_transient_redis_error(e):
+ logger.warning(
+ f"ClusterLock.refresh transient redis error for key "
+ f"{self.key}: {type(e).__name__}: {e}"
+ )
+ else:
+ logger.error(f"ClusterLock.refresh failed for key {self.key}: {e}")
with self._refresh_lock:
self._last_refresh = 0
return False
@@ -188,7 +232,15 @@ class AsyncClusterLock:
return None
except Exception as e:
- logger.error(f"AsyncClusterLock.try_acquire failed for key {self.key}: {e}")
+ if _is_transient_redis_error(e):
+ logger.warning(
+ f"AsyncClusterLock.try_acquire transient redis error for key "
+ f"{self.key}: {type(e).__name__}: {e}"
+ )
+ else:
+ logger.error(
+ f"AsyncClusterLock.try_acquire failed for key {self.key}: {e}"
+ )
return None
async def refresh(self) -> bool:
@@ -245,7 +297,13 @@ class AsyncClusterLock:
return False
except Exception as e:
- logger.error(f"AsyncClusterLock.refresh failed for key {self.key}: {e}")
+ if _is_transient_redis_error(e):
+ logger.warning(
+ f"AsyncClusterLock.refresh transient redis error for key "
+ f"{self.key}: {type(e).__name__}: {e}"
+ )
+ else:
+ logger.error(f"AsyncClusterLock.refresh failed for key {self.key}: {e}")
async with self._refresh_lock:
self._last_refresh = 0
return False
diff --git a/autogpt_platform/backend/backend/executor/cluster_lock_test.py b/autogpt_platform/backend/backend/executor/cluster_lock_test.py
index 137209e410..c9b970018c 100644
--- a/autogpt_platform/backend/backend/executor/cluster_lock_test.py
+++ b/autogpt_platform/backend/backend/executor/cluster_lock_test.py
@@ -10,11 +10,15 @@ import logging
import time
import uuid
from threading import Thread
+from unittest.mock import AsyncMock, MagicMock
import pytest
import redis
+from redis.exceptions import ClusterDownError
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import TimeoutError as RedisTimeoutError
-from .cluster_lock import ClusterLock
+from .cluster_lock import AsyncClusterLock, ClusterLock
logger = logging.getLogger(__name__)
@@ -536,6 +540,150 @@ class TestClusterLockRealWorldScenarios:
assert new_lock.try_acquire() == owner_id
+class TestClusterLockTransientErrorHandling:
+ """Transient redis-cluster errors must degrade gracefully — try_acquire
+ returns None, refresh returns False — and must be logged at WARNING, not
+ ERROR (so Sentry isn't flooded during a shard rotation)."""
+
+ @pytest.mark.parametrize(
+ "exc",
+ [
+ RedisConnectionError("connection lost"),
+ RedisTimeoutError("read timeout"),
+ ClusterDownError("CLUSTERDOWN The cluster is down"),
+ # redis-py's reconnect path dereferences `.host` on plain
+ # ConnectionError, surfacing as AttributeError — must also be
+ # treated as transient.
+ AttributeError("'ConnectionError' object has no attribute 'host'"),
+ ],
+ )
+ def test_try_acquire_returns_none_on_transient_error(self, exc, caplog):
+ fake_redis = MagicMock()
+ fake_redis.set.side_effect = exc
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)
+
+ with caplog.at_level(logging.WARNING):
+ assert lock.try_acquire() is None
+
+ assert lock._last_refresh == 0
+ # Must be WARNING, not ERROR — that's the whole point of this branch.
+ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]
+ assert records, "expected a try_acquire log record"
+ assert all(r.levelno == logging.WARNING for r in records)
+
+ @pytest.mark.parametrize(
+ "exc",
+ [
+ RedisConnectionError("connection lost"),
+ RedisTimeoutError("read timeout"),
+ ClusterDownError("CLUSTERDOWN The cluster is down"),
+ AttributeError("'ConnectionError' object has no attribute 'host'"),
+ ],
+ )
+ def test_refresh_returns_false_on_transient_error(self, exc, caplog):
+ fake_redis = MagicMock()
+ fake_redis.get.side_effect = exc
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)
+ # Pretend we own the lock so refresh runs the get() path.
+ lock._last_refresh = time.time() - 1000
+
+ with caplog.at_level(logging.WARNING):
+ assert lock.refresh() is False
+
+ assert lock._last_refresh == 0
+ records = [r for r in caplog.records if "refresh" in r.getMessage()]
+ assert records, "expected a refresh log record"
+ assert all(r.levelno == logging.WARNING for r in records)
+
+ def test_try_acquire_unknown_error_logged_at_error(self, caplog):
+ """A non-transient exception must still log at ERROR so we notice."""
+ fake_redis = MagicMock()
+ fake_redis.set.side_effect = RuntimeError("totally unexpected")
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)
+
+ with caplog.at_level(logging.WARNING):
+ assert lock.try_acquire() is None
+
+ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]
+ assert records, "expected a try_acquire log record"
+ assert any(r.levelno == logging.ERROR for r in records)
+
+ def test_try_acquire_unrelated_attribute_error_logs_at_error(self, caplog):
+ """A bare ``AttributeError`` whose message doesn't match the redis-py
+ reconnect signature must surface at ERROR — a typo on a ClusterLock
+ attribute would otherwise be silently swallowed as a 'transient' error."""
+ fake_redis = MagicMock()
+ fake_redis.set.side_effect = AttributeError(
+ "'ClusterLock' object has no attribute 'tymeout'"
+ )
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ lock = ClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)
+
+ with caplog.at_level(logging.WARNING):
+ assert lock.try_acquire() is None
+
+ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]
+ assert records, "expected a try_acquire log record"
+ assert any(r.levelno == logging.ERROR for r in records)
+ assert not any(r.levelno == logging.WARNING for r in records)
+
+
+class TestAsyncClusterLockTransientErrorHandling:
+ """Async sibling of ``TestClusterLockTransientErrorHandling``."""
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "exc",
+ [
+ RedisConnectionError("connection lost"),
+ RedisTimeoutError("read timeout"),
+ ClusterDownError("CLUSTERDOWN The cluster is down"),
+ AttributeError("'ConnectionError' object has no attribute 'host'"),
+ ],
+ )
+ async def test_try_acquire_returns_none_on_transient_error(self, exc, caplog):
+ fake_redis = MagicMock()
+ fake_redis.set = AsyncMock(side_effect=exc)
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ lock = AsyncClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)
+
+ with caplog.at_level(logging.WARNING):
+ assert await lock.try_acquire() is None
+
+ assert lock._last_refresh == 0
+ records = [r for r in caplog.records if "try_acquire" in r.getMessage()]
+ assert records, "expected a try_acquire log record"
+ assert all(r.levelno == logging.WARNING for r in records)
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "exc",
+ [
+ RedisConnectionError("connection lost"),
+ RedisTimeoutError("read timeout"),
+ ClusterDownError("CLUSTERDOWN The cluster is down"),
+ AttributeError("'ConnectionError' object has no attribute 'host'"),
+ ],
+ )
+ async def test_refresh_returns_false_on_transient_error(self, exc, caplog):
+ fake_redis = MagicMock()
+ fake_redis.get = AsyncMock(side_effect=exc)
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ lock = AsyncClusterLock(fake_redis, lock_key, str(uuid.uuid4()), timeout=60)
+ lock._last_refresh = time.time() - 1000
+
+ with caplog.at_level(logging.WARNING):
+ assert await lock.refresh() is False
+
+ assert lock._last_refresh == 0
+ records = [r for r in caplog.records if "refresh" in r.getMessage()]
+ assert records, "expected a refresh log record"
+ assert all(r.levelno == logging.WARNING for r in records)
+
+
if __name__ == "__main__":
# Run specific test for quick validation
pytest.main([__file__, "-v"])