AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): stop a wide graph fan-out from exhausting the Redis connection pool (#15197)
Bentlybacb6e119580
diff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.pyindex fcd5fb388c..bd9baf99b4 100644--- a/autogpt_platform/backend/backend/data/redis_client.py+++ b/autogpt_platform/backend/backend/data/redis_client.py@@ -44,6 +44,12 @@ USE_ANNOUNCED_ADDRESS = os.getenv("REDIS_USE_ANNOUNCED_ADDRESS", "").lower() in"yes",)+# Per-node pool cap. redis-py 8.x lowered the default from 2**31 to 100 and+# raises MaxConnectionsError instead of waiting, so one wide graph fan-out on+# an executor's shared loop failed lock and publish calls. 10000 matches the+# server's default `maxclients`, so the server limit is the one that binds.+REDIS_MAX_CONNECTIONS = int(os.getenv("REDIS_MAX_CONNECTIONS", "10000"))+# Retry transient cluster errors internally so a rotation blip never surfaces# as a graph-exec 500.REDIS_RETRY_ATTEMPTS = int(os.getenv("REDIS_RETRY_ATTEMPTS", "5"))@@ -103,6 +109,7 @@ def connect() -> RedisClient:socket_connect_timeout=SOCKET_CONNECT_TIMEOUT,socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,+ max_connections=REDIS_MAX_CONNECTIONS,address_remap=_address_remap,# Drives both per-command retries and the cluster-level retry counter.retry=_build_retry(),@@ -129,6 +136,7 @@ def connect_once(timeout: float) -> RedisClient:socket_connect_timeout=timeout,socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,+ max_connections=REDIS_MAX_CONNECTIONS,address_remap=_address_remap,retry=Retry(NoBackoff(), 0),)@@ -164,6 +172,7 @@ async def connect_async() -> AsyncRedisClient:socket_connect_timeout=SOCKET_CONNECT_TIMEOUT,socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,+ max_connections=REDIS_MAX_CONNECTIONS,address_remap=_address_remap,# redis-py 6.x AsyncRedisCluster ignores `retry_on_error` — the cluster# retry path uses a hardcoded {Timeout, Connection, ClusterDown} set.diff --git a/autogpt_platform/backend/backend/data/redis_client_pool_test.py b/autogpt_platform/backend/backend/data/redis_client_pool_test.pynew file mode 100644index 0000000000..7e50cbff9e--- /dev/null+++ b/autogpt_platform/backend/backend/data/redis_client_pool_test.py@@ -0,0 +1,117 @@+"""Connection-pool sizing for the cluster clients in ``redis_client``.++redis-py 8.x caps each cluster node's pool at 100 connections by default and+raises ``MaxConnectionsError`` the moment it is full. One executor runs every+node of a graph on a single loop, so a wide fan-out needs far more than 100+concurrent commands against the user's shard.+"""++import asyncio+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from redis import Redis+from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster+from redis.backoff import NoBackoff+from redis.cluster import RedisCluster+from redis.exceptions import RedisError+from redis.retry import Retry++import backend.data.redis_client as redis_client+from backend.util.testing import is_tcp_port_reachable++# redis-py 8.x's per-node default; the configured cap must sit well above it.+REDIS_PY_DEFAULT_MAX_CONNECTIONS = 100++# Lock SETs plus event SPUBLISHes in flight at once, all on one hash slot, as+# a wide fan-out produces for a single user's graph execution.+FAN_OUT_COMMANDS = 500+++def _has_live_cluster() -> bool:+ """A listening port is not enough: against a standalone Redis,+ ``get_redis_async`` fails cluster discovery and ``conn_retry`` keeps+ retrying it, so probe ``CLUSTER INFO`` once without retries instead."""+ if not is_tcp_port_reachable(redis_client.HOST, redis_client.PORT):+ return False+ probe = Redis(+ host=redis_client.HOST,+ port=redis_client.PORT,+ password=redis_client.PASSWORD,+ socket_timeout=1,+ socket_connect_timeout=1,+ retry=Retry(NoBackoff(), 0),+ decode_responses=True,+ )+ try:+ info = probe.execute_command("CLUSTER INFO")+ except RedisError:+ return False+ finally:+ probe.close()+ return isinstance(info, dict) and info.get("cluster_state") == "ok"+++def test_max_connections_default_is_above_redis_py_default() -> None:+ assert redis_client.REDIS_MAX_CONNECTIONS >= FAN_OUT_COMMANDS+ assert redis_client.REDIS_MAX_CONNECTIONS > REDIS_PY_DEFAULT_MAX_CONNECTIONS+++def test_connect_sets_max_connections() -> None:+ with patch.object(redis_client, "RedisCluster", autospec=True) as mock_cluster:+ mock_cluster.return_value = MagicMock(spec=RedisCluster)+ redis_client.connect()++ kwargs = mock_cluster.call_args.kwargs+ assert kwargs["max_connections"] == redis_client.REDIS_MAX_CONNECTIONS+++def test_connect_once_sets_max_connections() -> None:+ with patch.object(redis_client, "RedisCluster", autospec=True) as mock_cluster:+ mock_cluster.return_value = MagicMock(spec=RedisCluster)+ redis_client.connect_once(timeout=1)++ kwargs = mock_cluster.call_args.kwargs+ assert kwargs["max_connections"] == redis_client.REDIS_MAX_CONNECTIONS+++async def test_connect_async_sets_max_connections() -> None:+ with patch.object(redis_client, "AsyncRedisCluster", autospec=True) as mock_cluster:+ fake = MagicMock(spec=AsyncRedisCluster)+ fake.ping = AsyncMock()+ mock_cluster.return_value = fake+ await redis_client.connect_async()++ kwargs = mock_cluster.call_args.kwargs+ assert kwargs["max_connections"] == redis_client.REDIS_MAX_CONNECTIONS+++ not _has_live_cluster(),+ reason="local redis cluster not reachable",+)+async def test_wide_fan_out_does_not_exhaust_async_pool() -> None:+ """Hundreds of concurrent lock and publish commands on one shard must not+ raise MaxConnectionsError."""+ await redis_client.disconnect_async()+ client = await redis_client.get_redis_async()+ prefix = "{redis-pool-fan-out-test}"+ half = FAN_OUT_COMMANDS // 2+ try:+ locks = [+ client.set(f"{prefix}:lock:{i}", "1", nx=True, px=10_000)+ for i in range(half)+ ]+ publishes = [+ client.execute_command("SPUBLISH", f"{prefix}:events", f"event-{i}")+ for i in range(half)+ ]+ results = await asyncio.gather(*locks, *publishes, return_exceptions=True)++ errors = [r for r in results if isinstance(r, BaseException)]+ assert errors == [], f"{len(errors)} commands failed, first: {errors[0]!r}"+ finally:+ await client.delete(*[f"{prefix}:lock:{i}" for i in range(half)])+ await redis_client.disconnect_async()