AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): make async Redis clients loop-safe (#14259)
Nicholas Tindle02445b8788cc
diff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.pyindex 9e32ffcbbd..63ac49d857 100644--- a/autogpt_platform/backend/backend/data/redis_client.py+++ b/autogpt_platform/backend/backend/data/redis_client.py@@ -1,6 +1,8 @@import asyncioimport loggingimport os+from collections.abc import AsyncGenerator+from weakref import ReferenceType, WeakKeyDictionary, reffrom dotenv import load_dotenvfrom redis import Redis@@ -156,26 +158,64 @@ async def connect_async() -> AsyncRedisClient:return c-# One AsyncRedisCluster per event loop: the client binds to the loop it was-# first awaited on, so a module-level singleton breaks across test loops.-_async_clients: dict[int, AsyncRedisCluster] = {}+# Async Redis connections bind to the loop where they are first used.+_async_clients: WeakKeyDictionary[asyncio.AbstractEventLoop, AsyncRedisClient] = (+ WeakKeyDictionary()+)+_async_client_finalizers: WeakKeyDictionary[+ asyncio.AbstractEventLoop, AsyncGenerator[None, None]+] = WeakKeyDictionary()+_async_client_locks: WeakKeyDictionary[+ asyncio.AbstractEventLoop, ReferenceType[asyncio.Lock]+] = WeakKeyDictionary()+++def _get_async_client_lock(loop: asyncio.AbstractEventLoop) -> asyncio.Lock:+ lock_ref = _async_client_locks.get(loop)+ lock = lock_ref() if lock_ref is not None else None+ if lock is None:+ lock = asyncio.Lock()+ _async_client_locks[loop] = ref(lock)+ return lock@conn_retry("AsyncRedis", "Releasing connection")async def disconnect_async():loop = asyncio.get_running_loop()- c = _async_clients.pop(id(loop), None)- if c is not None:- await c.close()+ async with _get_async_client_lock(loop):+ client = _async_clients.pop(loop, None)+ finalizer = _async_client_finalizers.pop(loop, None)+ if finalizer is not None:+ await finalizer.aclose()+ elif client is not None:+ await client.close()async def get_redis_async() -> AsyncRedisClient:loop = asyncio.get_running_loop()- client = _async_clients.get(id(loop))- if client is None:- client = await connect_async()- _async_clients[id(loop)] = client- return client+ async with _get_async_client_lock(loop):+ client = _async_clients.get(loop)+ if client is None:+ client = await connect_async()+ finalizer = _close_async_client_on_loop_shutdown(client)+ await anext(finalizer)+ _async_clients[loop] = client+ _async_client_finalizers[loop] = finalizer+ return client+++async def _close_async_client_on_loop_shutdown(+ client: AsyncRedisClient,+) -> AsyncGenerator[None, None]:+ """Close a loop-owned client during ``loop.shutdown_asyncgens()``."""+ try:+ yield+ finally:+ loop = asyncio.get_running_loop()+ if _async_clients.get(loop) is client:+ _async_clients.pop(loop, None)+ _async_client_finalizers.pop(loop, None)+ await client.close()# Sharded pub/sub only delivers on the keyslot-owning shard; subscribersdiff --git a/autogpt_platform/backend/backend/data/redis_client_test.py b/autogpt_platform/backend/backend/data/redis_client_test.pyindex a83c1a0729..69d5d3ffb6 100644--- a/autogpt_platform/backend/backend/data/redis_client_test.py+++ b/autogpt_platform/backend/backend/data/redis_client_test.py@@ -3,6 +3,7 @@Patches the redis-py constructors + ``ping()`` so no real Redis is needed."""+import asynciofrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -22,6 +23,8 @@ def _reset_module_caches() -> None:"""Flush cached singletons between tests so each test sees a fresh connect."""redis_client.get_redis.cache_clear()redis_client._async_clients.clear()+ redis_client._async_client_finalizers.clear()+ redis_client._async_client_locks.clear()def test_connect_builds_redis_cluster() -> None:@@ -151,6 +154,59 @@ async def test_get_redis_async_caches_connect() -> None:mock_conn.assert_called_once()+async def test_get_redis_async_serializes_concurrent_connects() -> None:+ connect_started = asyncio.Event()+ allow_connect = asyncio.Event()+ fake = MagicMock(spec=AsyncRedisCluster)+ fake.close = AsyncMock()++ async def delayed_connect() -> AsyncRedisCluster:+ connect_started.set()+ await allow_connect.wait()+ return fake++ with patch.object(+ redis_client,+ "connect_async",+ new=AsyncMock(side_effect=delayed_connect),+ ) as mock_connect:+ first = asyncio.create_task(redis_client.get_redis_async())+ second = asyncio.create_task(redis_client.get_redis_async())+ await connect_started.wait()+ await asyncio.sleep(0)++ assert mock_connect.await_count == 1++ allow_connect.set()+ first_client, second_client = await asyncio.gather(first, second)+ await redis_client.disconnect_async()++ assert first_client is fake+ assert second_client is fake+ mock_connect.assert_awaited_once()+ fake.close.assert_awaited_once()+++def test_get_redis_async_closes_client_when_owning_loop_shuts_down() -> None:+ fake = MagicMock(spec=AsyncRedisCluster)+ fake.close = AsyncMock()++ with patch.object(redis_client, "connect_async", new=AsyncMock(return_value=fake)):+ loop = asyncio.new_event_loop()+ try:+ client = loop.run_until_complete(redis_client.get_redis_async())+ assert client is fake+ assert redis_client._async_clients[loop] is fake+ finally:+ loop.run_until_complete(loop.shutdown_asyncgens())+ loop.close()++ fake.close.assert_awaited_once()+ assert not redis_client._async_clients+ assert not redis_client._async_client_finalizers++def test_disconnect_closes_cached_client() -> None:with patch.object(redis_client, "connect", autospec=True) as mock_connect:fake = MagicMock(spec=RedisCluster)@@ -171,7 +227,8 @@ async def test_disconnect_async_closes_cached_client() -> None:await redis_client.disconnect_async()fake.close.assert_awaited_once()- assert redis_client._async_clients == {}+ assert not redis_client._async_clients+ assert not redis_client._async_client_finalizers@pytest.mark.asynciodiff --git a/autogpt_platform/backend/backend/util/test.py b/autogpt_platform/backend/backend/util/test.pyindex 26b255e019..6a26a80efc 100644--- a/autogpt_platform/backend/backend/util/test.py+++ b/autogpt_platform/backend/backend/util/test.py@@ -9,7 +9,7 @@ from autogpt_libs.auth import get_user_idfrom backend.api.rest_api import AgentServerfrom backend.blocks._base import Block, BlockSchema-from backend.data import db+from backend.data import db, redis_clientfrom backend.data.block import initialize_blocksfrom backend.data.db_manager import DatabaseManagerfrom backend.data.execution import (@@ -53,7 +53,10 @@ class SpinTestServer:return selfasync def __aexit__(self, exc_type, exc_val, exc_tb):- await db.disconnect()+ try:+ await redis_client.disconnect_async()+ finally:+ await db.disconnect()self.scheduler.__exit__(exc_type, exc_val, exc_tb)self.exec_manager.__exit__(exc_type, exc_val, exc_tb)