Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): make async Redis clients loop-safe (#14259)

Nicholas Tindle02445b8788cc
diff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.py
index 9e32ffcbbd..63ac49d857 100644
--- a/autogpt_platform/backend/backend/data/redis_client.py
+++ b/autogpt_platform/backend/backend/data/redis_client.py
@@ -1,6 +1,8 @@
import asyncio
import logging
import os
+from collections.abc import AsyncGenerator
+from weakref import ReferenceType, WeakKeyDictionary, ref
from dotenv import load_dotenv
from redis import Redis
@@ -156,26 +158,64 @@ async def connect_async() -> AsyncRedisClient:
return c
-# One AsyncRedisCluster per event loop: the client binds to the loop it was
-# first awaited on, so a module-level singleton breaks across test loops.
-_async_clients: dict[int, AsyncRedisCluster] = {}
+# Async Redis connections bind to the loop where they are first used.
+_async_clients: WeakKeyDictionary[asyncio.AbstractEventLoop, AsyncRedisClient] = (
+ WeakKeyDictionary()
+)
+_async_client_finalizers: WeakKeyDictionary[
+ asyncio.AbstractEventLoop, AsyncGenerator[None, None]
+] = WeakKeyDictionary()
+_async_client_locks: WeakKeyDictionary[
+ asyncio.AbstractEventLoop, ReferenceType[asyncio.Lock]
+] = WeakKeyDictionary()
+
+
+def _get_async_client_lock(loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
+ lock_ref = _async_client_locks.get(loop)
+ lock = lock_ref() if lock_ref is not None else None
+ if lock is None:
+ lock = asyncio.Lock()
+ _async_client_locks[loop] = ref(lock)
+ return lock
@conn_retry("AsyncRedis", "Releasing connection")
async def disconnect_async():
loop = asyncio.get_running_loop()
- c = _async_clients.pop(id(loop), None)
- if c is not None:
- await c.close()
+ async with _get_async_client_lock(loop):
+ client = _async_clients.pop(loop, None)
+ finalizer = _async_client_finalizers.pop(loop, None)
+ if finalizer is not None:
+ await finalizer.aclose()
+ elif client is not None:
+ await client.close()
async def get_redis_async() -> AsyncRedisClient:
loop = asyncio.get_running_loop()
- client = _async_clients.get(id(loop))
- if client is None:
- client = await connect_async()
- _async_clients[id(loop)] = client
- return client
+ async with _get_async_client_lock(loop):
+ client = _async_clients.get(loop)
+ if client is None:
+ client = await connect_async()
+ finalizer = _close_async_client_on_loop_shutdown(client)
+ await anext(finalizer)
+ _async_clients[loop] = client
+ _async_client_finalizers[loop] = finalizer
+ return client
+
+
+async def _close_async_client_on_loop_shutdown(
+ client: AsyncRedisClient,
+) -> AsyncGenerator[None, None]:
+ """Close a loop-owned client during ``loop.shutdown_asyncgens()``."""
+ try:
+ yield
+ finally:
+ loop = asyncio.get_running_loop()
+ if _async_clients.get(loop) is client:
+ _async_clients.pop(loop, None)
+ _async_client_finalizers.pop(loop, None)
+ await client.close()
# Sharded pub/sub only delivers on the keyslot-owning shard; subscribers
diff --git a/autogpt_platform/backend/backend/data/redis_client_test.py b/autogpt_platform/backend/backend/data/redis_client_test.py
index a83c1a0729..69d5d3ffb6 100644
--- a/autogpt_platform/backend/backend/data/redis_client_test.py
+++ b/autogpt_platform/backend/backend/data/redis_client_test.py
@@ -3,6 +3,7 @@
Patches the redis-py constructors + ``ping()`` so no real Redis is needed.
"""
+import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -22,6 +23,8 @@ def _reset_module_caches() -> None:
"""Flush cached singletons between tests so each test sees a fresh connect."""
redis_client.get_redis.cache_clear()
redis_client._async_clients.clear()
+ redis_client._async_client_finalizers.clear()
+ redis_client._async_client_locks.clear()
def test_connect_builds_redis_cluster() -> None:
@@ -151,6 +154,59 @@ async def test_get_redis_async_caches_connect() -> None:
mock_conn.assert_called_once()
+async def test_get_redis_async_serializes_concurrent_connects() -> None:
+ connect_started = asyncio.Event()
+ allow_connect = asyncio.Event()
+ fake = MagicMock(spec=AsyncRedisCluster)
+ fake.close = AsyncMock()
+
+ async def delayed_connect() -> AsyncRedisCluster:
+ connect_started.set()
+ await allow_connect.wait()
+ return fake
+
+ with patch.object(
+ redis_client,
+ "connect_async",
+ new=AsyncMock(side_effect=delayed_connect),
+ ) as mock_connect:
+ first = asyncio.create_task(redis_client.get_redis_async())
+ second = asyncio.create_task(redis_client.get_redis_async())
+ await connect_started.wait()
+ await asyncio.sleep(0)
+
+ assert mock_connect.await_count == 1
+
+ allow_connect.set()
+ first_client, second_client = await asyncio.gather(first, second)
+ await redis_client.disconnect_async()
+
+ assert first_client is fake
+ assert second_client is fake
+ mock_connect.assert_awaited_once()
+ fake.close.assert_awaited_once()
+
+
+def test_get_redis_async_closes_client_when_owning_loop_shuts_down() -> None:
+ fake = MagicMock(spec=AsyncRedisCluster)
+ fake.close = AsyncMock()
+
+ with patch.object(redis_client, "connect_async", new=AsyncMock(return_value=fake)):
+ loop = asyncio.new_event_loop()
+ try:
+ client = loop.run_until_complete(redis_client.get_redis_async())
+ assert client is fake
+ assert redis_client._async_clients[loop] is fake
+ finally:
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ loop.close()
+
+ fake.close.assert_awaited_once()
+ assert not redis_client._async_clients
+ assert not redis_client._async_client_finalizers
+
+
def test_disconnect_closes_cached_client() -> None:
with patch.object(redis_client, "connect", autospec=True) as mock_connect:
fake = MagicMock(spec=RedisCluster)
@@ -171,7 +227,8 @@ async def test_disconnect_async_closes_cached_client() -> None:
await redis_client.disconnect_async()
fake.close.assert_awaited_once()
- assert redis_client._async_clients == {}
+ assert not redis_client._async_clients
+ assert not redis_client._async_client_finalizers
@pytest.mark.asyncio
diff --git a/autogpt_platform/backend/backend/util/test.py b/autogpt_platform/backend/backend/util/test.py
index 26b255e019..6a26a80efc 100644
--- a/autogpt_platform/backend/backend/util/test.py
+++ b/autogpt_platform/backend/backend/util/test.py
@@ -9,7 +9,7 @@ from autogpt_libs.auth import get_user_id
from backend.api.rest_api import AgentServer
from backend.blocks._base import Block, BlockSchema
-from backend.data import db
+from backend.data import db, redis_client
from backend.data.block import initialize_blocks
from backend.data.db_manager import DatabaseManager
from backend.data.execution import (
@@ -53,7 +53,10 @@ class SpinTestServer:
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
- await db.disconnect()
+ try:
+ await redis_client.disconnect_async()
+ finally:
+ await db.disconnect()
self.scheduler.__exit__(exc_type, exc_val, exc_tb)
self.exec_manager.__exit__(exc_type, exc_val, exc_tb)