Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): release a cluster lock even after a failed refresh (#15602)

Nicholas Tindle86c13d4413ae
diff --git a/autogpt_platform/backend/backend/executor/cluster_lock.py b/autogpt_platform/backend/backend/executor/cluster_lock.py
index da4896395d..53e1912a96 100644
--- a/autogpt_platform/backend/backend/executor/cluster_lock.py
+++ b/autogpt_platform/backend/backend/executor/cluster_lock.py
@@ -184,12 +184,11 @@ class ClusterLock:
Owner-checked: only deletes the Redis key if the stored value still
matches our owner_id. Prevents wiping a successor's lock when the
- original key was force-released externally and re-acquired.
+ original key was force-released externally and re-acquired. Runs even
+ after a failed refresh, which may have been a transient Redis error
+ while this holder still owned the key; skipping the delete would leave
+ the key held until its TTL.
"""
- with self._refresh_lock:
- if self._last_refresh == 0:
- return
-
try:
self.redis.eval(_RELEASE_LUA, 1, self.key, self.owner_id)
except Exception:
@@ -329,16 +328,7 @@ class AsyncClusterLock:
return False
async def release(self):
- """Release the lock.
-
- Owner-checked: only deletes the Redis key if the stored value still
- matches our owner_id. Prevents wiping a successor's lock when the
- original key was force-released externally and re-acquired.
- """
- async with self._refresh_lock:
- if self._last_refresh == 0:
- return
-
+ """Release the lock; see :meth:`ClusterLock.release`."""
try:
await cast(Any, self.redis.eval(_RELEASE_LUA, 1, self.key, self.owner_id))
except Exception:
diff --git a/autogpt_platform/backend/backend/executor/cluster_lock_test.py b/autogpt_platform/backend/backend/executor/cluster_lock_test.py
index c9b970018c..4448d8c626 100644
--- a/autogpt_platform/backend/backend/executor/cluster_lock_test.py
+++ b/autogpt_platform/backend/backend/executor/cluster_lock_test.py
@@ -18,7 +18,7 @@ from redis.exceptions import ClusterDownError
from redis.exceptions import ConnectionError as RedisConnectionError
from redis.exceptions import TimeoutError as RedisTimeoutError
-from .cluster_lock import AsyncClusterLock, ClusterLock
+from .cluster_lock import _RELEASE_LUA, AsyncClusterLock, ClusterLock
logger = logging.getLogger(__name__)
@@ -145,6 +145,22 @@ class TestClusterLockBasic:
assert redis_client.exists(lock_key) == 1
assert redis_client.get(lock_key).decode("utf-8") == owner_b
+ def test_release_after_a_failed_refresh_still_frees_the_key(
+ self, redis_client, lock_key, owner_id
+ ):
+ """A refresh that hit a transient Redis error says nothing about who
+ owns the key; release must still free it rather than leave it held
+ until its TTL."""
+ lock = ClusterLock(redis_client, lock_key, owner_id, timeout=60)
+ assert lock.try_acquire() == owner_id
+ lock.redis = MagicMock(get=MagicMock(side_effect=RedisConnectionError("x")))
+ assert lock.refresh() is False
+ lock.redis = redis_client
+
+ lock.release()
+
+ assert redis_client.exists(lock_key) == 0
+
class TestClusterLockRefresh:
"""Lock refresh and TTL management."""
@@ -630,6 +646,22 @@ class TestClusterLockTransientErrorHandling:
assert any(r.levelno == logging.ERROR for r in records)
assert not any(r.levelno == logging.WARNING for r in records)
+ def test_release_after_a_transient_refresh_failure_runs_the_owner_checked_delete(
+ self,
+ ):
+ fake_redis = MagicMock()
+ fake_redis.set.return_value = True
+ fake_redis.get.side_effect = RedisConnectionError("connection lost")
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ owner = str(uuid.uuid4())
+ lock = ClusterLock(fake_redis, lock_key, owner, timeout=60)
+ assert lock.try_acquire() == owner
+ assert lock.refresh() is False
+
+ lock.release()
+
+ fake_redis.eval.assert_called_once_with(_RELEASE_LUA, 1, lock_key, owner)
+
class TestAsyncClusterLockTransientErrorHandling:
"""Async sibling of ``TestClusterLockTransientErrorHandling``."""
@@ -683,6 +715,24 @@ class TestAsyncClusterLockTransientErrorHandling:
assert records, "expected a refresh log record"
assert all(r.levelno == logging.WARNING for r in records)
+ @pytest.mark.asyncio
+ async def test_release_after_a_transient_refresh_failure_runs_the_owner_checked_delete(
+ self,
+ ):
+ fake_redis = MagicMock()
+ fake_redis.set = AsyncMock(return_value=True)
+ fake_redis.get = AsyncMock(side_effect=RedisConnectionError("connection lost"))
+ fake_redis.eval = AsyncMock(return_value=1)
+ lock_key = f"test_lock:{uuid.uuid4()}"
+ owner = str(uuid.uuid4())
+ lock = AsyncClusterLock(fake_redis, lock_key, owner, timeout=60)
+ assert await lock.try_acquire() == owner
+ assert await lock.refresh() is False
+
+ await lock.release()
+
+ fake_redis.eval.assert_awaited_once_with(_RELEASE_LUA, 1, lock_key, owner)
+
if __name__ == "__main__":
# Run specific test for quick validation