AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): release session cluster lock on completion (#12867)
Zamil Majdyf238c153a5bb
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager.py b/autogpt_platform/backend/backend/copilot/executor/manager.pyindex da113ccc50..02a2913883 100644--- a/autogpt_platform/backend/backend/copilot/executor/manager.py+++ b/autogpt_platform/backend/backend/copilot/executor/manager.py@@ -34,6 +34,7 @@ from .utils import (CancelCoPilotEvent,CoPilotExecutionEntry,create_copilot_queue_config,+ get_session_lock_key,)logger = TruncatedLogger(logging.getLogger(__name__), prefix="[CoPilotExecutor]")@@ -366,7 +367,7 @@ class CoPilotExecutor(AppProcess):# Try to acquire cluster-wide lockcluster_lock = ClusterLock(redis=redis.get_redis(),- key=f"copilot:session:{session_id}:lock",+ key=get_session_lock_key(session_id),owner_id=self.executor_id,timeout=settings.config.cluster_lock_timeout,)diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex b96e1821a1..a2b051d82b 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -82,6 +82,12 @@ COPILOT_CANCEL_EXCHANGE = Exchange()COPILOT_CANCEL_QUEUE_NAME = "copilot_cancel_queue"++def get_session_lock_key(session_id: str) -> str:+ """Redis key for the per-session cluster lock held by the executing pod."""+ return f"copilot:session:{session_id}:lock"++# CoPilot operations can include extended thinking and agent generation# which may take 30+ minutes to completeCOPILOT_CONSUMER_TIMEOUT_SECONDS = 60 * 60 # 1 hourdiff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex f4a26b7008..424964e075 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -35,7 +35,7 @@ from backend.data.redis_client import get_redis_asyncfrom backend.data.redis_helpers import hash_compare_and_setfrom .config import ChatConfig-from .executor.utils import COPILOT_CONSUMER_TIMEOUT_SECONDS+from .executor.utils import COPILOT_CONSUMER_TIMEOUT_SECONDS, get_session_lock_keyfrom .response_model import (ResponseType,StreamBaseResponse,@@ -851,6 +851,15 @@ async def mark_session_completed(logger.debug(f"Session {session_id} already completed/failed, skipping")return False+ # Force-release the executor's cluster lock so the next enqueued turn can+ # acquire it immediately. The lock holder's on_run_done will also release+ # (idempotent delete); doing it here unblocks cases where the task hangs+ # past the cancel timeout or a pod crash leaves the lock orphaned.+ try:+ await redis.delete(get_session_lock_key(session_id))+ except RedisError as e:+ logger.warning(f"Failed to release cluster lock for session {session_id}: {e}")+if error_message and not skip_error_publish:try:await publish_chunk(turn_id, StreamError(errorText=error_message))diff --git a/autogpt_platform/backend/backend/copilot/stream_registry_test.py b/autogpt_platform/backend/backend/copilot/stream_registry_test.pyindex 28ec199025..db26a5f524 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry_test.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry_test.py@@ -4,8 +4,10 @@ import asynciofrom unittest.mock import AsyncMock, patchimport pytest+from redis.exceptions import RedisErrorfrom backend.copilot import stream_registry+from backend.copilot.executor.utils import get_session_lock_key@pytest.fixture(autouse=True)@@ -221,3 +223,115 @@ async def test_stream_and_publish_consumer_break_then_aclose_releases_inner():await wrapper.aclose()assert inner_finally_ran.is_set()+++# ---------------------------------------------------------------------------+# mark_session_completed: the atomic meta flip to completed/failed must also+# release the per-session cluster lock, so the next enqueued turn's run+# handler can acquire it without waiting for the TTL (5 min default).+# ---------------------------------------------------------------------------+++class _FakeRedis:+ """Minimal async-Redis fake: only the calls mark_session_completed makes."""++ def __init__(self, meta: dict[str, str]):+ self._meta = dict(meta)+ self.deleted_keys: list[str] = []+ self.delete = AsyncMock(side_effect=self._record_delete)++ async def _record_delete(self, *keys: str):+ self.deleted_keys.extend(keys)+ for k in keys:+ self._meta.pop(k, None)+ return len(keys)++ async def hgetall(self, _key: str):+ return dict(self._meta)+++async def test_mark_session_completed_releases_cluster_lock_on_success():+ """CAS swap must be followed by a DELETE on the session's lock key so a+ stuck-because-of-stale-lock session becomes immediately claimable."""+ fake_redis = _FakeRedis({"status": "running", "turn_id": "turn-1"})++ with (+ patch.object(+ stream_registry, "get_redis_async", new=AsyncMock(return_value=fake_redis)+ ),+ patch.object(+ stream_registry, "hash_compare_and_set", new=AsyncMock(return_value=True)+ ),+ patch.object(stream_registry, "publish_chunk", new=AsyncMock()),+ patch.object(+ stream_registry.chat_db(),+ "set_turn_duration",+ new=AsyncMock(),+ create=True,+ ),+ ):+ result = await stream_registry.mark_session_completed("sess-1")++ assert result is True+ assert get_session_lock_key("sess-1") in fake_redis.deleted_keys+++async def test_mark_session_completed_skips_lock_release_when_already_completed():+ """CAS failure = someone else completed the session first; we must not+ delete their already-released lock, and we must NOT publish StreamFinish+ twice (the winning caller already published it)."""+ fake_redis = _FakeRedis({"status": "completed", "turn_id": "turn-1"})+ publish_mock = AsyncMock()++ with (+ patch.object(+ stream_registry, "get_redis_async", new=AsyncMock(return_value=fake_redis)+ ),+ patch.object(+ stream_registry, "hash_compare_and_set", new=AsyncMock(return_value=False)+ ),+ patch.object(stream_registry, "publish_chunk", new=publish_mock),+ ):+ result = await stream_registry.mark_session_completed("sess-1")++ assert result is False+ assert get_session_lock_key("sess-1") not in fake_redis.deleted_keys+ assert not any(+ isinstance(call.args[1], stream_registry.StreamFinish)+ for call in publish_mock.call_args_list+ ), "StreamFinish must NOT be re-published on the CAS-no-op branch"+++async def test_mark_session_completed_survives_lock_release_redis_error():+ """A Redis hiccup during lock DELETE must not prevent the StreamFinish+ publish — the client's SSE stream would otherwise hang on the stale meta+ status while Redis recovers."""+ fake_redis = _FakeRedis({"status": "running", "turn_id": "turn-1"})+ fake_redis.delete = AsyncMock(side_effect=RedisError("boom"))+ publish_mock = AsyncMock()++ with (+ patch.object(+ stream_registry, "get_redis_async", new=AsyncMock(return_value=fake_redis)+ ),+ patch.object(+ stream_registry, "hash_compare_and_set", new=AsyncMock(return_value=True)+ ),+ patch.object(stream_registry, "publish_chunk", new=publish_mock),+ patch.object(+ stream_registry.chat_db(),+ "set_turn_duration",+ new=AsyncMock(),+ create=True,+ ),+ ):+ result = await stream_registry.mark_session_completed("sess-1")++ assert result is True+ assert any(+ isinstance(call.args[1], stream_registry.StreamFinish)+ for call in publish_mock.call_args_list+ ), "StreamFinish must still be published even if lock DELETE raises"diff --git a/autogpt_platform/backend/backend/executor/cluster_lock.py b/autogpt_platform/backend/backend/executor/cluster_lock.pyindex 0732c3f6de..9fe8b744c4 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock.py@@ -4,7 +4,7 @@ import asyncioimport loggingimport threadingimport time-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Any, castif TYPE_CHECKING:from redis import Redis@@ -12,6 +12,17 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+# Lua CAS release: only delete the key if the stored value still matches our+# owner_id. Returns 1 on delete, 0 on no-op. This makes release() safe against+# the race where an external caller (e.g. mark_session_completed's force-release)+# deletes our key and a new owner acquires it before our release() fires — without+# the CAS guard, release() would wipe the successor's valid lock.+_RELEASE_LUA = (+ "if redis.call('get', KEYS[1]) == ARGV[1] then "+ "return redis.call('del', KEYS[1]) "+ "else return 0 end"+)+class ClusterLock:"""Simple Redis-based distributed lock for preventing duplicate execution."""@@ -116,13 +127,18 @@ class ClusterLock:return Falsedef release(self):- """Release the lock."""+ """Release the lock.++ Owner-checked: only deletes the Redis key if the stored value still+ matches our owner_id. Prevents wiping a successor's lock when the+ original key was force-released externally and re-acquired.+ """with self._refresh_lock:if self._last_refresh == 0:returntry:- self.redis.delete(self.key)+ self.redis.eval(_RELEASE_LUA, 1, self.key, self.owner_id)except Exception:pass@@ -237,13 +253,18 @@ class AsyncClusterLock:return Falseasync def release(self):- """Release the lock."""+ """Release the lock.++ Owner-checked: only deletes the Redis key if the stored value still+ matches our owner_id. Prevents wiping a successor's lock when the+ original key was force-released externally and re-acquired.+ """async with self._refresh_lock:if self._last_refresh == 0:returntry:- await self.redis.delete(self.key)+ await cast(Any, self.redis.eval(_RELEASE_LUA, 1, self.key, self.owner_id))except Exception:passdiff --git a/autogpt_platform/backend/backend/executor/cluster_lock_test.py b/autogpt_platform/backend/backend/executor/cluster_lock_test.pyindex c5d8965f0f..5491c51cad 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock_test.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock_test.py@@ -108,6 +108,33 @@ class TestClusterLockBasic:new_lock = ClusterLock(redis_client, lock_key, new_owner_id, timeout=60)assert new_lock.try_acquire() == new_owner_id+ def test_release_does_not_wipe_successor_lock(self, redis_client, lock_key):+ """Releasing after external delete+reacquire must NOT delete successor.++ Race: an external caller force-deletes the lock key, a new owner+ acquires it, then the original ClusterLock.release() runs. Owner-checked+ release must leave the successor's key intact.+ """+ owner_a = str(uuid.uuid4())+ owner_b = str(uuid.uuid4())++ lock_a = ClusterLock(redis_client, lock_key, owner_a, timeout=60)+ assert lock_a.try_acquire() == owner_a++ # External force-release (e.g. mark_session_completed).+ redis_client.delete(lock_key)++ # Successor acquires the same key.+ lock_b = ClusterLock(redis_client, lock_key, owner_b, timeout=60)+ assert lock_b.try_acquire() == owner_b++ # Original releases — must be a no-op on Redis because value != owner_a.+ lock_a.release()++ # Successor's lock is still intact.+ assert redis_client.exists(lock_key) == 1+ assert redis_client.get(lock_key).decode("utf-8") == owner_b+class TestClusterLockRefresh:"""Lock refresh and TTL management."""