Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): serialize at-reset joins and cancels across replicas (#15595)

Nicholas Tindle6902135e7137
diff --git a/autogpt_platform/backend/backend/api/features/chat/at_reset.py b/autogpt_platform/backend/backend/api/features/chat/at_reset.py
index 7a9f5319aa..1701a29b34 100644
--- a/autogpt_platform/backend/backend/api/features/chat/at_reset.py
+++ b/autogpt_platform/backend/backend/api/features/chat/at_reset.py
@@ -5,6 +5,8 @@ whether to still fire then, live in ``backend.copilot.at_reset``; the job is a
scheduled copilot turn, at most one of each kind per chat.
"""
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
from datetime import datetime
from typing import Annotated
@@ -22,6 +24,7 @@ from backend.copilot.at_reset import (
at_reset_schedule_id,
next_reset_run_at,
)
+from backend.copilot.at_reset_lock import at_reset_job_lock
from backend.copilot.continuation import RESUMED_AT_RESET_NOTE, as_model_prompt
from backend.copilot.model import ChatSessionInfo
from backend.copilot.rate_limit import RateLimitUnavailable
@@ -106,25 +109,27 @@ async def schedule_at_reset(
"""
session = await _validate_and_get_writable_session(session_id, user_id)
run_at = await _run_at(user_id, session, request.kind)
- message = (
- await _joined_with_waiting_send(user_id, session_id, request.message)
- if request.kind == "send" and request.message
- else None
- )
- await _scheduler().add_copilot_turn_schedule(
- user_id=user_id,
- session_id=session_id,
- message=message or _RESUME_DESCRIPTION,
- run_at=run_at,
- # The budget windows reset in UTC.
- user_timezone="UTC",
- # Anchored on the session, as for a message sent now.
- organization_id=session.organization_id or ctx.org_id,
- team_id=session.team_id if session.organization_id else ctx.team_id,
- expert_id=session.expert_id,
- purpose=request.kind,
- schedule_id=at_reset_schedule_id(request.kind, session_id),
- )
+ job_id = at_reset_schedule_id(request.kind, session_id)
+ async with _holding(job_id):
+ message = (
+ await _joined_with_waiting_send(user_id, session_id, request.message)
+ if request.kind == "send" and request.message
+ else None
+ )
+ await _scheduler().add_copilot_turn_schedule(
+ user_id=user_id,
+ session_id=session_id,
+ message=message or _RESUME_DESCRIPTION,
+ run_at=run_at,
+ # The budget windows reset in UTC.
+ user_timezone="UTC",
+ # Anchored on the session, as for a message sent now.
+ organization_id=session.organization_id or ctx.org_id,
+ team_id=session.team_id if session.organization_id else ctx.team_id,
+ expert_id=session.expert_id,
+ purpose=request.kind,
+ schedule_id=job_id,
+ )
return AtResetTurn(kind=request.kind, run_at=run_at, message=message)
@@ -159,9 +164,10 @@ async def cancel_at_reset(
"""Cancel the chat's resume or send at the budget reset; one that isn't
set is already cancelled."""
await _validate_and_get_session(session_id, user_id)
- await _scheduler().cancel_at_reset_turn(
- user_id=user_id, session_id=session_id, kind=kind
- )
+ async with _holding(at_reset_schedule_id(kind, session_id)):
+ await _scheduler().cancel_at_reset_turn(
+ user_id=user_id, session_id=session_id, kind=kind
+ )
return Response(status_code=204)
@@ -178,11 +184,30 @@ async def _run_at(
except NoResetAhead as e:
raise HTTPException(status_code=409, detail=e.reason) from e
except RateLimitUnavailable as e:
- raise HTTPException(
- status_code=503,
- detail="Rate limit service degraded, retry shortly",
- headers={"Retry-After": "30"},
- ) from e
+ raise _service_degraded() from e
+
+
+@asynccontextmanager
+async def _holding(job_id: str) -> AsyncIterator[None]:
+ """Hold the chat's job from reading it to setting it, on any replica.
+
+ Two tabs, or the scheduler putting the job back after it ran, would
+ otherwise each write over the other, and a send written meanwhile would
+ be lost. A 503 rather than going on without it.
+ """
+ async with at_reset_job_lock(job_id) as locked:
+ if not locked:
+ raise _service_degraded()
+ yield
+
+
+def _service_degraded() -> HTTPException:
+ """Redis, behind the budget and the job's lock, can't answer right now."""
+ return HTTPException(
+ status_code=503,
+ detail="Rate limit service degraded, retry shortly",
+ headers={"Retry-After": "30"},
+ )
async def _joined_with_waiting_send(user_id: str, session_id: str, message: str) -> str:
diff --git a/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py b/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py
index fbc178e0c4..49ec79b4bd 100644
--- a/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py
@@ -1,22 +1,28 @@
"""Route tests for /sessions/{session_id}/at-reset: setting, listing and
cancelling a resume or send at the budget reset.
-The session checks, the budget reads and the scheduler are mocked at the
-boundary; ``backend/copilot/at_reset_test.py`` covers when the turn fires.
+The session checks, the budget reads, the scheduler and Redis (the chat's job
+lock) are mocked at the boundary; ``backend/copilot/at_reset_test.py`` covers
+when the turn fires.
"""
+import asyncio
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock
import fastapi
import fastapi.testclient
+import httpx
import pytest
import pytest_mock
from autogpt_libs.auth.dependencies import get_request_context
from autogpt_libs.auth.jwt_utils import get_jwt_payload
+from redis.exceptions import ConnectionError as RedisConnectionError
from backend.api.features.chat import at_reset as at_reset_routes
+from backend.copilot import at_reset_lock
from backend.copilot.at_reset import NoResetAhead
+from backend.copilot.at_reset_lock_fixtures import FakeAtResetRedis
from backend.copilot.continuation import RESUMED_AT_RESET_NOTE, as_model_prompt
from backend.copilot.rate_limit import RateLimitUnavailable
from backend.executor.scheduler import CopilotTurnJobInfo
@@ -79,6 +85,14 @@ def run_at(mocker: pytest_mock.MockerFixture) -> AsyncMock:
)
[email protected](autouse=True)
+def job_locks(mocker: pytest_mock.MockerFixture) -> FakeAtResetRedis:
+ """Redis, where each chat's at-reset job is locked across replicas."""
+ redis = FakeAtResetRedis()
+ mocker.patch.object(at_reset_lock, "get_redis_async", AsyncMock(return_value=redis))
+ return redis
+
+
def _when(body: dict) -> datetime:
return datetime.fromisoformat(body["run_at"])
@@ -157,6 +171,58 @@ def test_a_second_send_joins_the_message_already_waiting(scheduler, session, run
assert scheduler.add_copilot_turn_schedule.await_args.kwargs["message"] == joined
+class _ChatJobs:
+ """The scheduler's jobs for the chat, in memory. A listing reads them and
+ then waits to be let go, the window two writers race in."""
+
+ def __init__(self) -> None:
+ self.jobs: dict[str, CopilotTurnJobInfo] = {}
+ self.listed = asyncio.Event()
+ self.let_go = asyncio.Event()
+
+ async def get_execution_schedules(self, **_) -> list[CopilotTurnJobInfo]:
+ jobs = list(self.jobs.values())
+ self.listed.set()
+ await self.let_go.wait()
+ return jobs
+
+ async def add_copilot_turn_schedule(
+ self, *, purpose: str, message: str, schedule_id: str, **_
+ ) -> None:
+ self.jobs[schedule_id] = _job(purpose, message)
+
+
+async def test_two_sends_at_once_both_end_up_in_the_waiting_message(
+ session, run_at, mocker: pytest_mock.MockerFixture
+):
+ # Two tabs, or two API replicas: each read the waiting message before the
+ # other set its own, so the second set replaced the first.
+ jobs = _ChatJobs()
+ mocker.patch.object(at_reset_routes, "get_scheduler_client", return_value=jobs)
+
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=app), base_url="http://test"
+ ) as http:
+
+ async def send(message: str) -> httpx.Response:
+ return await http.post(
+ f"/sessions/{SESSION}/at-reset",
+ json={"kind": "send", "message": message},
+ )
+
+ first = asyncio.create_task(send("draft the launch post"))
+ await jobs.listed.wait()
+ second = asyncio.create_task(send("also the changelog"))
+ await asyncio.sleep(0.2)
+ jobs.let_go.set()
+ responses = await asyncio.gather(first, second)
+
+ assert [r.status_code for r in responses] == [200, 200]
+ (waiting,) = jobs.jobs.values()
+ assert waiting.message == "draft the launch post\n\nalso the changelog"
+
+
def test_a_joined_send_longer_than_a_message_is_413(scheduler, session, run_at):
scheduler.get_execution_schedules.return_value = [_job("send", "x" * 60_000)]
response = client.post(
@@ -245,6 +311,33 @@ def test_unreadable_usage_is_503(scheduler, session, run_at):
assert response.headers["Retry-After"] == "30"
+ "method, path, body",
+ [
+ ("POST", f"/sessions/{SESSION}/at-reset", {"kind": "send", "message": "x"}),
+ ("POST", f"/sessions/{SESSION}/at-reset", {"kind": "resume"}),
+ ("DELETE", f"/sessions/{SESSION}/at-reset/send", None),
+ ],
+)
+def test_a_job_it_cannot_lock_is_503_and_left_alone(
+ method, path, body, scheduler, session, run_at, mocker: pytest_mock.MockerFixture
+):
+ # Going on without the lock could lose a send another tab just wrote.
+ mocker.patch.object(
+ at_reset_lock,
+ "get_redis_async",
+ AsyncMock(side_effect=RedisConnectionError("down")),
+ )
+
+ response = client.request(method, path, json=body)
+
+ assert response.status_code == 503
+ assert response.headers["Retry-After"] == "30"
+ scheduler.get_execution_schedules.assert_not_awaited()
+ scheduler.add_copilot_turn_schedule.assert_not_awaited()
+ scheduler.cancel_at_reset_turn.assert_not_awaited()
+
+
def _job(purpose: str, message: str) -> CopilotTurnJobInfo:
return CopilotTurnJobInfo.model_validate(
{
@@ -296,6 +389,25 @@ def test_cancel_is_204_whether_or_not_one_was_set(
)
+def test_a_cancel_holds_the_jobs_lock_while_it_cancels(scheduler, session, job_locks):
+ # A cancel landing between a put-back's check and its re-add finds no job
+ # to remove, and the re-add then brings the cancelled job back.
+ key = f"copilot:at_reset_lock:at-reset-send-{SESSION}"
+ held_while_cancelling: list[bool] = []
+
+ def cancel(**_) -> bool:
+ held_while_cancelling.append(job_locks.is_locked(key))
+ return True
+
+ scheduler.cancel_at_reset_turn.side_effect = cancel
+
+ response = client.delete(f"/sessions/{SESSION}/at-reset/send")
+
+ assert response.status_code == 204
+ assert held_while_cancelling == [True]
+ assert not job_locks.is_locked(key)
+
+
def test_cancel_an_unknown_kind_is_422(scheduler, session):
response = client.delete(f"/sessions/{SESSION}/at-reset/later")
diff --git a/autogpt_platform/backend/backend/copilot/at_reset_lock.py b/autogpt_platform/backend/backend/copilot/at_reset_lock.py
new file mode 100644
index 0000000000..95ab67b276
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/at_reset_lock.py
@@ -0,0 +1,105 @@
+"""One writer at a time on a chat's turn at the budget reset, on any replica.
+
+A chat has at most one resume and one send at the reset, each a scheduler job
+under the chat's own id (``at_reset_schedule_id``). Three things read that job
+and then set it: the API joining a new send onto the one waiting, the
+scheduler putting a job back after it ran, and the API cancelling one. API
+servers and schedulers each run as several replicas, so each of those holds
+this Redis lock, keyed on the job's id, from its read to its write.
+"""
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+from redis.asyncio.lock import Lock as AsyncRedisLock
+from redis.exceptions import LockNotOwnedError
+
+from backend.data.redis_client import get_redis_async
+
+logger = logging.getLogger(__name__)
+
+_KEY_PREFIX = "copilot:at_reset_lock:"
+# A read and a write to the scheduler take well under a second, but its client
+# retries a failing call for much longer, so the holder renews the lock while
+# it works. The expiry frees a chat whose holder died before letting go.
+_LOCK_TTL_SECONDS = 10
+_LOCK_RENEW_EVERY_SECONDS = 3
+# How long a writer waits for the one ahead of it.
+_LOCK_WAIT_SECONDS = 5
+# How long a lock call also waits on a Redis that doesn't answer. Its client
+# retries for far longer, and each caller has its own way on without the lock.
+_REDIS_DEADLINE_SECONDS = 2
+
+# Renewals under way, held so one isn't collected before it ends.
+_renewals: set[asyncio.Task[None]] = set()
+
+
+@asynccontextmanager
+async def at_reset_job_lock(job_id: str) -> AsyncIterator[bool]:
+ """Hold the at-reset job *job_id* from reading it to setting it.
+
+ Yields whether it is held: not when Redis can't be reached, or when
+ another writer kept it past the wait. Each caller decides whether to go
+ on without it.
+ """
+ lock = await _acquire(job_id)
+ if lock is None:
+ yield False
+ return
+ renewal = asyncio.create_task(_keep_renewed(lock, job_id))
+ _renewals.add(renewal)
+ renewal.add_done_callback(_renewals.discard)
+ try:
+ yield True
+ finally:
+ renewal.cancel()
+ await _release(lock, job_id)
+
+
+async def _acquire(job_id: str) -> AsyncRedisLock | None:
+ try:
+ async with asyncio.timeout(_LOCK_WAIT_SECONDS + _REDIS_DEADLINE_SECONDS):
+ redis = await get_redis_async()
+ lock = redis.lock(
+ f"{_KEY_PREFIX}{job_id}",
+ timeout=_LOCK_TTL_SECONDS,
+ blocking_timeout=_LOCK_WAIT_SECONDS,
+ )
+ if await lock.acquire():
+ return lock
+ logger.warning(f"At-reset job {job_id} stayed locked for {_LOCK_WAIT_SECONDS}s")
+ except Exception:
+ logger.warning(f"Could not lock at-reset job {job_id}", exc_info=True)
+ return None
+
+
+async def _keep_renewed(lock: AsyncRedisLock, job_id: str) -> None:
+ """Reset the lock's expiry while its holder works, so a slow scheduler
+ call doesn't let it lapse under the holder."""
+ while True:
+ await asyncio.sleep(_LOCK_RENEW_EVERY_SECONDS)
+ try:
+ async with asyncio.timeout(_REDIS_DEADLINE_SECONDS):
+ await lock.reacquire()
+ except LockNotOwnedError:
+ logger.warning(f"Lost the lock on at-reset job {job_id} while holding it")
+ return
+ except Exception:
+ # A blip: the lock outlives a few missed renewals.
+ logger.warning(
+ f"Could not renew the lock on at-reset job {job_id}", exc_info=True
+ )
+
+
+async def _release(lock: AsyncRedisLock, job_id: str) -> None:
+ try:
+ async with asyncio.timeout(_REDIS_DEADLINE_SECONDS):
+ await lock.release()
+ except Exception:
+ # Held past its expiry, so the next writer may have it now, or Redis
+ # went away; the expiry frees it either way.
+ logger.warning(
+ f"Could not release the lock on at-reset job {job_id}", exc_info=True
+ )
diff --git a/autogpt_platform/backend/backend/copilot/at_reset_lock_fixtures.py b/autogpt_platform/backend/backend/copilot/at_reset_lock_fixtures.py
new file mode 100644
index 0000000000..f8eb52088f
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/at_reset_lock_fixtures.py
@@ -0,0 +1,114 @@
+"""Redis as the at-reset tests see it, in memory: each chat's job lock, and
+the cancels the scheduler shares between its replicas."""
+
+import asyncio
+from collections.abc import Callable
+from datetime import timedelta
+from types import TracebackType
+
+
+class FakeJobLock:
+ """redis-py's async ``Lock``: one holder per name, waited for up to its
+ ``blocking_timeout``, its renewals counted."""
+
+ def __init__(
+ self, held: asyncio.Lock, blocking_timeout: float, renewals: list[float]
+ ) -> None:
+ self._held = held
+ self._wait = blocking_timeout
+ self._renewals = renewals
+
+ async def acquire(self) -> bool:
+ try:
+ async with asyncio.timeout(self._wait):
+ await self._held.acquire()
+ except TimeoutError:
+ return False
+ return True
+
+ async def reacquire(self) -> bool:
+ self._renewals.append(asyncio.get_running_loop().time())
+ return True
+
+ async def release(self) -> None:
+ self._held.release()
+
+
+class FakeTransaction:
+ """A ``pipeline(transaction=True)``: its commands run together on
+ ``execute``."""
+
+ def __init__(self, redis: "FakeAtResetRedis") -> None:
+ self._redis = redis
+ self._queued: list[Callable[[], object]] = []
+
+ async def __aenter__(self) -> "FakeTransaction":
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
+ self._queued.clear()
+
+ def zadd(self, key: str, mapping: dict[str, int], *, gt: bool) -> None:
+ self._queued.append(lambda: self._redis.add_scores(key, mapping, gt=gt))
+
+ def expire(self, key: str, ttl: timedelta) -> None:
+ self._queued.append(lambda: self._redis.expiries.update({key: ttl}))
+
+ async def execute(self) -> list[object]:
+ results = [run() for run in self._queued]
+ self._queued.clear()
+ return results
+
+
+class FakeAtResetRedis:
+ """What every replica sees in Redis, held in this process."""
+
+ def __init__(self) -> None:
+ # Each sorted set's members and scores, by key.
+ self.scores: dict[str, dict[str, float]] = {}
+ self.expiries: dict[str, timedelta] = {}
+ self.lock_options: dict[str, dict[str, float]] = {}
+ # When each lock's holder renewed it, by name.
+ self.renewals: dict[str, list[float]] = {}
+ self._locks: dict[str, asyncio.Lock] = {}
+
+ def lock(
+ self, name: str, *, timeout: float, blocking_timeout: float
+ ) -> FakeJobLock:
+ self.lock_options[name] = {
+ "timeout": timeout,
+ "blocking_timeout": blocking_timeout,
+ }
+ return FakeJobLock(
+ self._locks.setdefault(name, asyncio.Lock()),
+ blocking_timeout,
+ self.renewals.setdefault(name, []),
+ )
+
+ def is_locked(self, name: str) -> bool:
+ lock = self._locks.get(name)
+ return lock is not None and lock.locked()
+
+ def pipeline(self, *, transaction: bool) -> FakeTransaction:
+ assert transaction
+ return FakeTransaction(self)
+
+ def add_scores(self, key: str, mapping: dict[str, int], *, gt: bool) -> int:
+ """``ZADD``, with ``GT`` only ever moving a member's score up."""
+ members = self.scores.setdefault(key, {})
+ added = 0
+ for member, score in mapping.items():
+ if member not in members:
+ added += 1
+ elif gt and score <= members[member]:
+ continue
+ members[member] = score
+ return added
+
+ async def zscore(self, key: str, member: str) -> float | None:
+ return self.scores.get(key, {}).get(member)
diff --git a/autogpt_platform/backend/backend/copilot/at_reset_lock_test.py b/autogpt_platform/backend/backend/copilot/at_reset_lock_test.py
new file mode 100644
index 0000000000..643117c596
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/at_reset_lock_test.py
@@ -0,0 +1,176 @@
+"""The lock every writer of a chat's at-reset job holds, on any replica.
+
+Redis is faked at the boundary; the route and scheduler tests cover what each
+writer does when it can't have the lock.
+"""
+
+import asyncio
+import itertools
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+import pytest_mock
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import LockNotOwnedError
+
+from backend.copilot import at_reset_lock
+from backend.copilot.at_reset_lock import at_reset_job_lock
+from backend.copilot.at_reset_lock_fixtures import FakeAtResetRedis
+
+JOB = "at-reset-send-sess-1"
+KEY = f"copilot:at_reset_lock:{JOB}"
+
+
+def redis(mocker: pytest_mock.MockerFixture) -> FakeAtResetRedis:
+ fake = FakeAtResetRedis()
+ mocker.patch.object(at_reset_lock, "get_redis_async", AsyncMock(return_value=fake))
+ return fake
+
+
+def renew_often(mocker: pytest_mock.MockerFixture) -> None:
+ mocker.patch.object(at_reset_lock, "_LOCK_RENEW_EVERY_SECONDS", 0.01)
+
+
+async def test_it_holds_the_job_by_its_id_for_seconds(redis):
+ async with at_reset_job_lock(JOB) as locked:
+ assert locked
+ assert redis.is_locked(KEY)
+ assert not redis.is_locked(KEY)
+ # A holder that dies frees the chat in seconds, and a writer waits for
+ # the one ahead of it no longer than that.
+ assert redis.lock_options[KEY] == {"timeout": 10, "blocking_timeout": 5}
+
+
+async def test_it_lets_go_when_the_write_fails(redis):
+ with pytest.raises(RuntimeError):
+ async with at_reset_job_lock(JOB):
+ raise RuntimeError("scheduler down")
+ assert not redis.is_locked(KEY)
+
+
+async def test_without_redis_it_is_not_held(mocker: pytest_mock.MockerFixture):
+ mocker.patch.object(
+ at_reset_lock,
+ "get_redis_async",
+ AsyncMock(side_effect=RedisConnectionError("down")),
+ )
+ async with at_reset_job_lock(JOB) as locked:
+ assert locked is False
+
+
+async def test_it_is_not_held_when_another_writer_keeps_it_past_the_wait(
+ redis, mocker: pytest_mock.MockerFixture
+):
+ lock = MagicMock(acquire=AsyncMock(return_value=False), release=AsyncMock())
+ mocker.patch.object(redis, "lock", return_value=lock)
+ async with at_reset_job_lock(JOB) as locked:
+ assert locked is False
+ lock.release.assert_not_awaited()
+
+
+async def test_a_lock_held_past_its_expiry_is_let_go_quietly(
+ redis, mocker: pytest_mock.MockerFixture
+):
+ # The next writer may hold it by then; the write itself went through.
+ lock = MagicMock(
+ acquire=AsyncMock(return_value=True),
+ release=AsyncMock(side_effect=LockNotOwnedError("expired")),
+ )
+ mocker.patch.object(redis, "lock", return_value=lock)
+ async with at_reset_job_lock(JOB) as locked:
+ assert locked
+ lock.release.assert_awaited_once()
+
+
+async def test_a_slow_write_keeps_the_lock_until_it_lets_go(redis, renew_often):
+ # The scheduler's client retries a failing call for far longer than the
+ # lock's expiry, and the holder mustn't lose the lock under it.
+ async with at_reset_job_lock(JOB):
+ await asyncio.sleep(0.05)
+ renewed = len(redis.renewals[KEY])
+ assert renewed >= 2
+ await asyncio.sleep(0.05)
+ assert len(redis.renewals[KEY]) == renewed
+
+
+async def test_renewing_stops_once_the_lock_is_gone(
+ redis, renew_often, mocker: pytest_mock.MockerFixture
+):
+ lock = MagicMock(
+ acquire=AsyncMock(return_value=True),
+ reacquire=AsyncMock(side_effect=LockNotOwnedError("taken")),
+ release=AsyncMock(),
+ )
+ mocker.patch.object(redis, "lock", return_value=lock)
+ async with at_reset_job_lock(JOB) as locked:
+ assert locked
+ await asyncio.sleep(0.05)
+ lock.reacquire.assert_awaited_once()
+ lock.release.assert_awaited_once()
+
+
+async def test_a_renewal_redis_misses_is_tried_again(
+ redis, renew_often, mocker: pytest_mock.MockerFixture
+):
+ lock = MagicMock(
+ acquire=AsyncMock(return_value=True),
+ reacquire=AsyncMock(
+ side_effect=itertools.chain(
+ [RedisConnectionError("blip")], itertools.repeat(True)
+ )
+ ),
+ release=AsyncMock(),
+ )
+ mocker.patch.object(redis, "lock", return_value=lock)
+ async with at_reset_job_lock(JOB):
+ await asyncio.sleep(0.05)
+ assert lock.reacquire.await_count >= 2
+
+
+async def test_a_redis_that_never_answers_leaves_it_unheld(
+ redis, mocker: pytest_mock.MockerFixture
+):
+ # Its client retries for far longer than a writer should wait.
+ mocker.patch.object(at_reset_lock, "_LOCK_WAIT_SECONDS", 0.01)
+ mocker.patch.object(at_reset_lock, "_REDIS_DEADLINE_SECONDS", 0.02)
+ never = asyncio.Event()
+ lock = MagicMock(acquire=AsyncMock(side_effect=never.wait), release=AsyncMock())
+ mocker.patch.object(redis, "lock", return_value=lock)
+
+ async def hold() -> bool:
+ async with at_reset_job_lock(JOB) as locked:
+ return locked
+
+ assert await asyncio.wait_for(hold(), timeout=5) is False
+ lock.release.assert_not_awaited()
+
+
+async def test_a_release_redis_never_answers_doesnt_hold_up_the_writer(
+ redis, mocker: pytest_mock.MockerFixture
+):
+ mocker.patch.object(at_reset_lock, "_REDIS_DEADLINE_SECONDS", 0.02)
+ never = asyncio.Event()
+ lock = MagicMock(
+ acquire=AsyncMock(return_value=True),
+ release=AsyncMock(side_effect=never.wait),
+ )
+ mocker.patch.object(redis, "lock", return_value=lock)
+
+ async def hold() -> None:
+ async with at_reset_job_lock(JOB):
+ pass
+
+ await asyncio.wait_for(hold(), timeout=5)
+ lock.release.assert_awaited_once()
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index ce2f33dfed..ef1aa7506c 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -8,6 +8,7 @@ import time
import uuid
from collections.abc import Callable
from concurrent.futures import Future
+from contextlib import nullcontext
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Annotated, Any, Literal, Optional, Union
@@ -39,6 +40,7 @@ from backend.copilot.at_reset import (
fire_verdict,
load_at_reset_chat,
)
+from backend.copilot.at_reset_lock import at_reset_job_lock
from backend.copilot.config import CopilotLlmAuthProvider
from backend.copilot.continuation import RESUMED_AT_RESET_NOTE
from backend.copilot.credential_selection import CredentialPins
@@ -60,6 +62,7 @@ from backend.copilot.turn_queue import TurnRefusal, turn_refusal
from backend.data.db_accessors import experts_db
from backend.data.execution import ExecutionTrigger, GraphExecutionWithNodes
from backend.data.model import CredentialsMetaInput, GraphInput
+from backend.data.redis_client import get_redis_async
from backend.data.schedule import normalize_schedule_name
from backend.executor import schedule_events
from backend.executor import utils as execution_utils
@@ -951,10 +954,20 @@ async def _drop_job_from_routine(args: "CopilotTurnJobArgs") -> None:
# When a chat's resume or send at the reset was cancelled, by job id. A job
# cancelled while it runs is already out of the job store, so the cancel
# finds nothing to remove; before firing or putting itself back the job looks
-# here. In process, like the jobs themselves: one scheduler runs them.
+# here. Kept in Redis, since the replica that takes the cancel need not be the
+# one running the job, and in this process too, for when Redis can't be read.
_at_reset_cancels: dict[str, datetime] = {}
_at_reset_cancels_lock = threading.Lock()
_AT_RESET_CANCEL_MEMORY = timedelta(hours=2)
+_AT_RESET_CANCEL_KEY_PREFIX = "copilot:at_reset_cancel:"
+# Kept as a sorted set's score, in microseconds since the epoch, and only
+# ever moved later: a write that stalled and lands after a newer cancel can't
+# put the older time back.
+_AT_RESET_CANCEL_MEMBER = "cancelled_at"
+_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
+# How long writing or reading a cancel waits on Redis, whose client retries
+# for far longer, before going on with only this replica's memory of it.
+_AT_RESET_CANCEL_SHARE_TIMEOUT_SECONDS = 2
# An at-reset turn that can't fire yet (the chat is busy, the budget or the
# expert can't be read, the queue turned it away) tries again five minutes
@@ -977,7 +990,7 @@ async def _execute_at_reset_turn(args: "CopilotTurnJobArgs", kind: AtResetKind)
if chat is None:
return
verdict = await fire_verdict(kind, chat, args.created_at)
- if _cancelled_after_set(args):
+ if await _cancelled_after_set(args):
logger.info(
f"{kind}-at-reset turn for session {label} was cancelled "
"while it ran; dropping it"
@@ -1092,13 +1105,33 @@ async def _rearm_at_reset_turn(args: "CopilotTurnJobArgs", *, reason: str) -> No
async def _readd_at_reset_turn(
args: "CopilotTurnJobArgs", *, run_at: datetime, retries: int
) -> None:
- """Put an at-reset turn back under its own id, keeping when it was set."""
- if _cancelled_after_set(args):
- logger.info(
- f"Not putting back the {args.purpose}-at-reset turn for session "
- f"{_session_id_label(args)}: it was cancelled while it ran"
- )
- return
+ """Put an at-reset turn back under its own id, keeping when it was set.
+
+ Under the job's lock, so a send the user writes or a cancel they make
+ meanwhile, on whichever replica, lands wholly before or after it.
+ """
+ job_lock = (
+ at_reset_job_lock(args.schedule_id) if args.schedule_id else nullcontext(True)
+ )
+ async with job_lock as locked:
+ if not locked:
+ # Losing the user's turn would be worse than racing their write.
+ logger.warning(
+ f"Putting back the {args.purpose}-at-reset turn for session "
+ f"{_session_id_label(args)} without its lock"
+ )
+ if await _cancelled_after_set(args):
+ logger.info(
+ f"Not putting back the {args.purpose}-at-reset turn for session "
+ f"{_session_id_label(args)}: it was cancelled while it ran"
+ )
+ return
+ await _put_back_at_reset_turn(args, run_at=run_at, retries=retries)
+
+
+async def _put_back_at_reset_turn(
+ args: "CopilotTurnJobArgs", *, run_at: datetime, retries: int
+) -> None:
try:
# Under its own id this is safe to repeat, and a put-back lost on a
# connection the scheduler had just closed would drop the user's turn.
@@ -1147,20 +1180,64 @@ def _remember_at_reset_cancel(job_id: str) -> None:
for key in stale:
del _at_reset_cancels[key]
_at_reset_cancels[job_id] = now
+ try:
+ # Waited for: the cancel route holds the job's lock until this
+ # returns, and a put-back that takes the lock next must find it.
+ run_async(
+ _share_at_reset_cancel(job_id, now),
+ # The write keeps its own deadline; this one only backs it up.
+ timeout=_AT_RESET_CANCEL_SHARE_TIMEOUT_SECONDS + 1,
+ )
+ except Exception:
+ logger.warning(
+ f"Could not share the cancel of at-reset job {job_id}; only this "
+ "replica knows of it",
+ exc_info=True,
+ )
+
+async def _share_at_reset_cancel(job_id: str, cancelled_at: datetime) -> None:
+ key = f"{_AT_RESET_CANCEL_KEY_PREFIX}{job_id}"
+ micros = (cancelled_at - _EPOCH) // timedelta(microseconds=1)
+ async with asyncio.timeout(_AT_RESET_CANCEL_SHARE_TIMEOUT_SECONDS):
+ redis = await get_redis_async()
+ async with redis.pipeline(transaction=True) as pipe:
+ pipe.zadd(key, {_AT_RESET_CANCEL_MEMBER: micros}, gt=True)
+ pipe.expire(key, _AT_RESET_CANCEL_MEMORY)
+ await pipe.execute()
-def _cancelled_after_set(args: "CopilotTurnJobArgs") -> bool:
+
+async def _cancelled_after_set(args: "CopilotTurnJobArgs") -> bool:
"""Whether the user cancelled this job after it was set: setting it
again after a cancel makes a new job, set later than the cancel."""
if args.schedule_id is None:
return False
- with _at_reset_cancels_lock:
- cancelled_at = _at_reset_cancels.get(args.schedule_id)
+ cancelled_at = await _at_reset_cancelled_at(args.schedule_id)
if cancelled_at is None:
return False
return args.created_at is None or args.created_at <= cancelled_at
+async def _at_reset_cancelled_at(job_id: str) -> datetime | None:
+ """When the job was last cancelled, on any replica. This replica's own
+ memory stands in for Redis when Redis can't be read."""
+ with _at_reset_cancels_lock:
+ here = _at_reset_cancels.get(job_id)
+ try:
+ async with asyncio.timeout(_AT_RESET_CANCEL_SHARE_TIMEOUT_SECONDS):
+ redis = await get_redis_async()
+ micros = await redis.zscore(
+ f"{_AT_RESET_CANCEL_KEY_PREFIX}{job_id}", _AT_RESET_CANCEL_MEMBER
+ )
+ except Exception:
+ logger.warning(
+ f"Could not read the cancels of at-reset job {job_id}", exc_info=True
+ )
+ return here
+ anywhere = None if micros is None else _EPOCH + timedelta(microseconds=micros)
+ return max((at for at in (here, anywhere) if at is not None), default=None)
+
+
async def _handle_graph_validation_error(args: "GraphExecutionJobArgs") -> None:
logger.warning(
f"Scheduled Graph {args.graph_id} failed validation. Unscheduling graph"
diff --git a/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py b/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py
index c91223f3e8..66cb0f6850 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py
@@ -1,6 +1,9 @@
"""The scheduler's side of resume and send at the budget reset: the job's
id and shape, and what a due one does (``copilot/at_reset.py`` decides)."""
+import asyncio
+import logging
+import threading
from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
@@ -8,9 +11,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.schedulers.background import BackgroundScheduler
+from redis.exceptions import ConnectionError as RedisConnectionError
from backend.copilot.active_turns import ConcurrentTurnLimitError
from backend.copilot.at_reset import at_reset_schedule_id, reset_jitter_seconds
+from backend.copilot.at_reset_lock import at_reset_job_lock
+from backend.copilot.at_reset_lock_fixtures import FakeAtResetRedis, FakeTransaction
from backend.copilot.continuation import RESUMED_AT_RESET_NOTE
from backend.copilot.db import PaginatedMessages
from backend.copilot.executor.utils import ScheduledTurnOrigin
@@ -26,11 +32,14 @@ from backend.executor.scheduler import (
CopilotTurnJobArgs,
Jobstores,
Scheduler,
+ _at_reset_cancelled_at,
_execute_copilot_turn,
+ _share_at_reset_cancel,
)
_SCHEDULER = "backend.executor.scheduler"
_AT_RESET = "backend.copilot.at_reset"
+_LOCK = "backend.copilot.at_reset_lock"
USER = "user-1"
SESSION = "0d3f7a52-61c4-4f0e-a8b2-9c1e5d7f3a64"
SET_AT = datetime(2026, 10, 14, 15, 0, tzinfo=UTC)
@@ -74,11 +83,22 @@ async def _fire(args: CopilotTurnJobArgs) -> None:
@pytest.fixture(autouse=True)
def no_cancels():
- """Cancels are remembered in process: no test sees another's."""
+ """Each replica also remembers cancels itself: no test sees another's."""
with patch.dict(f"{_SCHEDULER}._at_reset_cancels", clear=True):
yield
[email protected](autouse=True)
+def shared_redis():
+ """Redis as every replica sees it: each chat's job lock, and the cancels."""
+ redis = FakeAtResetRedis()
+ with (
+ patch(f"{_SCHEDULER}.get_redis_async", AsyncMock(return_value=redis)),
+ patch(f"{_LOCK}.get_redis_async", AsyncMock(return_value=redis)),
+ ):
+ yield redis
+
+
@pytest.fixture
def fire_env():
"""A due at-reset turn on an idle, platform-route chat with budget left."""
@@ -112,14 +132,22 @@ def fire_env():
@pytest.fixture
def live_scheduler():
- """The service over an in-memory jobstore, paused so nothing fires."""
+ """The service over an in-memory jobstore, paused so nothing fires, with
+ the event loop its handlers hand their async work to."""
+ loop = asyncio.new_event_loop()
+ loop_thread = threading.Thread(target=loop.run_forever, daemon=True)
+ loop_thread.start()
service = Scheduler(register_system_tasks=False)
service.scheduler = BackgroundScheduler(
jobstores={Jobstores.EXECUTION.value: MemoryJobStore()}
)
service.scheduler.start(paused=True)
- yield service
+ with patch(f"{_SCHEDULER}._event_loop", loop):
+ yield service
service.scheduler.shutdown(wait=False)
+ loop.call_soon_threadsafe(loop.stop)
+ loop_thread.join()
+ loop.close()
def test_a_row_from_before_the_field_loads_as_a_followup():
@@ -370,6 +398,172 @@ async def test_a_send_set_again_after_a_cancel_still_fires(fire_env, live_schedu
fire_env.dispatch.assert_awaited_once()
+def _cancel_on_another_replica(service: Scheduler) -> None:
+ """A cancel taken by a replica other than the one running the job: that
+ replica's own memory of it stays there, and only Redis is shared."""
+ with patch.dict(f"{_SCHEDULER}._at_reset_cancels", clear=True):
+ service.cancel_at_reset_turn(USER, SESSION, "send")
+
+
+def test_a_cancel_is_kept_in_redis_for_two_hours(live_scheduler, shared_redis):
+ before = datetime.now(UTC)
+ assert live_scheduler.cancel_at_reset_turn(USER, SESSION, "send") is False
+ key = f"copilot:at_reset_cancel:at-reset-send-{SESSION}"
+ micros = shared_redis.scores[key]["cancelled_at"]
+ assert datetime(1970, 1, 1, tzinfo=UTC) + timedelta(microseconds=micros) >= before
+ assert shared_redis.expiries[key] == timedelta(hours=2)
+
+
+async def test_a_cancel_written_late_never_moves_the_time_back():
+ # A write that stalled can land after a newer cancel on another replica.
+ # The newer time has to stand, or a send set between the two would fire.
+ job = at_reset_schedule_id("send", SESSION)
+ newer = datetime.now(UTC)
+ await _share_at_reset_cancel(job, newer)
+ await _share_at_reset_cancel(job, newer - timedelta(seconds=30))
+ assert await _at_reset_cancelled_at(job) == newer
+
+
+async def test_a_redis_that_never_answers_leaves_this_replicas_own_cancels(
+ shared_redis,
+):
+ job = at_reset_schedule_id("send", SESSION)
+ here = datetime.now(UTC)
+
+ async def never_answers(*_: object) -> None:
+ await asyncio.Event().wait()
+
+ with (
+ patch.dict(f"{_SCHEDULER}._at_reset_cancels", {job: here}),
+ patch(f"{_SCHEDULER}._AT_RESET_CANCEL_SHARE_TIMEOUT_SECONDS", 0.05),
+ patch.object(shared_redis, "zscore", never_answers),
+ ):
+ assert await asyncio.wait_for(_at_reset_cancelled_at(job), timeout=2) == here
+
+
+async def test_a_cancel_redis_never_takes_gives_up_by_itself():
+ # Its waiter in the request thread gives up too, but the write itself
+ # must not linger and land late.
+ never = asyncio.Event()
+ started = asyncio.get_running_loop().time()
+ with (
+ patch(f"{_SCHEDULER}._AT_RESET_CANCEL_SHARE_TIMEOUT_SECONDS", 0.05),
+ patch.object(FakeTransaction, "execute", AsyncMock(side_effect=never.wait)),
+ pytest.raises(TimeoutError),
+ ):
+ await asyncio.wait_for(
+ _share_at_reset_cancel(
+ at_reset_schedule_id("send", SESSION), datetime.now(UTC)
+ ),
+ timeout=2,
+ )
+ assert asyncio.get_running_loop().time() - started < 1
+
+
+async def test_a_cancel_on_another_replica_stops_the_job_firing(
+ fire_env, live_scheduler
+):
+ _cancel_on_another_replica(live_scheduler)
+ await _fire(_args("send", created_at=datetime.now(UTC) - timedelta(minutes=1)))
+ fire_env.dispatch.assert_not_awaited()
+ fire_env.client.add_copilot_turn_schedule.assert_not_awaited()
+
+
+async def test_a_cancel_on_another_replica_keeps_the_job_from_being_put_back(
+ fire_env, live_scheduler
+):
+ # The user cancels while the queue turns the send away: the cancel finds
+ # no job to remove, so only the put-back's own check can stop it.
+ def refused_as_the_user_cancels(**_):
+ _cancel_on_another_replica(live_scheduler)
+ raise ConcurrentTurnLimitError("cap")
+
+ fire_env.dispatch.side_effect = refused_as_the_user_cancels
+ await _fire(_args("send", created_at=datetime.now(UTC) - timedelta(minutes=1)))
+ fire_env.client.add_copilot_turn_schedule.assert_not_awaited()
+
+
+async def test_a_send_set_again_after_a_cancel_on_another_replica_fires(
+ fire_env, live_scheduler
+):
+ _cancel_on_another_replica(live_scheduler)
+ await _fire(_args("send", created_at=datetime.now(UTC) + timedelta(seconds=1)))
+ fire_env.dispatch.assert_awaited_once()
+
+
+async def test_a_cancel_waits_for_a_put_back_under_way(fire_env, live_scheduler):
+ # A cancel landing between the put-back's check and its re-add found no
+ # job to remove, and the re-add then brought the cancelled job back.
+ job_id = at_reset_schedule_id("send", SESSION)
+ adding, let_go = asyncio.Event(), asyncio.Event()
+
+ async def put_back(**job) -> None:
+ adding.set()
+ await let_go.wait()
+ live_scheduler.add_copilot_turn_schedule(**job)
+
+ async def cancel_as_the_route_does() -> None:
+ async with at_reset_job_lock(job_id) as locked:
+ assert locked
+ live_scheduler.cancel_at_reset_turn(USER, SESSION, "send")
+
+ fire_env.client.add_copilot_turn_schedule.side_effect = put_back
+ fire_env.busy.return_value = True
+ set_at = datetime.now(UTC) - timedelta(minutes=1)
+ firing = asyncio.create_task(_fire(_args("send", created_at=set_at)))
+ await adding.wait()
+ cancelling = asyncio.create_task(cancel_as_the_route_does())
+ await asyncio.wait({cancelling}, timeout=0.2)
+ let_go.set()
+ await asyncio.gather(firing, cancelling)
+
+ assert (
+ live_scheduler.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)
+ is None
+ )
+
+
+async def test_a_put_back_that_cannot_lock_its_job_still_puts_it_back(fire_env, caplog):
+ # Losing the user's turn would be worse than racing their write.
+ fire_env.busy.return_value = True
+ down = AsyncMock(side_effect=RedisConnectionError("down"))
+ with (
+ patch(f"{_LOCK}.get_redis_async", down),
+ caplog.at_level(logging.WARNING, logger=_SCHEDULER),
+ ):
+ await _fire(_args("send"))
+ fire_env.client.add_copilot_turn_schedule.assert_awaited_once()
+ assert any("without its lock" in r.getMessage() for r in caplog.records)
+
+
+async def test_without_redis_a_replica_still_keeps_its_own_cancels(
+ fire_env, live_scheduler
+):
+ # As before cancels were shared: the replica that took the cancel keeps
+ # the job from coming back.
+ def refused_as_the_user_cancels(**_):
+ live_scheduler.cancel_at_reset_turn(USER, SESSION, "send")
+ raise ConcurrentTurnLimitError("cap")
+
+ fire_env.dispatch.side_effect = refused_as_the_user_cancels
+ down = AsyncMock(side_effect=RedisConnectionError("down"))
+ with (
+ patch(f"{_SCHEDULER}.get_redis_async", down),
+ patch(f"{_LOCK}.get_redis_async", down),
+ ):
+ await _fire(_args("send", created_at=datetime.now(UTC) - timedelta(minutes=1)))
+ fire_env.client.add_copilot_turn_schedule.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_a_builder_chat_keeps_its_tool_limits(fire_env):
chat = _chat()