AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): work notification queues concurrently and survive a channel swap (#14546)
Bentlycaf86b669d6a
diff --git a/autogpt_platform/backend/backend/notifications/consumer_test.py b/autogpt_platform/backend/backend/notifications/consumer_test.pynew file mode 100644index 0000000000..82500c291b--- /dev/null+++ b/autogpt_platform/backend/backend/notifications/consumer_test.py@@ -0,0 +1,641 @@+"""The queue consumer under load and under a lost channel.++Messages are worked concurrently up to the prefetch, a lost channel is a+reconnect rather than a retry of work already done, one hung handler cannot+hold the channel past the broker's ack timeout, and a consumer that fails+takes its siblings down with it so a restart cannot double-consume.+"""++import asyncio+import inspect+from collections.abc import Iterator+from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from aio_pika.exceptions import ChannelInvalidStateError+from postmarker.exceptions import ClientError++from backend.notifications import notifications as delivery+from backend.notifications.notifications import NotificationManager++[email protected](scope="session")+def server() -> None:+ return None++[email protected](scope="session", autouse=True)+def graph_cleanup() -> Iterator[None]:+ yield+++def _manager() -> NotificationManager:+ manager = NotificationManager.__new__(NotificationManager)+ manager.running = True+ return manager+++def _message(body: str = "{}") -> MagicMock:+ return MagicMock(+ body=body.encode(), ack=AsyncMock(), reject=AsyncMock(), nack=AsyncMock()+ )+++class _FakeIterator:+ """Stands in for `queue.iterator()`: yields the given messages, then+ either ends, waits for more like a live queue, or fails on exit."""++ def __init__(+ self,+ messages: list[Any],+ exit_error: BaseException | None = None,+ then_wait: bool = False,+ ):+ self._messages = list(messages)+ self._exit_error = exit_error+ self._then_wait = then_wait++ async def __aenter__(self):+ return self++ async def __aexit__(self, *_):+ if self._exit_error is not None:+ raise self._exit_error+ return False++ def __aiter__(self):+ return self++ async def __anext__(self):+ if self._messages:+ return self._messages.pop(0)+ if self._then_wait:+ await asyncio.Event().wait()+ raise StopAsyncIteration+++def _queue(+ messages: list[Any],+ exit_error: BaseException | None = None,+ then_wait: bool = False,+) -> MagicMock:+ return MagicMock(iterator=lambda: _FakeIterator(messages, exit_error, then_wait))+++# ── throughput ─────────────────────────────────────────────────────────────+++async def test_messages_are_worked_concurrently_up_to_the_prefetch():+ manager = _manager()+ total = delivery.CONSUMER_CONCURRENCY * 3+ active = peak = 0++ async def slow(_: str) -> bool:+ nonlocal active, peak+ active += 1+ peak = max(peak, active)+ await asyncio.sleep(0.02)+ active -= 1+ return True++ messages = [_message() for _ in range(total)]+ await manager._consume_queue(+ _queue(messages), slow, "q", delivery.Ordering.COMMUTATIVE+ )++ assert peak == delivery.CONSUMER_CONCURRENCY, (+ "the consumer must keep exactly as many messages in flight as the "+ "broker prefetches; fewer wastes the prefetch, more overruns it"+ )+ assert all(m.ack.await_count == 1 for m in messages)+++async def test_an_as_published_queue_works_one_message_at_a_time():+ """The declaration is what serialises the queue, not its name: a queue+ whose messages do not commute must never have two in flight, however many+ the broker prefetched."""+ manager = _manager()+ active = peak = 0++ async def slow(_: str) -> bool:+ nonlocal active, peak+ active += 1+ peak = max(peak, active)+ await asyncio.sleep(0.01)+ active -= 1+ return True++ messages = [_message() for _ in range(delivery.CONSUMER_CONCURRENCY * 2)]+ await manager._consume_queue(+ _queue(messages), slow, "q", delivery.Ordering.AS_PUBLISHED+ )++ assert peak == 1+ assert all(m.ack.await_count == 1 for m in messages)+++async def test_in_flight_work_finishes_before_the_consumer_returns():+ manager = _manager()+ done: list[int] = []++ async def slow(body: str) -> bool:+ await asyncio.sleep(0.01)+ done.append(int(body))+ return True++ messages = [_message(str(i)) for i in range(5)]+ await manager._consume_queue(+ _queue(messages), slow, "q", delivery.Ordering.COMMUTATIVE+ )++ assert sorted(done) == [0, 1, 2, 3, 4]+++# ── a lost channel ─────────────────────────────────────────────────────────+++async def test_a_lost_channel_on_ack_does_not_rerun_the_handler():+ """The work is done; the broker requeues the unacked delivery itself.+ Re-running the handler to 'retry' the ack was three more chances to send+ the same email."""+ manager = _manager()+ handler = AsyncMock(return_value=True)+ message = _message()+ message.ack.side_effect = ChannelInvalidStateError()++ with patch.object(delivery.asyncio, "sleep", AsyncMock()) as sleep:+ await manager._process_message_with_retry(message, handler, "q")++ assert handler.await_count == 1+ message.reject.assert_not_awaited()+ sleep.assert_not_awaited()+++async def test_a_lost_channel_on_reject_is_not_fatal():+ manager = _manager()+ message = _message()+ message.reject.side_effect = ChannelInvalidStateError()++ with patch.object(delivery.asyncio, "sleep", AsyncMock()):+ await manager._process_message_with_retry(+ message, AsyncMock(return_value=False), "q"+ )+++async def test_a_channel_swap_at_iterator_exit_asks_for_a_reconnect():+ """aio_pika's iterator nacks its buffer on exit; on a dead channel that+ fails once per buffered message and surfaces as one ExceptionGroup. That+ is a reconnect, and must not be logged as nine fatal errors."""+ manager = _manager()+ group = ExceptionGroup(+ "Unable to nack all messages", [ChannelInvalidStateError() for _ in range(9)]+ )++ with pytest.raises(ConnectionError, match="channel lost"):+ await manager._consume_queue(+ _queue([], exit_error=group),+ AsyncMock(return_value=True),+ "q",+ delivery.Ordering.COMMUTATIVE,+ )+++async def test_other_exception_groups_still_propagate():+ manager = _manager()+ group = ExceptionGroup("mixed", [ChannelInvalidStateError(), RuntimeError("x")])++ with pytest.raises(ExceptionGroup):+ await manager._consume_queue(+ _queue([], exit_error=group),+ AsyncMock(return_value=True),+ "q",+ delivery.Ordering.COMMUTATIVE,+ )+++async def test_one_failed_consumer_takes_its_siblings_with_it():+ """The four consumers share one channel. If one dies and the service is+ restarted around the other three, every message is handled twice."""+ manager = _manager()+ manager.rabbitmq_config = MagicMock()+ started = asyncio.Event()+ survivors: list[asyncio.Task[None]] = []++ async def consume(+ _queue: Any, _handler: Any, name: str, _ordering: delivery.Ordering+ ) -> None:+ task = asyncio.current_task()+ assert task is not None+ if name == delivery.PASS_WORK_QUEUE:+ await started.wait()+ raise ConnectionError("channel lost")+ survivors.append(task)+ await asyncio.Event().wait()++ channel = MagicMock(+ set_qos=AsyncMock(), get_queue=AsyncMock(return_value=MagicMock())+ )+ rabbit = MagicMock(connect=AsyncMock(), get_channel=AsyncMock(return_value=channel))++ with (+ patch.object(delivery.rabbitmq, "AsyncRabbitMQ", return_value=rabbit),+ patch.object(manager, "_consume_queue", consume),+ ):+ undecorated = inspect.unwrap(NotificationManager._run_service)+ run = asyncio.create_task(undecorated(manager))+ await asyncio.sleep(0.01)+ assert len(survivors) == 3+ started.set()+ with pytest.raises(ConnectionError):+ await run++ assert all(task.cancelled() for task in survivors)+++# ── one message cannot hold the channel open ───────────────────────────────+++async def test_a_hung_handler_is_a_transient_failure_not_a_channel_loss(monkeypatch):+ """RabbitMQ closes the channel when a delivery goes unacked for its+ consumer timeout, failing every other in-flight ack with it. A bounded+ wait turns that into one retried message."""+ monkeypatch.setattr(delivery, "MESSAGE_PROCESSING_TIMEOUT_SECONDS", 0.01)+ manager = _manager()+ message = _message()++ async def hang(_: str) -> bool:+ await asyncio.Event().wait()+ return True++ with patch.object(delivery.asyncio, "sleep", AsyncMock()) as sleep:+ await manager._process_message_with_retry(message, hang, "q")++ assert sleep.await_count == delivery.MAX_CONSUMER_RETRY_ATTEMPTS - 1+ message.reject.assert_awaited_once_with(requeue=False)+ message.ack.assert_not_awaited()+++# ── the handlers are supervised ────────────────────────────────────────────+++async def test_a_settle_failure_is_not_mistaken_for_a_handler_failure():+ """The work succeeded; only the ack failed. Retrying the handler would+ repeat the work, and the retry loop's own DLQ reject would then hide the+ broken channel behind a "sent to DLQ" line."""+ manager = _manager()+ handler = AsyncMock(return_value=True)+ message = _message()+ message.ack.side_effect = RuntimeError("channel is in a bad way")++ with patch.object(delivery.asyncio, "sleep", AsyncMock()) as sleep:+ with pytest.raises(RuntimeError, match="bad way"):+ await manager._process_message_with_retry(message, handler, "q")++ assert handler.await_count == 1+ sleep.assert_not_awaited()+ message.reject.assert_not_awaited()+++async def test_a_handler_that_cannot_settle_takes_the_consumer_down():+ """An ack that fails for a reason `_settle` does not absorb leaves the+ delivery unacked at the broker. Left running, enough of those pin the+ prefetch and the consumer looks alive while delivering nothing; failing+ it is what triggers the reconnect."""+ manager = _manager()+ broken = _message("0")+ broken.ack.side_effect = RuntimeError("channel is in a bad way")+ never = _message("1")++ with pytest.raises(ExceptionGroup) as raised:+ await manager._consume_queue(+ _queue([broken, never], then_wait=True),+ AsyncMock(return_value=True),+ "q",+ delivery.Ordering.COMMUTATIVE,+ )++ assert raised.value.subgroup(RuntimeError) is not None+++async def test_shutdown_lets_in_flight_handlers_settle_before_cancelling_them():+ """A handler cancelled between its send and its ack leaves an email+ delivered and the message unacked, which the broker then redelivers."""+ manager = _manager()+ message = _message()++ async def slow(_: str) -> bool:+ await asyncio.sleep(0.05)+ return True++ consumer = asyncio.create_task(+ manager._consume_queue(+ _queue([message], then_wait=True), slow, "q", delivery.Ordering.COMMUTATIVE+ )+ )+ await asyncio.sleep(0.01)+ consumer.cancel()+ with pytest.raises(asyncio.CancelledError):+ await consumer++ message.ack.assert_awaited_once()+++async def test_shutdown_does_not_wait_past_the_grace_for_a_handler(monkeypatch):+ """The other half of the grace, and the half with teeth: it is bounded.+ A handler sitting in its retry backoff is up to+ MAX_CONSUMER_RETRY_ATTEMPTS * MESSAGE_PROCESSING_TIMEOUT_SECONDS from+ returning, and `_shutdown_service` waits only SHUTDOWN_TIMEOUT_SECONDS+ for the consumers before it disconnects RabbitMQ. Waiting for such a+ handler to finish on its own puts that disconnect underneath it."""+ monkeypatch.setattr(delivery, "HANDLER_SHUTDOWN_GRACE_SECONDS", 0.05)+ manager = _manager()+ message = _message()+ cancelled = asyncio.Event()++ async def never(_: str) -> bool:+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelled.set()+ raise+ return True++ consumer = asyncio.create_task(+ manager._consume_queue(+ _queue([message], then_wait=True), never, "q", delivery.Ordering.COMMUTATIVE+ )+ )+ await asyncio.sleep(0.01)+ consumer.cancel()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(consumer, timeout=1)++ assert cancelled.is_set()+ # The honest half of the trade: this message is never settled, so the+ # broker redelivers it and anything the handler had already sent goes out+ # twice. The grace narrows that window, it does not close it.+ message.ack.assert_not_awaited()+++async def test_a_settle_failure_lets_its_siblings_finish_before_taking_them_down():+ """Where the two fixes meet, and they do not compose for free. Failing a+ handler is how the consumer is brought down, but a TaskGroup aborts by+ cancelling every sibling first and the task running the loop second, so+ the shutdown grace sits on the wrong side of the abort and never runs.+ Without a grace of its own on this path, one message that cannot be acked+ cancels the nine others the prefetch handed this consumer mid-send, and+ the broker redelivers every one of them as a second email."""+ manager = _manager()+ broken = _message("0")+ broken.ack.side_effect = RuntimeError("channel is in a bad way")+ sibling = _message("1")++ async def work(body: str) -> bool:+ if body == "1":+ await asyncio.sleep(0.05)+ return True++ with pytest.raises(ExceptionGroup) as raised:+ await manager._consume_queue(+ _queue([broken, sibling], then_wait=True),+ work,+ "q",+ delivery.Ordering.COMMUTATIVE,+ )++ assert raised.value.subgroup(RuntimeError) is not None+ sibling.ack.assert_awaited_once()++[email protected]("requeue_fails", [False, True])+async def test_no_handler_starts_while_a_settle_failure_waits_on_its_siblings(+ requeue_fails: bool,+):+ """The grace covers the handlers running when the settle failed, and no+ others. Each of those that finishes frees a slot, and a loop still pulling+ would start the next delivery in it, outside the grace: the abort then+ cancels that handler between its send and its ack, and the broker+ redelivers it as a second email. The prefetch is full and more is queued+ behind it, which is how a large fan-out looks when one ack fails. The+ channel that failed the ack may fail the requeue too, and that must not+ cut the grace short either."""+ manager = _manager()+ broken = _message("broken")+ broken.ack.side_effect = RuntimeError("channel is in a bad way")+ fast = _message("fast")+ slow = [_message(f"slow-{i}") for i in range(delivery.CONSUMER_CONCURRENCY - 2)]+ queued = [_message(f"queued-{i}") for i in range(delivery.CONSUMER_CONCURRENCY)]+ if requeue_fails:+ queued[0].nack.side_effect = RuntimeError("channel is in a bad way")+ sent: list[str] = []++ async def send(body: str) -> bool:+ sent.append(body)+ # `fast` frees a slot at once; the slow siblings hold the grace open+ # long enough that anything started in that slot is still mid-send+ # when the grace ends.+ await asyncio.sleep(+ {"broken": 0, "fast": 0.01}.get(body, 0.1 if "slow" in body else 0.5)+ )+ return True++ with pytest.raises(ExceptionGroup) as raised:+ await manager._consume_queue(+ _queue([broken, fast, *slow, *queued], then_wait=True),+ send,+ "q",+ delivery.Ordering.COMMUTATIVE,+ )++ assert raised.value.subgroup(RuntimeError) is not None+ sent_but_unacked = [+ m.body.decode()+ for m in [fast, *slow, *queued]+ if m.body.decode() in sent and not m.ack.await_count+ ]+ assert sent_but_unacked == []+ for sibling in [fast, *slow]:+ sibling.ack.assert_awaited_once()+ # The one delivery the loop had already pulled goes straight back to the+ # queue, rather than sitting unacked on a channel that may outlive it.+ queued[0].nack.assert_awaited_once_with(requeue=True)+++async def test_cancelling_the_service_still_lets_in_flight_handlers_settle():+ """The grace is only worth having if it survives the real shutdown path.+ `_shutdown_service` cancels the service task, not the consumers, and the+ cancel reaches them through `asyncio.gather`, which cancels its children+ and then waits for them before it raises — so `_run_service`'s own+ `finally` cannot land a second cancel on a consumer mid-grace and cut it+ short. Pinned here because none of that is visible from `_consume_queue`+ alone, where the other shutdown tests cancel the consumer directly."""+ manager = _manager()+ manager.rabbitmq_config = MagicMock()+ messages: list[MagicMock] = []++ async def slow(_: str) -> bool:+ await asyncio.sleep(0.05)+ return True++ def next_queue(_name: str) -> MagicMock:+ message = _message()+ messages.append(message)+ return _queue([message], then_wait=True)++ channel = MagicMock(+ set_qos=AsyncMock(), get_queue=AsyncMock(side_effect=next_queue)+ )+ rabbit = MagicMock(connect=AsyncMock(), get_channel=AsyncMock(return_value=channel))++ with (+ patch.object(delivery.rabbitmq, "AsyncRabbitMQ", return_value=rabbit),+ patch.object(manager, "_process_user_notification", slow),+ patch.object(manager, "_process_ops_notification", slow),+ patch.object(manager, "_process_audience_change", slow),+ patch.object(manager, "_process_pass_work", slow),+ ):+ undecorated = inspect.unwrap(NotificationManager._run_service)+ run = asyncio.create_task(undecorated(manager))+ await asyncio.sleep(0.01)+ run.cancel()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(run, timeout=1)++ assert len(messages) == 4+ for message in messages:+ message.ack.assert_awaited_once()+++# ── permanent delivery failures ────────────────────────────────────────────+++ ("code", "attempts"),+ [(406, 1), (300, 1), (10, delivery.MAX_CONSUMER_RETRY_ATTEMPTS)],+ ids=["inactive-recipient", "invalid-request", "bad-token-is-transient"],+)+async def test_postmark_rejections_that_cannot_succeed_go_straight_to_the_dlq(+ code: int, attempts: int+):+ """Classified at the retry boundary, so every consumer that sends mail+ gets it, not only the user-notification one."""+ manager = _manager()+ handler = AsyncMock(side_effect=ClientError("no", error_code=code))+ message = _message()++ with patch.object(delivery.asyncio, "sleep", AsyncMock()) as sleep:+ await manager._process_message_with_retry(message, handler, "q")++ assert handler.await_count == attempts+ assert sleep.await_count == attempts - 1+ message.reject.assert_awaited_once_with(requeue=False)+ message.ack.assert_not_awaited()+++# ── ordering within a queue ────────────────────────────────────────────────+++def _audience(action: delivery.AudienceAction, email: str) -> MagicMock:+ return _message(+ delivery.AudienceEventModel(+ action=action, email=email, user_id="u"+ ).model_dump_json()+ )+++async def test_audience_changes_for_one_email_keep_their_published_order():+ """Concurrency drops FIFO within a queue, and the audience handlers are+ the ones that are not commutative: add_to_changelog and+ remove_from_changelog for the same address leave the subscriber in+ whichever group won the race. Churn then resubscribe is one Stripe burst+ apart, so the two are published back to back."""+ manager = _manager()+ applied: list[str] = []++ async def remove(_: str) -> None:+ # A removal is a lookup and then a delete; an add is one call.+ await asyncio.sleep(0.02)+ applied.append("remove")++ async def add(_: str) -> None:+ applied.append("add")++ messages = [+ _audience(delivery.AudienceAction.REMOVE_CHANGELOG, "[email protected]"),+ _audience(delivery.AudienceAction.ADD_CHANGELOG, "[email protected]"),+ ]++ with (+ patch.object(delivery.mailerlite, "remove_from_changelog", remove),+ patch.object(delivery.mailerlite, "add_to_changelog", add),+ ):+ await manager._consume_queue(+ _queue(messages),+ manager._process_audience_change,+ delivery.AUDIENCE_QUEUE,+ delivery.Ordering.AS_PUBLISHED,+ )++ assert applied == ["remove", "add"], (+ "the resubscribe must land after the churn removal it was published "+ "after; reordered, the returning customer is left out of the changelog"+ )+++async def test_every_consumer_declares_how_its_queue_is_ordered():+ """The ordering decision lives next to the handler in `_run_service`, so+ a new consumer cannot be added without making it. Pinned here because+ getting it wrong is silent: the suite stays green and the symptom is a+ rare wrong end state."""+ manager = _manager()+ manager.rabbitmq_config = MagicMock()+ declared: dict[str, delivery.Ordering] = {}++ async def consume(+ _queue: Any, _handler: Any, name: str, ordering: delivery.Ordering+ ) -> None:+ declared[name] = ordering+ await asyncio.Event().wait()++ channel = MagicMock(+ set_qos=AsyncMock(), get_queue=AsyncMock(return_value=MagicMock())+ )+ rabbit = MagicMock(connect=AsyncMock(), get_channel=AsyncMock(return_value=channel))++ with (+ patch.object(delivery.rabbitmq, "AsyncRabbitMQ", return_value=rabbit),+ patch.object(manager, "_consume_queue", consume),+ ):+ undecorated = inspect.unwrap(NotificationManager._run_service)+ run = asyncio.create_task(undecorated(manager))+ await asyncio.sleep(0.01)+ run.cancel()+ with pytest.raises(asyncio.CancelledError):+ await run++ assert declared == {+ delivery.USER_NOTIFICATIONS_QUEUE: delivery.Ordering.COMMUTATIVE,+ delivery.OPS_NOTIFICATIONS_QUEUE: delivery.Ordering.COMMUTATIVE,+ delivery.AUDIENCE_QUEUE: delivery.Ordering.AS_PUBLISHED,+ delivery.PASS_WORK_QUEUE: delivery.Ordering.COMMUTATIVE,+ }diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex a6da10b5b3..9b2e5ad5b2 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -11,9 +11,12 @@ import loggingfrom concurrent.futures import Futurefrom concurrent.futures import TimeoutError as FutureTimeoutErrorfrom datetime import date, datetime, timezone+from enum import Enum, autofrom typing import Awaitable, Callable, Coroutineimport aio_pika+from aio_pika.exceptions import ChannelInvalidStateError+from postmarker.exceptions import ClientErrorfrom prisma.enums import NotificationTypefrom backend.data import rabbitmq@@ -68,11 +71,84 @@ def _utc_today() -> date:return datetime.now(tz=timezone.utc).date()+def _is_channel_loss(error: BaseException) -> bool:+ """Every leaf is the queue iterator failing to nack on a dead channel.++ `split` walks nested groups too: the TaskGroup wraps whatever the+ iterator's exit raised, so the nack failures can sit one level down.+ """+ if not isinstance(error, BaseExceptionGroup):+ return False+ _, rest = error.split(ChannelInvalidStateError)+ return rest is None+++def _is_permanent_delivery_failure(error: ClientError) -> bool:+ return error.error_code in PERMANENT_POSTMARK_ERROR_CODES++MAX_CONSUMER_RETRY_ATTEMPTS = 3CONSUMER_RETRY_BACKOFF_SECONDS = 2+# Messages a consumer works on at once, and the broker prefetch to match. The+# work is I/O bound (DatabaseManager RPCs, Postmark); one at a time left the+# other prefetched messages waiting in memory, and a large fan-out from one+# scheduled pass drained at a fraction of the rate the pass published it.+CONSUMER_CONCURRENCY = 10+# Hard ceiling on one message's processing. RabbitMQ closes the channel when a+# delivered message goes unacknowledged for its consumer timeout (30 minutes+# by default), and every other in-flight ack on that channel then fails too;+# a bounded wait turns one hung call into one retried message instead.+MESSAGE_PROCESSING_TIMEOUT_SECONDS = 300+# How long in-flight handlers get to settle before they are cancelled, on+# both of the ways a consumer comes down: a shutdown, and a handler that+# cannot settle its own message. A handler cancelled between its send and its+# ack leaves an email delivered and the message unacked, which the broker then+# redelivers.+#+# The guarantee is narrower than that reads, so state it plainly: a handler+# that finishes inside the grace is acked, and a handler still running at the+# end of it is cancelled and redelivered exactly as it was before. The second+# case is not hypothetical — a briefing pass does DatabaseManager RPCs, then+# renders, then calls Postmark, whose own client timeout is 30s — so this+# narrows the double-send window rather than closing it.+#+# It is not simply raised to cover that: the grace has to fit inside+# SHUTDOWN_TIMEOUT_SECONDS with room for the cancel itself, and that sets+# CLEANUP_TIMEOUT_SECONDS, the whole budget the process gets before its+# supervisor stops waiting and kills it. Covering a 30s send means pushing all+# three past a typical termination grace, trading a rare redelivered email for+# a reliable hard kill mid-send.+HANDLER_SHUTDOWN_GRACE_SECONDS = 5+# Postmark error codes that will never succeed on retry: 300 is a malformed+# request (bad address), 406 an inactive recipient (hard bounce, spam+# complaint or suppression). Retrying only delays the dead-letter by seconds.+PERMANENT_POSTMARK_ERROR_CODES = frozenset({300, 406})SHUTDOWN_TIMEOUT_SECONDS = 10CLEANUP_TIMEOUT_SECONDS = SHUTDOWN_TIMEOUT_SECONDS * 2 + 5++class Ordering(Enum):+ """Whether one queue's messages may be worked out of order.++ Every consumer declares this where it is registered in `_run_service`,+ because that is where a handler is added and the question has to be+ answered: working the prefetch concurrently drops FIFO within the queue,+ which is only safe where two of its messages commute.++ COMMUTATIVE means they do — a second briefing or a second ops mail is its+ own unit of work — and the queue runs at CONSUMER_CONCURRENCY.+ AS_PUBLISHED means they do not, and the queue is worked one message at a+ time however many the broker prefetches.+ """++ COMMUTATIVE = auto()+ AS_PUBLISHED = auto()++ @property+ def concurrency(self) -> int:+ return 1 if self is Ordering.AS_PUBLISHED else CONSUMER_CONCURRENCY++__all__ = ["NotificationManager","NotificationManagerClient",@@ -334,48 +410,147 @@ class NotificationManager(AppService):logger.info(f"[{self.service_name}] Started notification service")channel = await self.rabbit.get_channel()- await channel.set_qos(prefetch_count=10)+ await channel.set_qos(prefetch_count=CONSUMER_CONCURRENCY)+ # Each consumer declares whether its messages commute, here where a+ # handler is added, rather than in a list somewhere else that a new+ # handler is easy to leave out of. See `Ordering`.consumers = {- USER_NOTIFICATIONS_QUEUE: self._process_user_notification,- OPS_NOTIFICATIONS_QUEUE: self._process_ops_notification,- AUDIENCE_QUEUE: self._process_audience_change,- PASS_WORK_QUEUE: self._process_pass_work,+ USER_NOTIFICATIONS_QUEUE: (+ self._process_user_notification,+ Ordering.COMMUTATIVE,+ ),+ OPS_NOTIFICATIONS_QUEUE: (+ self._process_ops_notification,+ Ordering.COMMUTATIVE,+ ),+ # ADD_CHANGELOG and REMOVE_CHANGELOG for one address are a+ # resubscribe and a churn, and MailerLite is left in whichever+ # state finished last. A removal is a lookup then a delete while+ # an add is one call, so a churn-then-resubscribe pair, one Stripe+ # burst apart, reorders under concurrency and leaves a paying+ # customer out of the changelog. Volume here is one message per+ # subscription lifecycle event, so one at a time costs nothing.+ AUDIENCE_QUEUE: (self._process_audience_change, Ordering.AS_PUBLISHED),+ PASS_WORK_QUEUE: (self._process_pass_work, Ordering.COMMUTATIVE),}tasks = [asyncio.create_task(- self._consume_queue(await channel.get_queue(name), handler, name)+ self._consume_queue(+ await channel.get_queue(name), handler, name, ordering+ ))- for name, handler in consumers.items()+ for name, (handler, ordering) in consumers.items()]try:await asyncio.gather(*tasks)except asyncio.CancelledError:logger.info("Service shutdown requested")+ raise+ finally:+ # The four consumers share one channel, so they live and die+ # together. When one fails and `continuous_retry` runs this again,+ # the other three must not be left consuming beside their+ # replacements: every message would then be handled twice.for task in tasks:task.cancel()await asyncio.gather(*tasks, return_exceptions=True)- raiseasync def _consume_queue(self,queue: aio_pika.abc.AbstractQueue,process_func: Callable[[str], Awaitable[bool]],queue_name: str,+ ordering: Ordering,):+ """Work up to this queue's concurrency in messages at once.++ The prefetch already delivered that many; handling them one after+ another only kept the rest waiting in memory. `ordering` is the+ caller's declaration that the queue's messages commute; an+ AS_PUBLISHED queue stays at one message at a time. It has no default+ on purpose, so a new consumer cannot be registered without answering+ the question.+ """logger.info(f"Starting consumer for queue: {queue_name}")+ slots = asyncio.Semaphore(ordering.concurrency)+ in_flight: set[asyncio.Task[None]] = set()+ draining = asyncio.Event()++ async def handle(message: aio_pika.abc.AbstractIncomingMessage) -> None:+ try:+ await self._process_message_with_retry(+ message, process_func, queue_name+ )+ except Exception:+ # Failing here is how the consumer is brought down, but a+ # TaskGroup aborts by cancelling every sibling first and only+ # then the task running the loop, so the grace below would+ # never get a look in: the other messages the prefetch handed+ # this consumer would be cancelled between their send and+ # their ack, and redelivered as second emails. Give them the+ # same bounded grace a shutdown gives, here, before the+ # exception leaves this task and the group aborts. The grace+ # covers only the handlers already running, so the loop must+ # stop starting new ones in the slots they free.+ draining.set()+ siblings = in_flight - {asyncio.current_task()}+ if siblings:+ await asyncio.wait(siblings, timeout=HANDLER_SHUTDOWN_GRACE_SECONDS)+ raise+ finally:+ slots.release()++ # The TaskGroup is the supervisor: a handler that fails to settle its+ # message (an ack or reject failing for a reason other than a lost+ # channel, which `_settle` absorbs) cancels the loop and surfaces+ # here, so the consumer reconnects instead of sitting alive with+ # unacked deliveries slowly pinning the prefetch. Both ways out --+ # that failure and a shutdown -- reach the handlers as a cancellation,+ # and both give them a bounded grace to settle first.try:- async with queue.iterator() as queue_iter:- async for message in queue_iter:- if not self.running:- break- await self._process_message_with_retry(- message, process_func, queue_name- )+ async with asyncio.TaskGroup() as group:+ try:+ async with queue.iterator() as queue_iter:+ async for message in queue_iter:+ if not self.running:+ break+ await slots.acquire()+ if draining.is_set():+ slots.release()+ await self._requeue(message, queue_name)+ # Park until the failing handler's grace is+ # up and the abort cancels this task. Leaving+ # the iterator now would cancel the consumer+ # and nack its buffer mid-grace, and anything+ # that raised there would abort the grace.+ await asyncio.Event().wait()+ task = group.create_task(handle(message))+ in_flight.add(task)+ task.add_done_callback(in_flight.discard)+ except asyncio.CancelledError:+ # Shutdown. Let in-flight handlers settle before the+ # TaskGroup cancels them, or a send that has already+ # gone out is redelivered because its ack never went.+ if in_flight:+ await asyncio.wait(+ in_flight, timeout=HANDLER_SHUTDOWN_GRACE_SECONDS+ )+ raiseexcept asyncio.CancelledError:logger.info(f"Consumer for {queue_name} cancelled")raise+ except ExceptionGroup as failures:+ # The iterator's exit nacks whatever it still buffered; when the+ # channel has been swapped out under it (a robust reconnect), every+ # one of those fails the same way. The broker requeues unacked+ # deliveries on its own, so this is a reconnect, not a data loss.+ if not _is_channel_loss(failures):+ logger.exception(f"Fatal error in consumer for {queue_name}")+ raise+ logger.warning(f"Consumer for {queue_name} lost its channel; reconnecting")+ raise ConnectionError(f"RabbitMQ channel lost for {queue_name}") from Noneexcept Exception as e:logger.exception(f"Fatal error in consumer for {queue_name}: {e}")raise@@ -391,43 +566,120 @@ class NotificationManager(AppService):``process_func`` MUST be idempotent: the same body is replayed on eachattempt, so a partial success (Postmark accepted the email but a later- write failed) re-runs on retry.+ write failed) re-runs on retry. The processing timeout is a second,+ quieter way to get one: it cancels the coroutine, but a Postmark send+ runs in a worker thread via ``asyncio.to_thread`` and cancelling the+ await does not stop the thread, so a send that is merely slow can+ still land and then be retried. It takes a stall ten times Postmark's+ own 30s client timeout to reach that, which is why the bound is worth+ having anyway."""+ # Only the handler runs inside the retried block. Settling happens in+ # the `else` and after the loop, so an ack or reject that fails for a+ # reason `_settle` does not absorb propagates to the consumer instead+ # of being mistaken for a handler failure and re-running the work.last_error: Exception | None = Nonefor attempt in range(MAX_CONSUMER_RETRY_ATTEMPTS):try:body = message.body.decode()- if await process_func(body):- await message.ack()- return- logger.warning(- f"Message in {queue_name} rejected (process_func returned False)"+ processed = await asyncio.wait_for(+ process_func(body), timeout=MESSAGE_PROCESSING_TIMEOUT_SECONDS)- await message.reject(requeue=False)- returnexcept UnicodeDecodeError as e:logger.warning(f"Undecodable message in {queue_name}, sending to DLQ: {e}")- await message.reject(requeue=False)+ await self._settle(message, "reject", queue_name)return+ except asyncio.TimeoutError:+ last_error = TimeoutError(+ f"processing exceeded {MESSAGE_PROCESSING_TIMEOUT_SECONDS}s"+ )+ except ClientError as e:+ if not _is_permanent_delivery_failure(e):+ last_error = e+ else:+ # A suppressed or malformed address does not become+ # deliverable by waiting six seconds; retrying only puts+ # the dead-letter off and logs three warnings for one.+ logger.warning(+ f"Permanent delivery failure in {queue_name} "+ f"(Postmark {e.error_code}), sending to DLQ"+ )+ await self._settle(message, "reject", queue_name)+ returnexcept Exception as e:last_error = e- if attempt == MAX_CONSUMER_RETRY_ATTEMPTS - 1:- break- delay = CONSUMER_RETRY_BACKOFF_SECONDS * (2**attempt)+ else:+ if processed:+ await self._settle(message, "ack", queue_name)+ returnlogger.warning(- f"Transient failure on attempt {attempt + 1}/"- f"{MAX_CONSUMER_RETRY_ATTEMPTS} in {queue_name}: {e}. "- f"Retrying in {delay}s.",+ f"Message in {queue_name} rejected (process_func returned False)")- await asyncio.sleep(delay)+ await self._settle(message, "reject", queue_name)+ return+ if attempt == MAX_CONSUMER_RETRY_ATTEMPTS - 1:+ break+ delay = CONSUMER_RETRY_BACKOFF_SECONDS * (2**attempt)+ logger.warning(+ f"Transient failure on attempt {attempt + 1}/"+ f"{MAX_CONSUMER_RETRY_ATTEMPTS} in {queue_name}: {last_error}. "+ f"Retrying in {delay}s.",+ )+ await asyncio.sleep(delay)logger.exception(f"Sending message to DLQ from {queue_name} after "f"{MAX_CONSUMER_RETRY_ATTEMPTS} attempts. Last error: {last_error}",exc_info=last_error,)- await message.reject(requeue=False)+ await self._settle(message, "reject", queue_name)++ async def _settle(+ self,+ message: aio_pika.abc.AbstractIncomingMessage,+ action: str,+ queue_name: str,+ ) -> None:+ """Ack or dead-letter a message, tolerating a channel that has gone.++ The work is already done by the time this runs. If the channel was+ swapped out underneath (a robust reconnect) the broker has requeued+ the delivery itself; re-running the handler to "retry" the ack would+ be a second send for anything not claimed, and the redelivery is what+ actually settles it. So a lost channel is logged and dropped here,+ never retried.+ """+ try:+ if action == "ack":+ await message.ack()+ else:+ await message.reject(requeue=False)+ except ChannelInvalidStateError:+ logger.warning(+ f"Could not {action} a message in {queue_name}: channel lost, "+ "the broker will redeliver it"+ )++ async def _requeue(+ self,+ message: aio_pika.abc.AbstractIncomingMessage,+ queue_name: str,+ ) -> None:+ """Hand back a delivery the consumer pulled but will not work.++ Only reached while the consumer is coming down on another handler's+ failure, so a nack that fails is logged, not raised: raising would+ abort the grace the other handlers are settling in, and an unsettled+ delivery is redelivered once its channel goes anyway.+ """+ try:+ await message.nack(requeue=True)+ except Exception as e:+ logger.warning(+ f"Could not requeue a message in {queue_name}, the broker will "+ f"redeliver it when the channel closes: {e}"+ )async def _shutdown_service(self) -> None:"""Stop consumers completely before closing their RabbitMQ connection."""diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex 9ec4004964..89ecf9f192 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -206,7 +206,11 @@ filterwarnings = [][tool.ruff]-target-version = "py310"+# The floor the code needs, not what CI or the image run: ExceptionGroup and+# TaskGroup are 3.11 builtins and py310 flagged them as undefined names. Kept+# at the minimum so ruff still reports 3.12+ syntax, which would break the+# 3.11 leg of the CI matrix.+target-version = "py311"extend-exclude = ["test/fixtures"]# `poetry run format` runs `ruff check --fix .`, which would otherwise rewrite# the vendored fixture it walks into.