AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): retry-with-backoff in notification consumer before DLQ reject (#13049)
Zamil Majdye838383b1f26
diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex d0c476f75a..6fbbe13e8a 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -47,6 +47,14 @@ from backend.util.settings import AppEnvironment, Settingslogger = TruncatedLogger(logging.getLogger(__name__), "[NotificationManager]")settings = Settings()+# Retry settings for transient failures in queue consumers. With these defaults+# the consumer makes 3 attempts with 2s + 4s = 6s of backoff between them+# before reject-to-DLQ — enough to ride out the most common transient failures+# (DB connection blip, Postmark 5xx, brief network outage) without trapping+# truly broken messages on the queue for long.+MAX_CONSUMER_RETRY_ATTEMPTS = 3+CONSUMER_RETRY_BACKOFF_SECONDS = 2+NOTIFICATION_EXCHANGE = Exchange(name="notifications", type=ExchangeType.TOPIC)DEAD_LETTER_EXCHANGE = Exchange(name="dead_letter", type=ExchangeType.TOPIC)@@ -605,154 +613,208 @@ class NotificationManager(AppService):return Noneasync def _process_admin_message(self, message: str) -> bool:- """Process a single notification, sending to an admin, returning whether to put into the failed queue"""- try:- event = self._parse_message(message)- if not event:- return False- logger.debug(f"Processing notification for admin: {event}")- recipient_email = settings.config.refund_notification_email- await self.email_sender.send_templated(event.type, recipient_email, event)- return True- except Exception as e:- logger.exception(f"Error processing notification for admin queue: {e}")+ """Process an admin notification.++ Returns False for permanent failures (e.g. unparseable message) — the+ consumer treats False as "do not retry, send to DLQ". Transient+ failures (DB blip, Postmark 5xx, network timeout) propagate as+ exceptions so the consumer's retry-with-backoff loop can recover.+ """+ event = self._parse_message(message)+ if not event:return False+ logger.debug(f"Processing notification for admin: {event}")+ recipient_email = settings.config.refund_notification_email+ await self.email_sender.send_templated(event.type, recipient_email, event)+ return Trueasync def _process_immediate(self, message: str) -> bool:- """Process a single notification immediately, returning whether to put into the failed queue"""- try:- event = self._parse_message(message)- if not event:- return False- logger.debug(f"Processing immediate notification: {event}")+ """Process an immediate notification.- recipient_email = await get_database_manager_async_client(- should_retry=False- ).get_user_email_by_id(event.user_id)- if not recipient_email:- logger.warning(f"User email not found for user {event.user_id}")- return False-- should_send = await self._should_email_user_based_on_preference(- event.user_id, event.type- )- if not should_send:- logger.debug(- f"User {event.user_id} does not want to receive {event.type} notifications"- )- return True+ See `_process_admin_message` for the False vs. raise contract.+ """+ event = self._parse_message(message)+ if not event:+ return False+ logger.debug(f"Processing immediate notification: {event}")- unsub_link = generate_unsubscribe_link(event.user_id)+ recipient_email = await get_database_manager_async_client(+ should_retry=False+ ).get_user_email_by_id(event.user_id)+ if not recipient_email:+ logger.warning(f"User email not found for user {event.user_id}")+ return False- await self.email_sender.send_templated(- notification=event.type,- user_email=recipient_email,- data=event,- user_unsub_link=unsub_link,+ should_send = await self._should_email_user_based_on_preference(+ event.user_id, event.type+ )+ if not should_send:+ logger.debug(+ f"User {event.user_id} does not want to receive {event.type} notifications")return True- except Exception as e:- logger.exception(f"Error processing notification for immediate queue: {e}")- return False++ unsub_link = generate_unsubscribe_link(event.user_id)++ await self.email_sender.send_templated(+ notification=event.type,+ user_email=recipient_email,+ data=event,+ user_unsub_link=unsub_link,+ )+ return Trueasync def _process_batch(self, message: str) -> bool:- """Process a single notification with a batching strategy, returning whether to put into the failed queue"""- try:- event = self._parse_message(message)- if not event:- return False- logger.info(f"Processing batch notification: {event}")+ """Process a batched notification.++ See `_process_admin_message` for the False vs. raise contract. The+ inner per-chunk try/except below intentionally absorbs Postmark API+ errors (406/422/oversized) so one bad notification doesn't kill the+ rest of the batch — those are permanent failures, not transient.+ Anything outside that inner block (DB calls, template load) is left+ to bubble up so the consumer's retry loop can recover.+ """+ event = self._parse_message(message)+ if not event:+ return False+ logger.info(f"Processing batch notification: {event}")- recipient_email = await get_database_manager_async_client(- should_retry=False- ).get_user_email_by_id(event.user_id)- if not recipient_email:- logger.warning(f"User email not found for user {event.user_id}")- return False+ recipient_email = await get_database_manager_async_client(+ should_retry=False+ ).get_user_email_by_id(event.user_id)+ if not recipient_email:+ logger.warning(f"User email not found for user {event.user_id}")+ return False- should_send = await self._should_email_user_based_on_preference(- event.user_id, event.type+ should_send = await self._should_email_user_based_on_preference(+ event.user_id, event.type+ )+ if not should_send:+ logger.info(+ f"User {event.user_id} does not want to receive {event.type} notifications")- if not should_send:- logger.info(- f"User {event.user_id} does not want to receive {event.type} notifications"- )- return True+ return True- should_send = await self._should_batch(event.user_id, event.type, event)+ should_send = await self._should_batch(event.user_id, event.type, event)- if not should_send:- logger.info("Batch not old enough to send")- return False- batch = await get_database_manager_async_client(- should_retry=False- ).get_user_notification_batch(event.user_id, event.type)- if not batch or not batch.notifications:- logger.warning(f"Batch not found for user {event.user_id}")- return False- unsub_link = generate_unsubscribe_link(event.user_id)-- batch_messages = [- NotificationEventModel[- get_notif_data_type(db_event.type)- ].model_validate(- {- "id": db_event.id, # Include ID from database- "user_id": event.user_id,- "type": db_event.type,- "data": db_event.data,- "created_at": db_event.created_at,- }- )- for db_event in batch.notifications- ]-- # Split batch into chunks to avoid exceeding email size limits- # Start with a reasonable chunk size and adjust dynamically- MAX_EMAIL_SIZE = 4_500_000 # 4.5MB to leave buffer under 5MB limit- chunk_size = 100 # Initial chunk size- successfully_sent_count = 0- failed_indices = []-- i = 0- while i < len(batch_messages):- # Try progressively smaller chunks if needed- chunk_sent = False- for attempt_size in [chunk_size, 50, 25, 10, 5, 1]:- chunk = batch_messages[i : i + attempt_size]- chunk_ids = [- msg.id for msg in chunk if msg.id- ] # Extract IDs for removal-- try:- # Try to render the email to check its size- template = self.email_sender._get_template(event.type)- (- _,- test_message,- ) = await self.email_sender.formatter.format_email(- base_template=template.base_template,- subject_template=template.subject_template,- content_template=template.body_template,- data={"notifications": chunk},- unsubscribe_link=f"{self.email_sender.formatter.env.globals.get('base_url', '')}/profile/settings",+ if not should_send:+ # The notification has already been persisted into the batch in+ # _should_batch. Ack the queue message — the next message for+ # this user/type, or the periodic flush, will deliver the batch+ # once it matures. Returning False here would DLQ the trigger.+ logger.info("Batch not old enough to send (notification persisted)")+ return True+ batch = await get_database_manager_async_client(+ should_retry=False+ ).get_user_notification_batch(event.user_id, event.type)+ if not batch or not batch.notifications:+ logger.warning(f"Batch not found for user {event.user_id}")+ return False+ unsub_link = generate_unsubscribe_link(event.user_id)++ batch_messages = [+ NotificationEventModel[get_notif_data_type(db_event.type)].model_validate(+ {+ "id": db_event.id, # Include ID from database+ "user_id": event.user_id,+ "type": db_event.type,+ "data": db_event.data,+ "created_at": db_event.created_at,+ }+ )+ for db_event in batch.notifications+ ]++ # Split batch into chunks to avoid exceeding email size limits+ # Start with a reasonable chunk size and adjust dynamically+ MAX_EMAIL_SIZE = 4_500_000 # 4.5MB to leave buffer under 5MB limit+ chunk_size = 100 # Initial chunk size+ successfully_sent_count = 0+ failed_indices = []++ i = 0+ while i < len(batch_messages):+ # Try progressively smaller chunks if needed+ chunk_sent = False+ for attempt_size in [chunk_size, 50, 25, 10, 5, 1]:+ chunk = batch_messages[i : i + attempt_size]+ chunk_ids = [+ msg.id for msg in chunk if msg.id+ ] # Extract IDs for removal++ try:+ # Try to render the email to check its size+ template = self.email_sender._get_template(event.type)+ (+ _,+ test_message,+ ) = await self.email_sender.formatter.format_email(+ base_template=template.base_template,+ subject_template=template.subject_template,+ content_template=template.body_template,+ data={"notifications": chunk},+ unsubscribe_link=f"{self.email_sender.formatter.env.globals.get('base_url', '')}/profile/settings",+ )++ if len(test_message) < MAX_EMAIL_SIZE:+ # Size is acceptable, send the email+ logger.info(+ f"Sending email with {len(chunk)} notifications "+ f"(size: {len(test_message):,} chars)")- if len(test_message) < MAX_EMAIL_SIZE:- # Size is acceptable, send the email- logger.info(- f"Sending email with {len(chunk)} notifications "- f"(size: {len(test_message):,} chars)"- )+ await self.email_sender.send_templated(+ notification=event.type,+ user_email=recipient_email,+ data=chunk,+ user_unsub_link=unsub_link,+ )- await self.email_sender.send_templated(- notification=event.type,- user_email=recipient_email,- data=chunk,- user_unsub_link=unsub_link,+ # Remove successfully sent notifications immediately+ if chunk_ids:+ try:+ await get_database_manager_async_client(+ should_retry=False+ ).remove_notifications_from_batch(+ event.user_id, event.type, chunk_ids+ )+ logger.info(+ f"Removed {len(chunk_ids)} sent notifications from batch"+ )+ except Exception as e:+ logger.warning(+ f"Failed to remove sent notifications: {e}"+ )+ # Continue anyway - better to risk duplicates than lose emails++ # Track successful sends+ successfully_sent_count += len(chunk)++ # Update chunk_size for next iteration based on success+ if (+ attempt_size == chunk_size+ and len(test_message) < MAX_EMAIL_SIZE * 0.7+ ):+ # If we're well under limit, try larger chunks next time+ chunk_size = min(chunk_size + 10, 100)+ elif len(test_message) > MAX_EMAIL_SIZE * 0.9:+ # If we're close to limit, use smaller chunks+ chunk_size = max(attempt_size - 10, 1)++ i += len(chunk)+ chunk_sent = True+ break+ else:+ # Message is too large even after size reduction+ if attempt_size == 1:+ logger.warning(+ f"Failed to send notification at index {i}: "+ f"Single notification exceeds email size limit "+ f"({len(test_message):,} chars > {MAX_EMAIL_SIZE:,} chars). "+ f"Removing permanently from batch - will not retry.")- # Remove successfully sent notifications immediately+ # Remove the oversized notification permanently - it will NEVER fitif chunk_ids:try:await get_database_manager_async_client(@@ -761,245 +823,191 @@ class NotificationManager(AppService):event.user_id, event.type, chunk_ids)logger.info(- f"Removed {len(chunk_ids)} sent notifications from batch"+ f"Removed oversized notification {chunk_ids[0]} from batch permanently")except Exception as e:logger.warning(- f"Failed to remove sent notifications: {e}"+ f"Failed to remove oversized notification: {e}")- # Continue anyway - better to risk duplicates than lose emails-- # Track successful sends- successfully_sent_count += len(chunk)-- # Update chunk_size for next iteration based on success- if (- attempt_size == chunk_size- and len(test_message) < MAX_EMAIL_SIZE * 0.7- ):- # If we're well under limit, try larger chunks next time- chunk_size = min(chunk_size + 10, 100)- elif len(test_message) > MAX_EMAIL_SIZE * 0.9:- # If we're close to limit, use smaller chunks- chunk_size = max(attempt_size - 10, 1)-- i += len(chunk)++ failed_indices.append(i)+ i += 1chunk_sent = Truebreak- else:- # Message is too large even after size reduction- if attempt_size == 1:+ # Try smaller chunk size+ continue+ except Exception as e:+ # Check if it's a Postmark API error+ if attempt_size == 1:+ # Single notification failed - determine the actual cause+ error_message = str(e).lower()+ error_type = type(e).__name__++ # Check for HTTP 406 - Inactive recipient (common in Postmark errors)+ if "406" in error_message or "inactive" in error_message:+ logger.warning(+ f"Failed to send notification at index {i}: "+ f"Recipient marked as inactive by Postmark. "+ f"Error: {e}. Disabling ALL notifications for this user."+ )++ # 1. Mark email as unverified+ try:+ await set_user_email_verification(event.user_id, False)+ logger.info(+ f"Set email verification to false for user {event.user_id}"+ )+ except Exception as deactivation_error:logger.warning(- f"Failed to send notification at index {i}: "- f"Single notification exceeds email size limit "- f"({len(test_message):,} chars > {MAX_EMAIL_SIZE:,} chars). "- f"Removing permanently from batch - will not retry."+ f"Failed to deactivate email for user {event.user_id}: "+ f"{deactivation_error}")- # Remove the oversized notification permanently - it will NEVER fit- if chunk_ids:- try:- await get_database_manager_async_client(- should_retry=False- ).remove_notifications_from_batch(- event.user_id, event.type, chunk_ids- )- logger.info(- f"Removed oversized notification {chunk_ids[0]} from batch permanently"- )- except Exception as e:- logger.warning(- f"Failed to remove oversized notification: {e}"- )-- failed_indices.append(i)- i += 1- chunk_sent = True- break- # Try smaller chunk size- continue- except Exception as e:- # Check if it's a Postmark API error- if attempt_size == 1:- # Single notification failed - determine the actual cause- error_message = str(e).lower()- error_type = type(e).__name__-- # Check for HTTP 406 - Inactive recipient (common in Postmark errors)- if "406" in error_message or "inactive" in error_message:+ # 2. Disable all notification preferences+ try:+ await disable_all_user_notifications(event.user_id)+ logger.info(+ f"Disabled all notification preferences for user {event.user_id}"+ )+ except Exception as disable_error:logger.warning(- f"Failed to send notification at index {i}: "- f"Recipient marked as inactive by Postmark. "- f"Error: {e}. Disabling ALL notifications for this user."+ f"Failed to disable notification preferences: {disable_error}")- # 1. Mark email as unverified- try:- await set_user_email_verification(- event.user_id, False- )- logger.info(- f"Set email verification to false for user {event.user_id}"- )- except Exception as deactivation_error:- logger.warning(- f"Failed to deactivate email for user {event.user_id}: "- f"{deactivation_error}"- )+ # 3. Clear ALL notification batches for this user+ try:+ await get_database_manager_async_client(+ should_retry=False+ ).clear_all_user_notification_batches(event.user_id)+ logger.info(+ f"Cleared ALL notification batches for user {event.user_id}"+ )+ except Exception as remove_error:+ logger.warning(+ f"Failed to clear batches for inactive recipient: {remove_error}"+ )- # 2. Disable all notification preferences- try:- await disable_all_user_notifications(event.user_id)- logger.info(- f"Disabled all notification preferences for user {event.user_id}"- )- except Exception as disable_error:- logger.warning(- f"Failed to disable notification preferences: {disable_error}"- )+ # Stop processing - we've nuked everything for this user+ return True+ # Check for HTTP 422 - Malformed data+ elif "422" in error_message or "unprocessable" in error_message:+ logger.warning(+ f"Failed to send notification at index {i}: "+ f"Malformed notification data rejected by Postmark. "+ f"Error: {e}. Removing from batch permanently."+ )- # 3. Clear ALL notification batches for this user+ # Remove from batch - 422 means bad data that won't fix itself+ if chunk_ids:try:await get_database_manager_async_client(should_retry=False- ).clear_all_user_notification_batches(event.user_id)+ ).remove_notifications_from_batch(+ event.user_id, event.type, chunk_ids+ )logger.info(- f"Cleared ALL notification batches for user {event.user_id}"+ "Removed malformed notification from batch permanently")except Exception as remove_error:logger.warning(- f"Failed to clear batches for inactive recipient: {remove_error}"+ f"Failed to remove malformed notification: {remove_error}")+ # Check if it's a ValueError for size limit+ elif isinstance(e, ValueError) and "too large" in error_message:+ logger.warning(+ f"Failed to send notification at index {i}: "+ f"Notification size exceeds email limit. "+ f"Error: {e}. Skipping this notification."+ )+ # Other API errors+ else:+ logger.warning(+ f"Failed to send notification at index {i}: "+ f"Email API error ({error_type}): {e}. "+ f"Skipping this notification."+ )- # Stop processing - we've nuked everything for this user- return True- # Check for HTTP 422 - Malformed data- elif (- "422" in error_message- or "unprocessable" in error_message- ):- logger.warning(- f"Failed to send notification at index {i}: "- f"Malformed notification data rejected by Postmark. "- f"Error: {e}. Removing from batch permanently."- )-- # Remove from batch - 422 means bad data that won't fix itself- if chunk_ids:- try:- await get_database_manager_async_client(- should_retry=False- ).remove_notifications_from_batch(- event.user_id, event.type, chunk_ids- )- logger.info(- "Removed malformed notification from batch permanently"- )- except Exception as remove_error:- logger.warning(- f"Failed to remove malformed notification: {remove_error}"- )- # Check if it's a ValueError for size limit- elif (- isinstance(e, ValueError)- and "too large" in error_message- ):- logger.warning(- f"Failed to send notification at index {i}: "- f"Notification size exceeds email limit. "- f"Error: {e}. Skipping this notification."- )- # Other API errors- else:- logger.warning(- f"Failed to send notification at index {i}: "- f"Email API error ({error_type}): {e}. "- f"Skipping this notification."- )-- failed_indices.append(i)- i += 1- chunk_sent = True- break- # Try smaller chunk- continue+ failed_indices.append(i)+ i += 1+ chunk_sent = True+ break+ # Try smaller chunk+ continue- if not chunk_sent:- # Should not reach here due to single notification handling- logger.warning(- f"Failed to send notifications starting at index {i}"- )- failed_indices.append(i)- i += 1+ if not chunk_sent:+ # Should not reach here due to single notification handling+ logger.warning(f"Failed to send notifications starting at index {i}")+ failed_indices.append(i)+ i += 1- # Check what remains in the batch (notifications are removed as sent)- remaining_batch = await get_database_manager_async_client(- should_retry=False- ).get_user_notification_batch(event.user_id, event.type)+ # Check what remains in the batch (notifications are removed as sent)+ remaining_batch = await get_database_manager_async_client(+ should_retry=False+ ).get_user_notification_batch(event.user_id, event.type)- if not remaining_batch or not remaining_batch.notifications:- logger.info(- f"All {successfully_sent_count} notifications sent and removed from batch"- )- else:- remaining_count = len(remaining_batch.notifications)- logger.warning(- f"Sent {successfully_sent_count} notifications. "- f"{remaining_count} remain in batch for retry due to errors."- )- return True- except Exception as e:- logger.exception(f"Error processing notification for batch queue: {e}")- return False+ if not remaining_batch or not remaining_batch.notifications:+ logger.info(+ f"All {successfully_sent_count} notifications sent and removed from batch"+ )+ else:+ remaining_count = len(remaining_batch.notifications)+ logger.warning(+ f"Sent {successfully_sent_count} notifications. "+ f"{remaining_count} remain in batch for retry due to errors."+ )+ return Trueasync def _process_summary(self, message: str) -> bool:- """Process a single notification with a summary strategy, returning whether to put into the failed queue"""+ """Process a summary notification.++ See `_process_admin_message` for the False vs. raise contract.+ """+ logger.info(f"Processing summary notification: {message}")try:- logger.info(f"Processing summary notification: {message}")event = BaseEventModel.model_validate_json(message)model = SummaryParamsEventModel[get_summary_params_type(event.type)].model_validate_json(message)+ except ValueError as e:+ logger.warning(f"Unparseable summary notification (sending to DLQ): {e}")+ return False- logger.info(f"Processing summary notification: {model}")+ logger.info(f"Processing summary notification: {model}")- recipient_email = await get_database_manager_async_client(- should_retry=False- ).get_user_email_by_id(event.user_id)- if not recipient_email:- logger.warning(f"User email not found for user {event.user_id}")- return False- should_send = await self._should_email_user_based_on_preference(- event.user_id, event.type+ recipient_email = await get_database_manager_async_client(+ should_retry=False+ ).get_user_email_by_id(event.user_id)+ if not recipient_email:+ logger.warning(f"User email not found for user {event.user_id}")+ return False+ should_send = await self._should_email_user_based_on_preference(+ event.user_id, event.type+ )+ if not should_send:+ logger.info(+ f"User {event.user_id} does not want to receive {event.type} notifications")- if not should_send:- logger.info(- f"User {event.user_id} does not want to receive {event.type} notifications"- )- return True+ return True- summary_data = await self._gather_summary_data(- event.user_id, event.type, model.data- )+ summary_data = await self._gather_summary_data(+ event.user_id, event.type, model.data+ )- unsub_link = generate_unsubscribe_link(event.user_id)+ unsub_link = generate_unsubscribe_link(event.user_id)- data = NotificationEventModel(- user_id=event.user_id,- type=event.type,- data=summary_data,- )+ data = NotificationEventModel(+ user_id=event.user_id,+ type=event.type,+ data=summary_data,+ )- await self.email_sender.send_templated(- notification=event.type,- user_email=recipient_email,- data=data,- user_unsub_link=unsub_link,- )- return True- except Exception as e:- logger.exception(f"Error processing notification for summary queue: {e}")- return False+ await self.email_sender.send_templated(+ notification=event.type,+ user_email=recipient_email,+ data=data,+ user_unsub_link=unsub_link,+ )+ return Trueasync def _consume_queue(self,@@ -1007,7 +1015,13 @@ class NotificationManager(AppService):process_func: Callable[[str], Awaitable[bool]],queue_name: str,):- """Continuously consume messages from a queue using async iteration"""+ """Continuously consume messages from a queue using async iteration.++ On transient failures (raised exceptions), retry in-process with+ exponential backoff up to MAX_CONSUMER_RETRY_ATTEMPTS times before+ rejecting to the DLQ. process_func returning False means+ "unrecoverable, do not retry" — reject immediately.+ """logger.info(f"Starting consumer for queue: {queue_name}")try:@@ -1015,25 +1029,9 @@ class NotificationManager(AppService):async for message in queue_iter:if not self.running:break-- try:- async with message.process():- result = await process_func(message.body.decode())- if not result:- # Message will be rejected when exiting context without exception- raise aio_pika.exceptions.MessageProcessError(- "Processing failed"- )- except aio_pika.exceptions.MessageProcessError:- # Let message.process() handle the rejection- pass- except Exception as e:- logger.warning(- f"Error processing message in {queue_name}: {e}",- exc_info=True,- )- # Let message.process() handle the rejection- raise+ await self._process_message_with_retry(+ message, process_func, queue_name+ )except asyncio.CancelledError:logger.info(f"Consumer for {queue_name} cancelled")raise@@ -1041,6 +1039,67 @@ class NotificationManager(AppService):logger.exception(f"Fatal error in consumer for {queue_name}: {e}")raise+ async def _process_message_with_retry(+ self,+ message: aio_pika.abc.AbstractIncomingMessage,+ process_func: Callable[[str], Awaitable[bool]],+ queue_name: str,+ ):+ """Run process_func against one message with retry-with-backoff.++ Acks on success, rejects (no requeue → DLQ) on permanent failure or+ after exhausting retry attempts.++ ``process_func`` MUST be idempotent: the same message body is replayed+ on each attempt, so a partial-success on one attempt (e.g. Postmark+ accepted the email but the database commit failed afterwards) will+ re-run on retry. The notification consumers tolerate this because+ their permanent side-effect (Postmark send) is the last step before+ ack, but anyone wiring a new ``process_func`` here must keep the+ contract or risk duplicate sends.+ """+ last_error: Exception | None = None+ for attempt in range(MAX_CONSUMER_RETRY_ATTEMPTS):+ try:+ # Decoding inside the try so a UnicodeDecodeError on a+ # malformed body is treated as a permanent failure and goes+ # to the DLQ instead of crashing the consumer task.+ body = message.body.decode()+ if await process_func(body):+ await message.ack()+ return+ # process_func returned False = explicit "do not retry"+ logger.warning(+ f"Message in {queue_name} rejected (process_func returned False)"+ )+ await message.reject(requeue=False)+ return+ except UnicodeDecodeError as e:+ logger.warning(+ f"Undecodable message in {queue_name}, sending to DLQ: {e}"+ )+ await message.reject(requeue=False)+ return+ except Exception as e:+ last_error = e+ is_last_attempt = attempt == MAX_CONSUMER_RETRY_ATTEMPTS - 1+ if is_last_attempt:+ break+ delay = CONSUMER_RETRY_BACKOFF_SECONDS * (2**attempt)+ logger.warning(+ f"Transient failure on attempt {attempt + 1}/"+ f"{MAX_CONSUMER_RETRY_ATTEMPTS} in {queue_name}: {e}. "+ f"Retrying in {delay}s.",+ )+ await asyncio.sleep(delay)+ # All retries exhausted — log full context and send to DLQ.+ logger.exception(+ f"Sending message to DLQ from {queue_name} after "+ f"{MAX_CONSUMER_RETRY_ATTEMPTS} attempts. Last error: {last_error}",+ exc_info=last_error,+ )+ await message.reject(requeue=False)+def run_service(self):# Queue the main _run_service taskasyncio.run_coroutine_threadsafe(self._run_service(), self.shared_event_loop)diff --git a/autogpt_platform/backend/backend/notifications/test_notifications.py b/autogpt_platform/backend/backend/notifications/test_notifications.pyindex 4934257400..a9dd733f34 100644--- a/autogpt_platform/backend/backend/notifications/test_notifications.py+++ b/autogpt_platform/backend/backend/notifications/test_notifications.py@@ -324,11 +324,7 @@ class TestNotificationErrorHandling:data = kwargs.get("data", {}).get("notifications", [])if data and len(data) == 1:# Check notification content to identify index 3- if any(- "Test Agent 3" in str(n.data)- for n in data- if hasattr(n, "data")- ):+ if any("Test Agent 3" in str(n.data) for n in data):# Return oversized message for index 3return ("subject", "x" * 5_000_000) # Over 4.5MB limitreturn ("subject", "normal sized content")@@ -345,11 +341,7 @@ class TestNotificationErrorHandling:if isinstance(data, list) and len(data) == 1:# Track which notification was sent based on contentfor i, notif in enumerate(notifications):- if any(- f"Test Agent {i}" in str(n.data)- for n in data- if hasattr(n, "data")- ):+ if any(f"Test Agent {i}" in str(n.data) for n in data):successful_indices.append(i)return Nonereturn None@@ -448,11 +440,7 @@ class TestNotificationErrorHandling:if isinstance(data, list) and len(data) == 1:# Track which notification based on contentfor i, notif in enumerate(notifications):- if any(- f"Test Agent {i}" in str(n.data)- for n in data- if hasattr(n, "data")- ):+ if any(f"Test Agent {i}" in str(n.data) for n in data):# Index 1 has generic API errorif i == 1:failed_indices.append(i)@@ -559,11 +547,7 @@ class TestNotificationErrorHandling:if isinstance(data, list) and len(data) == 1:# Track which notification was sentfor i, notif in enumerate(notifications):- if any(- f"Test Agent {i}" in str(n.data)- for n in data- if hasattr(n, "data")- ):+ if any(f"Test Agent {i}" in str(n.data) for n in data):successful_indices.append(i)return Nonereturn None # Success@@ -598,3 +582,229 @@ class TestNotificationErrorHandling:# Info message about successful sends should be loggedinfo_calls = [call[0][0] for call in mock_logger.info.call_args_list]assert any("sent and removed" in call.lower() for call in info_calls)+++class TestConsumerRetryWithBackoff:+ """Verify _process_message_with_retry retries transient failures and only DLQs after exhausting attempts."""++ @pytest.fixture+ def manager(self):+ # Build a NotificationManager without invoking __init__ side effects+ # (RabbitMQ connections, threads). We only exercise the+ # _process_message_with_retry method.+ m = NotificationManager.__new__(NotificationManager)+ return m++ def _make_message(self, body: bytes = b"{}"):+ msg = MagicMock()+ msg.body = body+ msg.ack = AsyncMock()+ msg.reject = AsyncMock()+ return msg++ @pytest.mark.asyncio+ async def test_acks_on_success(self, manager):+ msg = self._make_message()+ process = AsyncMock(return_value=True)+ await manager._process_message_with_retry(msg, process, "test_q")+ assert process.call_count == 1+ msg.ack.assert_awaited_once()+ msg.reject.assert_not_called()++ @pytest.mark.asyncio+ async def test_rejects_when_process_returns_false_no_retry(self, manager):+ msg = self._make_message()+ process = AsyncMock(return_value=False)+ await manager._process_message_with_retry(msg, process, "test_q")+ # Explicit False = unrecoverable, do not retry+ assert process.call_count == 1+ msg.reject.assert_awaited_once_with(requeue=False)+ msg.ack.assert_not_called()++ @pytest.mark.asyncio+ async def test_retries_transient_then_succeeds(self, manager):+ msg = self._make_message()+ # Fail twice, succeed on 3rd attempt+ process = AsyncMock(+ side_effect=[RuntimeError("blip"), RuntimeError("blip"), True]+ )+ with patch(+ "backend.notifications.notifications.asyncio.sleep", new=AsyncMock()+ ) as sleep:+ await manager._process_message_with_retry(msg, process, "test_q")+ assert process.call_count == 3+ assert sleep.await_count == 2 # backoff sleeps before attempts 2 and 3+ msg.ack.assert_awaited_once()+ msg.reject.assert_not_called()++ @pytest.mark.asyncio+ async def test_dlqs_after_exhausting_retries(self, manager):+ msg = self._make_message()+ process = AsyncMock(side_effect=RuntimeError("permanent boom"))+ with patch(+ "backend.notifications.notifications.asyncio.sleep", new=AsyncMock()+ ):+ await manager._process_message_with_retry(msg, process, "test_q")+ # Defaults: 3 attempts total+ assert process.call_count == 3+ msg.reject.assert_awaited_once_with(requeue=False)+ msg.ack.assert_not_called()++ @pytest.mark.asyncio+ async def test_backoff_doubles_per_attempt(self, manager):+ msg = self._make_message()+ process = AsyncMock(side_effect=RuntimeError("boom"))+ sleeps = []++ async def fake_sleep(s):+ sleeps.append(s)++ with patch(+ "backend.notifications.notifications.asyncio.sleep", side_effect=fake_sleep+ ):+ await manager._process_message_with_retry(msg, process, "test_q")+ # CONSUMER_RETRY_BACKOFF_SECONDS = 2, so delays are 2*(2**0)=2, 2*(2**1)=4+ assert sleeps == [2, 4]++ @pytest.mark.asyncio+ async def test_inner_process_raises_so_retry_triggers(self, manager):+ """Regression: the four `_process_*` methods must let transient+ exceptions propagate so `_process_message_with_retry` can retry them.+ If they swallow exceptions and return False instead, the retry loop+ treats it as a permanent failure and DLQs on the first attempt.+ """+ # Stand up just enough state for _process_immediate to reach the+ # email_sender call (mocked) and raise.+ manager.email_sender = MagicMock()+ manager.email_sender.send_templated = AsyncMock(+ side_effect=RuntimeError("postmark 502")+ )+ manager._should_email_user_based_on_preference = AsyncMock(return_value=True)++ with patch(+ "backend.notifications.notifications.get_database_manager_async_client"+ ) as mock_db_client, patch(+ "backend.notifications.notifications.generate_unsubscribe_link",+ return_value="unsub",+ ):+ mock_db = MagicMock()+ mock_db.get_user_email_by_id = AsyncMock(return_value="[email protected]")+ mock_db_client.return_value = mock_db+ event = MagicMock()+ event.user_id = "u1"+ event.type = MagicMock()+ # _process_immediate uses self._parse_message+ manager._parse_message = MagicMock(return_value=event)++ with pytest.raises(RuntimeError, match="postmark 502"):+ await manager._process_immediate("{}")++ @pytest.mark.asyncio+ async def test_admin_message_returns_true_on_send(self, manager):+ manager.email_sender = MagicMock()+ manager.email_sender.send_templated = AsyncMock()+ event = MagicMock()+ event.type = MagicMock()+ manager._parse_message = MagicMock(return_value=event)+ result = await manager._process_admin_message("{}")+ assert result is True+ manager.email_sender.send_templated.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_admin_message_returns_false_on_unparseable(self, manager):+ manager._parse_message = MagicMock(return_value=None)+ result = await manager._process_admin_message("garbage")+ assert result is False++ @pytest.mark.asyncio+ async def test_admin_message_propagates_send_failure(self, manager):+ manager.email_sender = MagicMock()+ manager.email_sender.send_templated = AsyncMock(+ side_effect=RuntimeError("postmark 502")+ )+ event = MagicMock()+ event.type = MagicMock()+ manager._parse_message = MagicMock(return_value=event)+ with pytest.raises(RuntimeError, match="postmark 502"):+ await manager._process_admin_message("{}")++ @pytest.mark.asyncio+ async def test_immediate_returns_false_on_unparseable(self, manager):+ manager._parse_message = MagicMock(return_value=None)+ result = await manager._process_immediate("garbage")+ assert result is False++ @pytest.mark.asyncio+ async def test_immediate_returns_false_when_no_email(self, manager):+ event = MagicMock()+ event.user_id = "u1"+ event.type = MagicMock()+ manager._parse_message = MagicMock(return_value=event)+ with patch(+ "backend.notifications.notifications.get_database_manager_async_client"+ ) as mock_db_client:+ mock_db = MagicMock()+ mock_db.get_user_email_by_id = AsyncMock(return_value=None)+ mock_db_client.return_value = mock_db+ result = await manager._process_immediate("{}")+ assert result is False++ @pytest.mark.asyncio+ async def test_immediate_returns_true_when_user_opted_out(self, manager):+ event = MagicMock()+ event.user_id = "u1"+ event.type = MagicMock()+ manager._parse_message = MagicMock(return_value=event)+ manager._should_email_user_based_on_preference = AsyncMock(return_value=False)+ with patch(+ "backend.notifications.notifications.get_database_manager_async_client"+ ) as mock_db_client:+ mock_db = MagicMock()+ mock_db.get_user_email_by_id = AsyncMock(return_value="[email protected]")+ mock_db_client.return_value = mock_db+ result = await manager._process_immediate("{}")+ # Opted-out = "delivered" from queue's POV, no retry needed+ assert result is True++ @pytest.mark.asyncio+ async def test_undecodable_body_dlqs_without_retry(self, manager):+ """Decode failure must reject-to-DLQ, not crash the consumer task."""+ msg = MagicMock()+ # Lone continuation byte — not valid UTF-8.+ msg.body = b"\xff\xfe\x00"+ msg.ack = AsyncMock()+ msg.reject = AsyncMock()+ process = AsyncMock()+ await manager._process_message_with_retry(msg, process, "test_q")+ process.assert_not_called()+ msg.ack.assert_not_called()+ msg.reject.assert_awaited_once_with(requeue=False)++ @pytest.mark.asyncio+ async def test_summary_unparseable_returns_false_not_raise(self, manager):+ """_process_summary must return False (DLQ) on parse failure,+ not raise a ValidationError that triggers retry-with-backoff.+ """+ result = await manager._process_summary("{not json")+ assert result is False++ @pytest.mark.asyncio+ async def test_batch_not_old_enough_acks(self, manager):+ """When the batch isn't ready to flush, the trigger message must be+ acked — the notification is already persisted. Returning False would+ DLQ the trigger and silently drop a future delivery.+ """+ event = MagicMock()+ event.user_id = "u1"+ event.type = MagicMock()+ manager._parse_message = MagicMock(return_value=event)+ manager._should_email_user_based_on_preference = AsyncMock(return_value=True)+ manager._should_batch = AsyncMock(return_value=False)+ with patch(+ "backend.notifications.notifications.get_database_manager_async_client"+ ) as mock_db_client:+ mock_db = MagicMock()+ mock_db.get_user_email_by_id = AsyncMock(return_value="[email protected]")+ mock_db_client.return_value = mock_db+ result = await manager._process_batch("{}")+ assert result is True