AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): migrate notification service to fully async to resolve RabbitMQ connection issues (#10564)
Zamil Majdye2af2f454dd8
diff --git a/autogpt_platform/backend/backend/blocks/google/calendar.py b/autogpt_platform/backend/backend/blocks/google/calendar.pyindex c75e0ec9eb..339daab430 100644--- a/autogpt_platform/backend/backend/blocks/google/calendar.py+++ b/autogpt_platform/backend/backend/blocks/google/calendar.py@@ -21,6 +21,8 @@ from ._auth import (GoogleCredentialsInput,)+settings = Settings()+class CalendarEvent(BaseModel):"""Structured representation of a Google Calendar event."""@@ -221,8 +223,8 @@ class GoogleCalendarReadEventsBlock(Block):else None),token_uri="https://oauth2.googleapis.com/token",- client_id=Settings().secrets.google_client_id,- client_secret=Settings().secrets.google_client_secret,+ client_id=settings.secrets.google_client_id,+ client_secret=settings.secrets.google_client_secret,scopes=credentials.scopes,)return build("calendar", "v3", credentials=creds)@@ -569,8 +571,8 @@ class GoogleCalendarCreateEventBlock(Block):else None),token_uri="https://oauth2.googleapis.com/token",- client_id=Settings().secrets.google_client_id,- client_secret=Settings().secrets.google_client_secret,+ client_id=settings.secrets.google_client_id,+ client_secret=settings.secrets.google_client_secret,scopes=credentials.scopes,)return build("calendar", "v3", credentials=creds)diff --git a/autogpt_platform/backend/backend/blocks/google/gmail.py b/autogpt_platform/backend/backend/blocks/google/gmail.pyindex faa34a6015..eaa384d232 100644--- a/autogpt_platform/backend/backend/blocks/google/gmail.py+++ b/autogpt_platform/backend/backend/blocks/google/gmail.py@@ -21,6 +21,8 @@ from ._auth import (GoogleCredentialsInput,)+settings = Settings()+def serialize_email_recipients(recipients: list[str]) -> str:"""Serialize recipients list to comma-separated string."""@@ -255,8 +257,8 @@ class GmailReadBlock(Block):else None),token_uri="https://oauth2.googleapis.com/token",- client_id=Settings().secrets.google_client_id,- client_secret=Settings().secrets.google_client_secret,+ client_id=settings.secrets.google_client_id,+ client_secret=settings.secrets.google_client_secret,scopes=credentials.scopes,)return build("gmail", "v1", credentials=creds)diff --git a/autogpt_platform/backend/backend/blocks/slant3d/order.py b/autogpt_platform/backend/backend/blocks/slant3d/order.pyindex f1cab18d27..43a5802468 100644--- a/autogpt_platform/backend/backend/blocks/slant3d/order.py+++ b/autogpt_platform/backend/backend/blocks/slant3d/order.py@@ -3,8 +3,7 @@ from typing import Listfrom backend.data.block import BlockOutput, BlockSchemafrom backend.data.model import APIKeyCredentials, SchemaField-from backend.util import settings-from backend.util.settings import BehaveAs+from backend.util.settings import BehaveAs, Settingsfrom ._api import (TEST_CREDENTIALS,@@ -16,6 +15,8 @@ from ._api import ()from .base import Slant3DBlockBase+settings = Settings()+class Slant3DCreateOrderBlock(Slant3DBlockBase):"""Block for creating new orders"""@@ -280,7 +281,7 @@ class Slant3DGetOrdersBlock(Slant3DBlockBase):input_schema=self.Input,output_schema=self.Output,# This block is disabled for cloud hosted because it allows access to all orders for the account- disabled=settings.Settings().config.behave_as == BehaveAs.CLOUD,+ disabled=settings.config.behave_as == BehaveAs.CLOUD,test_input={"credentials": TEST_CREDENTIALS_INPUT},test_credentials=TEST_CREDENTIALS,test_output=[diff --git a/autogpt_platform/backend/backend/blocks/slant3d/webhook.py b/autogpt_platform/backend/backend/blocks/slant3d/webhook.pyindex 8a690cf1ad..22f87b468d 100644--- a/autogpt_platform/backend/backend/blocks/slant3d/webhook.py+++ b/autogpt_platform/backend/backend/blocks/slant3d/webhook.py@@ -9,8 +9,7 @@ from backend.data.block import ()from backend.data.model import SchemaFieldfrom backend.integrations.providers import ProviderName-from backend.util import settings-from backend.util.settings import AppEnvironment, BehaveAs+from backend.util.settings import AppEnvironment, BehaveAs, Settingsfrom ._api import (TEST_CREDENTIALS,@@ -19,6 +18,8 @@ from ._api import (Slant3DCredentialsInput,)+settings = Settings()+class Slant3DTriggerBase:"""Base class for Slant3D webhook triggers"""@@ -76,8 +77,8 @@ class Slant3DOrderWebhookBlock(Slant3DTriggerBase, Block):),# All webhooks are currently subscribed to for all orders. This works for self hosted, but not for cloud hosted proddisabled=(- settings.Settings().config.behave_as == BehaveAs.CLOUD- and settings.Settings().config.app_env != AppEnvironment.LOCAL+ settings.config.behave_as == BehaveAs.CLOUD+ and settings.config.app_env != AppEnvironment.LOCAL),categories={BlockCategory.DEVELOPER_TOOLS},input_schema=self.Input,diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.pyindex 83a62fc73b..8b57a1d6bd 100644--- a/autogpt_platform/backend/backend/data/user.py+++ b/autogpt_platform/backend/backend/data/user.py@@ -21,6 +21,7 @@ from backend.util.json import SafeJsonfrom backend.util.settings import Settingslogger = logging.getLogger(__name__)+settings = Settings()async def get_or_create_user(user_data: dict) -> User:@@ -332,7 +333,7 @@ async def get_user_email_verification(user_id: str) -> bool:def generate_unsubscribe_link(user_id: str) -> str:"""Generate a link to unsubscribe from all notifications"""# Create an HMAC using a secret key- secret_key = Settings().secrets.unsubscribe_secret_key+ secret_key = settings.secrets.unsubscribe_secret_keysignature = hmac.new(secret_key.encode("utf-8"), user_id.encode("utf-8"), hashlib.sha256).digest()@@ -343,7 +344,7 @@ def generate_unsubscribe_link(user_id: str) -> str:).decode("utf-8")logger.info(f"Generating unsubscribe link for user {user_id}")- base_url = Settings().config.platform_base_url+ base_url = settings.config.platform_base_urlreturn f"{base_url}/api/email/unsubscribe?token={quote_plus(token)}"@@ -355,7 +356,7 @@ async def unsubscribe_user_by_token(token: str) -> None:user_id, received_signature_hex = decoded.split(":", 1)# Verify the signature- secret_key = Settings().secrets.unsubscribe_secret_key+ secret_key = settings.secrets.unsubscribe_secret_keyexpected_signature = hmac.new(secret_key.encode("utf-8"), user_id.encode("utf-8"), hashlib.sha256).digest()diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex bdf59bc809..00b0dbbd83 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -163,23 +163,6 @@ class DatabaseManagerClient(AppServiceClient):spend_credits = _(d.spend_credits)get_credits = _(d.get_credits)- # User Comms - async- get_active_user_ids_in_timerange = _(d.get_active_user_ids_in_timerange)- get_user_email_by_id = _(d.get_user_email_by_id)- get_user_email_verification = _(d.get_user_email_verification)- get_user_notification_preference = _(d.get_user_notification_preference)-- # Notifications - async- create_or_add_to_user_notification_batch = _(- d.create_or_add_to_user_notification_batch- )- empty_user_notification_batch = _(d.empty_user_notification_batch)- get_all_batches_by_type = _(d.get_all_batches_by_type)- get_user_notification_batch = _(d.get_user_notification_batch)- get_user_notification_oldest_message_in_batch = _(- d.get_user_notification_oldest_message_in_batch- )-# Block error monitoringget_block_error_stats = _(d.get_block_error_stats)@@ -209,3 +192,20 @@ class DatabaseManagerAsyncClient(AppServiceClient):update_user_integrations = d.update_user_integrationsget_execution_kv_data = d.get_execution_kv_dataset_execution_kv_data = d.set_execution_kv_data++ # User Comms+ get_active_user_ids_in_timerange = d.get_active_user_ids_in_timerange+ get_user_email_by_id = d.get_user_email_by_id+ get_user_email_verification = d.get_user_email_verification+ get_user_notification_preference = d.get_user_notification_preference++ # Notifications+ create_or_add_to_user_notification_batch = (+ d.create_or_add_to_user_notification_batch+ )+ empty_user_notification_batch = d.empty_user_notification_batch+ get_all_batches_by_type = d.get_all_batches_by_type+ get_user_notification_batch = d.get_user_notification_batch+ get_user_notification_oldest_message_in_batch = (+ d.get_user_notification_oldest_message_in_batch+ )diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex e0c44b4d70..cdb4d33a46 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -1,11 +1,10 @@import asyncioimport logging-import multiprocessingimport osimport threadingimport timefrom collections import defaultdict-from concurrent.futures import Future, ProcessPoolExecutor+from concurrent.futures import Future, ThreadPoolExecutorfrom contextlib import asynccontextmanagerfrom typing import TYPE_CHECKING, Any, Optional, TypeVar, cast@@ -102,6 +101,22 @@ utilization_gauge = Gauge("Ratio of active graph runs to max graph workers",)+# Thread-local storage for ExecutionProcessor instances+_tls = threading.local()+++def init_worker():+ """Initialize ExecutionProcessor instance in thread-local storage"""+ _tls.processor = ExecutionProcessor()+ _tls.processor.on_graph_executor_start()+++def execute_graph(+ graph_exec_entry: "GraphExecutionEntry", cancel_event: threading.Event+):+ """Execute graph using thread-local ExecutionProcessor instance"""+ return _tls.processor.on_graph_execution(graph_exec_entry, cancel_event)+T = TypeVar("T")@@ -366,7 +381,7 @@ async def _enqueue_next_nodes(]-class Executor:+class ExecutionProcessor:"""This class contains event handlers for the process pool executor events.@@ -389,10 +404,9 @@ class Executor:9. Node executor enqueues the next executed nodes to the node execution queue."""- @classmethod@async_error_logged(swallow=True)async def on_node_execution(- cls,+ self,node_exec: NodeExecutionEntry,node_exec_progress: NodeExecutionProgress,nodes_input_masks: Optional[dict[str, dict[str, JsonValue]]],@@ -411,7 +425,7 @@ class Executor:node = await db_client.get_node(node_exec.node_id)execution_stats = NodeExecutionStats()- timing_info, status = await cls._on_node_execution(+ timing_info, status = await self._on_node_execution(node=node,node_exec=node_exec,node_exec_progress=node_exec_progress,@@ -454,10 +468,9 @@ class Executor:return execution_stats- @classmethod@async_time_measuredasync def _on_node_execution(- cls,+ self,node: Node,node_exec: NodeExecutionEntry,node_exec_progress: NodeExecutionProgress,@@ -497,7 +510,7 @@ class Executor:async for output_name, output_data in execute_node(node=node,- creds_manager=cls.creds_manager,+ creds_manager=self.creds_manager,data=node_exec,execution_stats=stats,nodes_input_masks=nodes_input_masks,@@ -537,29 +550,27 @@ class Executor:return status- @classmethod@func_retry- def on_graph_executor_start(cls):+ def on_graph_executor_start(self):configure_logging()set_service_name("GraphExecutor")- cls.pid = os.getpid()- cls.creds_manager = IntegrationCredentialsManager()- cls.node_execution_loop = asyncio.new_event_loop()- cls.node_evaluation_loop = asyncio.new_event_loop()- cls.node_execution_thread = threading.Thread(- target=cls.node_execution_loop.run_forever, daemon=True+ self.tid = threading.get_ident()+ self.creds_manager = IntegrationCredentialsManager()+ self.node_execution_loop = asyncio.new_event_loop()+ self.node_evaluation_loop = asyncio.new_event_loop()+ self.node_execution_thread = threading.Thread(+ target=self.node_execution_loop.run_forever, daemon=True)- cls.node_evaluation_thread = threading.Thread(- target=cls.node_evaluation_loop.run_forever, daemon=True+ self.node_evaluation_thread = threading.Thread(+ target=self.node_evaluation_loop.run_forever, daemon=True)- cls.node_execution_thread.start()- cls.node_evaluation_thread.start()- logger.info(f"[GraphExecutor] {cls.pid} started")+ self.node_execution_thread.start()+ self.node_evaluation_thread.start()+ logger.info(f"[GraphExecutor] {self.tid} started")- @classmethod@error_logged(swallow=False)def on_graph_execution(- cls,+ self,graph_exec: GraphExecutionEntry,cancel: threading.Event,):@@ -615,7 +626,7 @@ class Executor:else:exec_stats = exec_meta.stats.to_db()- timing_info, status = cls._on_graph_execution(+ timing_info, status = self._on_graph_execution(graph_exec=graph_exec,cancel=cancel,log_metadata=log_metadata,@@ -641,7 +652,7 @@ class Executor:user_id=graph_exec.user_id,execution_status=status,),- cls.node_execution_loop,+ self.node_execution_loop,).result(timeout=60.0)if activity_status is not None:exec_stats.activity_status = activity_status@@ -652,7 +663,7 @@ class Executor:)# Communication handling- cls._handle_agent_run_notif(db_client, graph_exec, exec_stats)+ self._handle_agent_run_notif(db_client, graph_exec, exec_stats)finally:update_graph_execution_state(@@ -662,9 +673,8 @@ class Executor:stats=exec_stats,)- @classmethoddef _charge_usage(- cls,+ self,node_exec: NodeExecutionEntry,execution_count: int,) -> int:@@ -714,22 +724,14 @@ class Executor:return total_cost- @classmethod@time_measureddef _on_graph_execution(- cls,+ self,graph_exec: GraphExecutionEntry,cancel: threading.Event,log_metadata: LogMetadata,execution_stats: GraphExecutionStats,) -> ExecutionStatus:-- # Agent execution is uninterrupted.- import signal-- signal.signal(signal.SIGINT, signal.SIG_IGN)- signal.signal(signal.SIGTERM, signal.SIG_IGN)-"""Returns:dict: The execution statistics of the graph execution.@@ -786,7 +788,7 @@ class Executor:# Charge usage (may raise) ------------------------------try:- cost = cls._charge_usage(+ cost = self._charge_usage(node_exec=queued_node_exec,execution_count=increment_execution_count(graph_exec.user_id),)@@ -806,7 +808,7 @@ class Executor:status=ExecutionStatus.FAILED,)- cls._handle_low_balance_notif(+ self._handle_low_balance_notif(db_client,graph_exec.user_id,graph_exec.graph_id,@@ -825,7 +827,7 @@ class Executor:# Kick off async node execution -------------------------node_execution_task = asyncio.run_coroutine_threadsafe(- cls.on_node_execution(+ self.on_node_execution(node_exec=queued_node_exec,node_exec_progress=running_node_execution[node_id],nodes_input_masks=nodes_input_masks,@@ -834,7 +836,7 @@ class Executor:execution_stats_lock,),),- cls.node_execution_loop,+ self.node_execution_loop,)running_node_execution[node_id].add_task(node_exec_id=queued_node_exec.node_exec_id,@@ -875,7 +877,7 @@ class Executor:node_output_found = Truerunning_node_evaluation[node_id] = (asyncio.run_coroutine_threadsafe(- cls._process_node_output(+ self._process_node_output(output=output,node_id=node_id,graph_exec=graph_exec,@@ -883,7 +885,7 @@ class Executor:nodes_input_masks=nodes_input_masks,execution_queue=execution_queue,),- cls.node_evaluation_loop,+ self.node_evaluation_loop,))if (@@ -926,7 +928,7 @@ class Executor:raisefinally:- cls._cleanup_graph_execution(+ self._cleanup_graph_execution(execution_queue=execution_queue,running_node_execution=running_node_execution,running_node_evaluation=running_node_evaluation,@@ -937,10 +939,9 @@ class Executor:db_client=db_client,)- @classmethod@error_logged(swallow=True)def _cleanup_graph_execution(- cls,+ self,execution_queue: ExecutionQueue[NodeExecutionEntry],running_node_execution: dict[str, "NodeExecutionProgress"],running_node_evaluation: dict[str, Future],@@ -991,10 +992,9 @@ class Executor:clean_exec_files(graph_exec_id)- @classmethod@async_error_logged(swallow=True)async def _process_node_output(- cls,+ self,output: ExecutionOutputEntry,node_id: str,graph_exec: GraphExecutionEntry,@@ -1027,9 +1027,8 @@ class Executor:):execution_queue.add(next_execution)- @classmethoddef _handle_agent_run_notif(- cls,+ self,db_client: "DatabaseManagerClient",graph_exec: GraphExecutionEntry,exec_stats: GraphExecutionStats,@@ -1065,9 +1064,8 @@ class Executor:))- @classmethoddef _handle_low_balance_notif(- cls,+ self,db_client: "DatabaseManagerClient",user_id: str,graph_id: str,@@ -1132,11 +1130,11 @@ class ExecutionManager(AppProcess):return self._stop_consuming@property- def executor(self) -> ProcessPoolExecutor:+ def executor(self) -> ThreadPoolExecutor:if self._executor is None:- self._executor = ProcessPoolExecutor(+ self._executor = ThreadPoolExecutor(max_workers=self.pool_size,- initializer=Executor.on_graph_executor_start,+ initializer=init_worker,)return self._executor@@ -1313,11 +1311,9 @@ class ExecutionManager(AppProcess):_ack_message(reject=True)return- cancel_event = multiprocessing.Manager().Event()+ cancel_event = threading.Event()- future = self.executor.submit(- Executor.on_graph_execution, graph_exec_entry, cancel_event- )+ future = self.executor.submit(execute_graph, graph_exec_entry, cancel_event)self.active_graph_runs[graph_exec_id] = (future, cancel_event)self._update_prompt_metrics()diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex cde3e2144c..a67e27973c 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -1,6 +1,7 @@import asyncioimport loggingimport os+import threadingfrom enum import Enumfrom typing import Optionalfrom urllib.parse import parse_qs, urlencode, urlparse, urlunparse@@ -11,7 +12,6 @@ from apscheduler.jobstores.memory import MemoryJobStorefrom apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStorefrom apscheduler.schedulers.blocking import BlockingSchedulerfrom apscheduler.triggers.cron import CronTrigger-from autogpt_libs.utils.cache import thread_cachedfrom dotenv import load_dotenvfrom pydantic import BaseModel, Field, ValidationErrorfrom sqlalchemy import MetaData, create_engine@@ -30,6 +30,7 @@ from backend.monitoring import (from backend.util.cloud_storage import cleanup_expired_files_asyncfrom backend.util.exceptions import NotAuthorizedError, NotFoundErrorfrom backend.util.logging import PrefixFilter+from backend.util.retry import func_retryfrom backend.util.service import AppService, AppServiceClient, endpoint_to_async, exposefrom backend.util.settings import Config@@ -69,13 +70,23 @@ def job_listener(event):logger.info(f"Job {event.job_id} completed successfully.")-@thread_cached+_event_loop: asyncio.AbstractEventLoop | None = None+++@func_retrydef get_event_loop():- return asyncio.new_event_loop()+ """Get the shared event loop."""+ if _event_loop is None:+ raise RuntimeError("Event loop not initialized. Scheduler not started.")+ return _event_loopdef execute_graph(**kwargs):- get_event_loop().run_until_complete(_execute_graph(**kwargs))+ """Execute graph in the shared event loop and wait for completion."""+ loop = get_event_loop()+ future = asyncio.run_coroutine_threadsafe(_execute_graph(**kwargs), loop)+ # Wait for completion to ensure job doesn't exit prematurely+ future.result(timeout=300) # 5 minute timeout for graph executionasync def _execute_graph(**kwargs):@@ -99,7 +110,10 @@ async def _execute_graph(**kwargs):def cleanup_expired_files():"""Clean up expired files from cloud storage."""- get_event_loop().run_until_complete(cleanup_expired_files_async())+ loop = get_event_loop()+ future = asyncio.run_coroutine_threadsafe(cleanup_expired_files_async(), loop)+ # Wait for completion+ future.result(timeout=300) # 5 minute timeout for cleanup# Monitoring functions are now imported from monitoring module@@ -175,6 +189,17 @@ class Scheduler(AppService):def run_service(self):load_dotenv()++ # Initialize the event loop for async jobs+ global _event_loop+ _event_loop = asyncio.new_event_loop()++ # Use daemon thread since it should die with the main service+ event_loop_thread = threading.Thread(+ target=_event_loop.run_forever, daemon=True, name="SchedulerEventLoop"+ )+ event_loop_thread.start()+db_schema, db_url = _extract_schema_from_url(os.getenv("DIRECT_URL"))self.scheduler = BlockingScheduler(jobstores={diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex 67327d2985..052fc453fd 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -1,8 +1,7 @@import asyncioimport logging-from concurrent.futures import ProcessPoolExecutorfrom datetime import datetime, timedelta, timezone-from typing import Callable+from typing import Awaitable, Callableimport aio_pikafrom prisma.enums import NotificationType@@ -28,7 +27,7 @@ from backend.data.notifications import (from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfigfrom backend.data.user import generate_unsubscribe_linkfrom backend.notifications.email import EmailSender-from backend.util.clients import get_database_manager_client+from backend.util.clients import get_database_manager_async_clientfrom backend.util.logging import TruncatedLoggerfrom backend.util.metrics import discord_send_alertfrom backend.util.retry import continuous_retry@@ -43,8 +42,6 @@ NOTIFICATION_EXCHANGE = Exchange(name="notifications", type=ExchangeType.TOPIC)DEAD_LETTER_EXCHANGE = Exchange(name="dead_letter", type=ExchangeType.TOPIC)EXCHANGES = [NOTIFICATION_EXCHANGE, DEAD_LETTER_EXCHANGE]-background_executor = ProcessPoolExecutor(max_workers=2)-def create_notification_config() -> RabbitMQConfig:"""Create RabbitMQ configuration for notifications"""@@ -202,7 +199,8 @@ class NotificationManager(AppService):@exposedef queue_weekly_summary(self):- background_executor.submit(lambda: asyncio.run(self._queue_weekly_summary()))+ # Use the existing event loop instead of creating a new one with asyncio.run()+ asyncio.create_task(self._queue_weekly_summary())async def _queue_weekly_summary(self):"""Process weekly summary for specified notification types"""@@ -211,7 +209,7 @@ class NotificationManager(AppService):processed_count = 0current_time = datetime.now(tz=timezone.utc)start_time = current_time - timedelta(days=7)- users = get_database_manager_client().get_active_user_ids_in_timerange(+ users = await get_database_manager_async_client().get_active_user_ids_in_timerange(end_time=current_time.isoformat(),start_time=start_time.isoformat(),)@@ -235,9 +233,12 @@ class NotificationManager(AppService):@exposedef process_existing_batches(self, notification_types: list[NotificationType]):- background_executor.submit(self._process_existing_batches, notification_types)+ # Use the existing event loop instead of creating a new process+ asyncio.create_task(self._process_existing_batches(notification_types))- def _process_existing_batches(self, notification_types: list[NotificationType]):+ async def _process_existing_batches(+ self, notification_types: list[NotificationType]+ ):"""Process existing batches for specified notification types"""try:processed_count = 0@@ -245,13 +246,15 @@ class NotificationManager(AppService):for notification_type in notification_types:# Get all batches for this notification type- batches = get_database_manager_client().get_all_batches_by_type(- notification_type+ batches = (+ await get_database_manager_async_client().get_all_batches_by_type(+ notification_type+ ))for batch in batches:# Check if batch has aged out- oldest_message = get_database_manager_client().get_user_notification_oldest_message_in_batch(+ oldest_message = await get_database_manager_async_client().get_user_notification_oldest_message_in_batch(batch.user_id, notification_type)@@ -266,10 +269,8 @@ class NotificationManager(AppService):# If batch has aged out, process itif oldest_message.created_at + max_delay < current_time:- recipient_email = (- get_database_manager_client().get_user_email_by_id(- batch.user_id- )+ recipient_email = await get_database_manager_async_client().get_user_email_by_id(+ batch.user_id)if not recipient_email:@@ -278,7 +279,7 @@ class NotificationManager(AppService):)continue- should_send = self._should_email_user_based_on_preference(+ should_send = await self._should_email_user_based_on_preference(batch.user_id, notification_type)@@ -287,15 +288,13 @@ class NotificationManager(AppService):f"User {batch.user_id} does not want to receive {notification_type} notifications")# Clear the batch- get_database_manager_client().empty_user_notification_batch(+ await get_database_manager_async_client().empty_user_notification_batch(batch.user_id, notification_type)continue- batch_data = (- get_database_manager_client().get_user_notification_batch(- batch.user_id, notification_type- )+ batch_data = await get_database_manager_async_client().get_user_notification_batch(+ batch.user_id, notification_type)if not batch_data or not batch_data.notifications:@@ -303,7 +302,7 @@ class NotificationManager(AppService):f"Batch data not found for user {batch.user_id}")# Clear the batch- get_database_manager_client().empty_user_notification_batch(+ await get_database_manager_async_client().empty_user_notification_batch(batch.user_id, notification_type)continue@@ -339,7 +338,7 @@ class NotificationManager(AppService):)# Clear the batch- get_database_manager_client().empty_user_notification_batch(+ await get_database_manager_async_client().empty_user_notification_batch(batch.user_id, notification_type)@@ -384,18 +383,20 @@ class NotificationManager(AppService):except Exception as e:logger.exception(f"Error queueing notification: {e}")- def _should_email_user_based_on_preference(+ async def _should_email_user_based_on_preference(self, user_id: str, event_type: NotificationType) -> bool:"""Check if a user wants to receive a notification based on their preferences and email verification status"""- validated_email = get_database_manager_client().get_user_email_verification(- user_id+ validated_email = (+ await get_database_manager_async_client().get_user_email_verification(+ user_id+ ))preference = (- get_database_manager_client()- .get_user_notification_preference(user_id)- .preferences.get(event_type, True)- )+ await get_database_manager_async_client().get_user_notification_preference(+ user_id+ )+ ).preferences.get(event_type, True)# only if both are true, should we email this personreturn validated_email and preference@@ -479,18 +480,16 @@ class NotificationManager(AppService):else:raise ValueError("Invalid event type or params")- def _should_batch(+ async def _should_batch(self, user_id: str, event_type: NotificationType, event: NotificationEventModel) -> bool:- get_database_manager_client().create_or_add_to_user_notification_batch(+ await get_database_manager_async_client().create_or_add_to_user_notification_batch(user_id, event_type, event)- oldest_message = (- get_database_manager_client().get_user_notification_oldest_message_in_batch(- user_id, event_type- )+ oldest_message = await get_database_manager_async_client().get_user_notification_oldest_message_in_batch(+ user_id, event_type)if not oldest_message:logger.error(@@ -519,7 +518,7 @@ class NotificationManager(AppService):logger.error(f"Error parsing message due to non matching schema {e}")return None- def _process_admin_message(self, message: str) -> bool:+ async def _process_admin_message(self, message: str) -> bool:"""Process a single notification, sending to an admin, returning whether to put into the failed queue"""try:event = self._parse_message(message)@@ -533,7 +532,7 @@ class NotificationManager(AppService):logger.exception(f"Error processing notification for admin queue: {e}")return False- def _process_immediate(self, message: str) -> bool:+ async def _process_immediate(self, message: str) -> bool:"""Process a single notification immediately, returning whether to put into the failed queue"""try:event = self._parse_message(message)@@ -541,14 +540,16 @@ class NotificationManager(AppService):return Falselogger.debug(f"Processing immediate notification: {event}")- recipient_email = get_database_manager_client().get_user_email_by_id(- event.user_id+ recipient_email = (+ await get_database_manager_async_client().get_user_email_by_id(+ event.user_id+ ))if not recipient_email:logger.error(f"User email not found for user {event.user_id}")return False- should_send = self._should_email_user_based_on_preference(+ should_send = await self._should_email_user_based_on_preference(event.user_id, event.type)if not should_send:@@ -570,7 +571,7 @@ class NotificationManager(AppService):logger.exception(f"Error processing notification for immediate queue: {e}")return False- def _process_batch(self, message: str) -> bool:+ async def _process_batch(self, message: str) -> bool:"""Process a single notification with a batching strategy, returning whether to put into the failed queue"""try:event = self._parse_message(message)@@ -578,14 +579,16 @@ class NotificationManager(AppService):return Falselogger.info(f"Processing batch notification: {event}")- recipient_email = get_database_manager_client().get_user_email_by_id(- event.user_id+ recipient_email = (+ await get_database_manager_async_client().get_user_email_by_id(+ event.user_id+ ))if not recipient_email:logger.error(f"User email not found for user {event.user_id}")return False- should_send = self._should_email_user_based_on_preference(+ should_send = await self._should_email_user_based_on_preference(event.user_id, event.type)if not should_send:@@ -594,13 +597,15 @@ class NotificationManager(AppService):)return True- should_send = self._should_batch(event.user_id, event.type, event)+ should_send = await self._should_batch(event.user_id, event.type, event)if not should_send:logger.info("Batch not old enough to send")return False- batch = get_database_manager_client().get_user_notification_batch(- event.user_id, event.type+ batch = (+ await get_database_manager_async_client().get_user_notification_batch(+ event.user_id, event.type+ ))if not batch or not batch.notifications:logger.error(f"Batch not found for user {event.user_id}")@@ -702,7 +707,7 @@ class NotificationManager(AppService):logger.info(f"Successfully sent all {successfully_sent_count} notifications, clearing batch")- get_database_manager_client().empty_user_notification_batch(+ await get_database_manager_async_client().empty_user_notification_batch(event.user_id, event.type)else:@@ -715,7 +720,7 @@ class NotificationManager(AppService):logger.exception(f"Error processing notification for batch queue: {e}")return False- def _process_summary(self, message: str) -> bool:+ async def _process_summary(self, message: str) -> bool:"""Process a single notification with a summary strategy, returning whether to put into the failed queue"""try:logger.info(f"Processing summary notification: {message}")@@ -726,13 +731,15 @@ class NotificationManager(AppService):logger.info(f"Processing summary notification: {model}")- recipient_email = get_database_manager_client().get_user_email_by_id(- event.user_id+ recipient_email = (+ await get_database_manager_async_client().get_user_email_by_id(+ event.user_id+ ))if not recipient_email:logger.error(f"User email not found for user {event.user_id}")return False- should_send = self._should_email_user_based_on_preference(+ should_send = await self._should_email_user_based_on_preference(event.user_id, event.type)if not should_send:@@ -767,7 +774,7 @@ class NotificationManager(AppService):async def _consume_queue(self,queue: aio_pika.abc.AbstractQueue,- process_func: Callable[[str], bool],+ process_func: Callable[[str], Awaitable[bool]],queue_name: str,):"""Continuously consume messages from a queue using async iteration"""@@ -781,7 +788,7 @@ class NotificationManager(AppService):try:async with message.process():- result = process_func(message.body.decode())+ result = await process_func(message.body.decode())if not result:# Message will be rejected when exiting context without exceptionraise aio_pika.exceptions.MessageProcessError(diff --git a/autogpt_platform/backend/backend/util/metrics.py b/autogpt_platform/backend/backend/util/metrics.pyindex f169887307..3752ded608 100644--- a/autogpt_platform/backend/backend/util/metrics.py+++ b/autogpt_platform/backend/backend/util/metrics.py@@ -7,14 +7,16 @@ from sentry_sdk.integrations.logging import LoggingIntegrationfrom backend.util.settings import Settings+settings = Settings()+def sentry_init():- sentry_dsn = Settings().secrets.sentry_dsn+ sentry_dsn = settings.secrets.sentry_dsnsentry_sdk.init(dsn=sentry_dsn,traces_sample_rate=1.0,profiles_sample_rate=1.0,- environment=f"app:{Settings().config.app_env.value}-behave:{Settings().config.behave_as.value}",+ environment=f"app:{settings.config.app_env.value}-behave:{settings.config.behave_as.value}",_experiments={"enable_logs": True},integrations=[LoggingIntegration(sentry_logs_level=logging.INFO),@@ -33,9 +35,7 @@ def sentry_capture_error(error: Exception):async def discord_send_alert(content: str):from backend.blocks.discord import SendDiscordMessageBlockfrom backend.data.model import APIKeyCredentials, CredentialsMetaInput, ProviderName- from backend.util.settings import Settings- settings = Settings()creds = APIKeyCredentials(provider="discord",api_key=SecretStr(settings.secrets.discord_bot_token),