Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): migrate notification service to fully async to resolve RabbitMQ connection issues (#10564)

Zamil Majdye2af2f454dd8
diff --git a/autogpt_platform/backend/backend/blocks/google/calendar.py b/autogpt_platform/backend/backend/blocks/google/calendar.py
index c75e0ec9eb..339daab430 100644
--- a/autogpt_platform/backend/backend/blocks/google/calendar.py
+++ b/autogpt_platform/backend/backend/blocks/google/calendar.py
@@ -21,6 +21,8 @@ from ._auth import (
GoogleCredentialsInput,
)
+settings = Settings()
+
class CalendarEvent(BaseModel):
"""Structured representation of a Google Calendar event."""
@@ -221,8 +223,8 @@ class GoogleCalendarReadEventsBlock(Block):
else None
),
token_uri="https://oauth2.googleapis.com/token",
- client_id=Settings().secrets.google_client_id,
- client_secret=Settings().secrets.google_client_secret,
+ client_id=settings.secrets.google_client_id,
+ client_secret=settings.secrets.google_client_secret,
scopes=credentials.scopes,
)
return build("calendar", "v3", credentials=creds)
@@ -569,8 +571,8 @@ class GoogleCalendarCreateEventBlock(Block):
else None
),
token_uri="https://oauth2.googleapis.com/token",
- client_id=Settings().secrets.google_client_id,
- client_secret=Settings().secrets.google_client_secret,
+ client_id=settings.secrets.google_client_id,
+ client_secret=settings.secrets.google_client_secret,
scopes=credentials.scopes,
)
return build("calendar", "v3", credentials=creds)
diff --git a/autogpt_platform/backend/backend/blocks/google/gmail.py b/autogpt_platform/backend/backend/blocks/google/gmail.py
index faa34a6015..eaa384d232 100644
--- a/autogpt_platform/backend/backend/blocks/google/gmail.py
+++ b/autogpt_platform/backend/backend/blocks/google/gmail.py
@@ -21,6 +21,8 @@ from ._auth import (
GoogleCredentialsInput,
)
+settings = Settings()
+
def serialize_email_recipients(recipients: list[str]) -> str:
"""Serialize recipients list to comma-separated string."""
@@ -255,8 +257,8 @@ class GmailReadBlock(Block):
else None
),
token_uri="https://oauth2.googleapis.com/token",
- client_id=Settings().secrets.google_client_id,
- client_secret=Settings().secrets.google_client_secret,
+ client_id=settings.secrets.google_client_id,
+ client_secret=settings.secrets.google_client_secret,
scopes=credentials.scopes,
)
return build("gmail", "v1", credentials=creds)
diff --git a/autogpt_platform/backend/backend/blocks/slant3d/order.py b/autogpt_platform/backend/backend/blocks/slant3d/order.py
index f1cab18d27..43a5802468 100644
--- a/autogpt_platform/backend/backend/blocks/slant3d/order.py
+++ b/autogpt_platform/backend/backend/blocks/slant3d/order.py
@@ -3,8 +3,7 @@ from typing import List
from backend.data.block import BlockOutput, BlockSchema
from backend.data.model import APIKeyCredentials, SchemaField
-from backend.util import settings
-from backend.util.settings import BehaveAs
+from backend.util.settings import BehaveAs, Settings
from ._api import (
TEST_CREDENTIALS,
@@ -16,6 +15,8 @@ from ._api import (
)
from .base import Slant3DBlockBase
+settings = Settings()
+
class Slant3DCreateOrderBlock(Slant3DBlockBase):
"""Block for creating new orders"""
@@ -280,7 +281,7 @@ class Slant3DGetOrdersBlock(Slant3DBlockBase):
input_schema=self.Input,
output_schema=self.Output,
# This block is disabled for cloud hosted because it allows access to all orders for the account
- disabled=settings.Settings().config.behave_as == BehaveAs.CLOUD,
+ disabled=settings.config.behave_as == BehaveAs.CLOUD,
test_input={"credentials": TEST_CREDENTIALS_INPUT},
test_credentials=TEST_CREDENTIALS,
test_output=[
diff --git a/autogpt_platform/backend/backend/blocks/slant3d/webhook.py b/autogpt_platform/backend/backend/blocks/slant3d/webhook.py
index 8a690cf1ad..22f87b468d 100644
--- a/autogpt_platform/backend/backend/blocks/slant3d/webhook.py
+++ b/autogpt_platform/backend/backend/blocks/slant3d/webhook.py
@@ -9,8 +9,7 @@ from backend.data.block import (
)
from backend.data.model import SchemaField
from backend.integrations.providers import ProviderName
-from backend.util import settings
-from backend.util.settings import AppEnvironment, BehaveAs
+from backend.util.settings import AppEnvironment, BehaveAs, Settings
from ._api import (
TEST_CREDENTIALS,
@@ -19,6 +18,8 @@ from ._api import (
Slant3DCredentialsInput,
)
+settings = Settings()
+
class Slant3DTriggerBase:
"""Base class for Slant3D webhook triggers"""
@@ -76,8 +77,8 @@ class Slant3DOrderWebhookBlock(Slant3DTriggerBase, Block):
),
# All webhooks are currently subscribed to for all orders. This works for self hosted, but not for cloud hosted prod
disabled=(
- settings.Settings().config.behave_as == BehaveAs.CLOUD
- and settings.Settings().config.app_env != AppEnvironment.LOCAL
+ settings.config.behave_as == BehaveAs.CLOUD
+ and settings.config.app_env != AppEnvironment.LOCAL
),
categories={BlockCategory.DEVELOPER_TOOLS},
input_schema=self.Input,
diff --git a/autogpt_platform/backend/backend/data/user.py b/autogpt_platform/backend/backend/data/user.py
index 83a62fc73b..8b57a1d6bd 100644
--- a/autogpt_platform/backend/backend/data/user.py
+++ b/autogpt_platform/backend/backend/data/user.py
@@ -21,6 +21,7 @@ from backend.util.json import SafeJson
from backend.util.settings import Settings
logger = logging.getLogger(__name__)
+settings = Settings()
async def get_or_create_user(user_data: dict) -> User:
@@ -332,7 +333,7 @@ async def get_user_email_verification(user_id: str) -> bool:
def generate_unsubscribe_link(user_id: str) -> str:
"""Generate a link to unsubscribe from all notifications"""
# Create an HMAC using a secret key
- secret_key = Settings().secrets.unsubscribe_secret_key
+ secret_key = settings.secrets.unsubscribe_secret_key
signature = hmac.new(
secret_key.encode("utf-8"), user_id.encode("utf-8"), hashlib.sha256
).digest()
@@ -343,7 +344,7 @@ def generate_unsubscribe_link(user_id: str) -> str:
).decode("utf-8")
logger.info(f"Generating unsubscribe link for user {user_id}")
- base_url = Settings().config.platform_base_url
+ base_url = settings.config.platform_base_url
return f"{base_url}/api/email/unsubscribe?token={quote_plus(token)}"
@@ -355,7 +356,7 @@ async def unsubscribe_user_by_token(token: str) -> None:
user_id, received_signature_hex = decoded.split(":", 1)
# Verify the signature
- secret_key = Settings().secrets.unsubscribe_secret_key
+ secret_key = settings.secrets.unsubscribe_secret_key
expected_signature = hmac.new(
secret_key.encode("utf-8"), user_id.encode("utf-8"), hashlib.sha256
).digest()
diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.py
index bdf59bc809..00b0dbbd83 100644
--- a/autogpt_platform/backend/backend/executor/database.py
+++ b/autogpt_platform/backend/backend/executor/database.py
@@ -163,23 +163,6 @@ class DatabaseManagerClient(AppServiceClient):
spend_credits = _(d.spend_credits)
get_credits = _(d.get_credits)
- # User Comms - async
- get_active_user_ids_in_timerange = _(d.get_active_user_ids_in_timerange)
- get_user_email_by_id = _(d.get_user_email_by_id)
- get_user_email_verification = _(d.get_user_email_verification)
- get_user_notification_preference = _(d.get_user_notification_preference)
-
- # Notifications - async
- create_or_add_to_user_notification_batch = _(
- d.create_or_add_to_user_notification_batch
- )
- empty_user_notification_batch = _(d.empty_user_notification_batch)
- get_all_batches_by_type = _(d.get_all_batches_by_type)
- get_user_notification_batch = _(d.get_user_notification_batch)
- get_user_notification_oldest_message_in_batch = _(
- d.get_user_notification_oldest_message_in_batch
- )
-
# Block error monitoring
get_block_error_stats = _(d.get_block_error_stats)
@@ -209,3 +192,20 @@ class DatabaseManagerAsyncClient(AppServiceClient):
update_user_integrations = d.update_user_integrations
get_execution_kv_data = d.get_execution_kv_data
set_execution_kv_data = d.set_execution_kv_data
+
+ # User Comms
+ get_active_user_ids_in_timerange = d.get_active_user_ids_in_timerange
+ get_user_email_by_id = d.get_user_email_by_id
+ get_user_email_verification = d.get_user_email_verification
+ get_user_notification_preference = d.get_user_notification_preference
+
+ # Notifications
+ create_or_add_to_user_notification_batch = (
+ d.create_or_add_to_user_notification_batch
+ )
+ empty_user_notification_batch = d.empty_user_notification_batch
+ get_all_batches_by_type = d.get_all_batches_by_type
+ get_user_notification_batch = d.get_user_notification_batch
+ get_user_notification_oldest_message_in_batch = (
+ d.get_user_notification_oldest_message_in_batch
+ )
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index e0c44b4d70..cdb4d33a46 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -1,11 +1,10 @@
import asyncio
import logging
-import multiprocessing
import os
import threading
import time
from collections import defaultdict
-from concurrent.futures import Future, ProcessPoolExecutor
+from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
@@ -102,6 +101,22 @@ utilization_gauge = Gauge(
"Ratio of active graph runs to max graph workers",
)
+# Thread-local storage for ExecutionProcessor instances
+_tls = threading.local()
+
+
+def init_worker():
+ """Initialize ExecutionProcessor instance in thread-local storage"""
+ _tls.processor = ExecutionProcessor()
+ _tls.processor.on_graph_executor_start()
+
+
+def execute_graph(
+ graph_exec_entry: "GraphExecutionEntry", cancel_event: threading.Event
+):
+ """Execute graph using thread-local ExecutionProcessor instance"""
+ return _tls.processor.on_graph_execution(graph_exec_entry, cancel_event)
+
T = TypeVar("T")
@@ -366,7 +381,7 @@ async def _enqueue_next_nodes(
]
-class Executor:
+class ExecutionProcessor:
"""
This class contains event handlers for the process pool executor events.
@@ -389,10 +404,9 @@ class Executor:
9. Node executor enqueues the next executed nodes to the node execution queue.
"""
- @classmethod
@async_error_logged(swallow=True)
async def on_node_execution(
- cls,
+ self,
node_exec: NodeExecutionEntry,
node_exec_progress: NodeExecutionProgress,
nodes_input_masks: Optional[dict[str, dict[str, JsonValue]]],
@@ -411,7 +425,7 @@ class Executor:
node = await db_client.get_node(node_exec.node_id)
execution_stats = NodeExecutionStats()
- timing_info, status = await cls._on_node_execution(
+ timing_info, status = await self._on_node_execution(
node=node,
node_exec=node_exec,
node_exec_progress=node_exec_progress,
@@ -454,10 +468,9 @@ class Executor:
return execution_stats
- @classmethod
@async_time_measured
async def _on_node_execution(
- cls,
+ self,
node: Node,
node_exec: NodeExecutionEntry,
node_exec_progress: NodeExecutionProgress,
@@ -497,7 +510,7 @@ class Executor:
async for output_name, output_data in execute_node(
node=node,
- creds_manager=cls.creds_manager,
+ creds_manager=self.creds_manager,
data=node_exec,
execution_stats=stats,
nodes_input_masks=nodes_input_masks,
@@ -537,29 +550,27 @@ class Executor:
return status
- @classmethod
@func_retry
- def on_graph_executor_start(cls):
+ def on_graph_executor_start(self):
configure_logging()
set_service_name("GraphExecutor")
- cls.pid = os.getpid()
- cls.creds_manager = IntegrationCredentialsManager()
- cls.node_execution_loop = asyncio.new_event_loop()
- cls.node_evaluation_loop = asyncio.new_event_loop()
- cls.node_execution_thread = threading.Thread(
- target=cls.node_execution_loop.run_forever, daemon=True
+ self.tid = threading.get_ident()
+ self.creds_manager = IntegrationCredentialsManager()
+ self.node_execution_loop = asyncio.new_event_loop()
+ self.node_evaluation_loop = asyncio.new_event_loop()
+ self.node_execution_thread = threading.Thread(
+ target=self.node_execution_loop.run_forever, daemon=True
)
- cls.node_evaluation_thread = threading.Thread(
- target=cls.node_evaluation_loop.run_forever, daemon=True
+ self.node_evaluation_thread = threading.Thread(
+ target=self.node_evaluation_loop.run_forever, daemon=True
)
- cls.node_execution_thread.start()
- cls.node_evaluation_thread.start()
- logger.info(f"[GraphExecutor] {cls.pid} started")
+ self.node_execution_thread.start()
+ self.node_evaluation_thread.start()
+ logger.info(f"[GraphExecutor] {self.tid} started")
- @classmethod
@error_logged(swallow=False)
def on_graph_execution(
- cls,
+ self,
graph_exec: GraphExecutionEntry,
cancel: threading.Event,
):
@@ -615,7 +626,7 @@ class Executor:
else:
exec_stats = exec_meta.stats.to_db()
- timing_info, status = cls._on_graph_execution(
+ timing_info, status = self._on_graph_execution(
graph_exec=graph_exec,
cancel=cancel,
log_metadata=log_metadata,
@@ -641,7 +652,7 @@ class Executor:
user_id=graph_exec.user_id,
execution_status=status,
),
- cls.node_execution_loop,
+ self.node_execution_loop,
).result(timeout=60.0)
if activity_status is not None:
exec_stats.activity_status = activity_status
@@ -652,7 +663,7 @@ class Executor:
)
# Communication handling
- cls._handle_agent_run_notif(db_client, graph_exec, exec_stats)
+ self._handle_agent_run_notif(db_client, graph_exec, exec_stats)
finally:
update_graph_execution_state(
@@ -662,9 +673,8 @@ class Executor:
stats=exec_stats,
)
- @classmethod
def _charge_usage(
- cls,
+ self,
node_exec: NodeExecutionEntry,
execution_count: int,
) -> int:
@@ -714,22 +724,14 @@ class Executor:
return total_cost
- @classmethod
@time_measured
def _on_graph_execution(
- cls,
+ self,
graph_exec: GraphExecutionEntry,
cancel: threading.Event,
log_metadata: LogMetadata,
execution_stats: GraphExecutionStats,
) -> ExecutionStatus:
-
- # Agent execution is uninterrupted.
- import signal
-
- signal.signal(signal.SIGINT, signal.SIG_IGN)
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
-
"""
Returns:
dict: The execution statistics of the graph execution.
@@ -786,7 +788,7 @@ class Executor:
# Charge usage (may raise) ------------------------------
try:
- cost = cls._charge_usage(
+ cost = self._charge_usage(
node_exec=queued_node_exec,
execution_count=increment_execution_count(graph_exec.user_id),
)
@@ -806,7 +808,7 @@ class Executor:
status=ExecutionStatus.FAILED,
)
- cls._handle_low_balance_notif(
+ self._handle_low_balance_notif(
db_client,
graph_exec.user_id,
graph_exec.graph_id,
@@ -825,7 +827,7 @@ class Executor:
# Kick off async node execution -------------------------
node_execution_task = asyncio.run_coroutine_threadsafe(
- cls.on_node_execution(
+ self.on_node_execution(
node_exec=queued_node_exec,
node_exec_progress=running_node_execution[node_id],
nodes_input_masks=nodes_input_masks,
@@ -834,7 +836,7 @@ class Executor:
execution_stats_lock,
),
),
- cls.node_execution_loop,
+ self.node_execution_loop,
)
running_node_execution[node_id].add_task(
node_exec_id=queued_node_exec.node_exec_id,
@@ -875,7 +877,7 @@ class Executor:
node_output_found = True
running_node_evaluation[node_id] = (
asyncio.run_coroutine_threadsafe(
- cls._process_node_output(
+ self._process_node_output(
output=output,
node_id=node_id,
graph_exec=graph_exec,
@@ -883,7 +885,7 @@ class Executor:
nodes_input_masks=nodes_input_masks,
execution_queue=execution_queue,
),
- cls.node_evaluation_loop,
+ self.node_evaluation_loop,
)
)
if (
@@ -926,7 +928,7 @@ class Executor:
raise
finally:
- cls._cleanup_graph_execution(
+ self._cleanup_graph_execution(
execution_queue=execution_queue,
running_node_execution=running_node_execution,
running_node_evaluation=running_node_evaluation,
@@ -937,10 +939,9 @@ class Executor:
db_client=db_client,
)
- @classmethod
@error_logged(swallow=True)
def _cleanup_graph_execution(
- cls,
+ self,
execution_queue: ExecutionQueue[NodeExecutionEntry],
running_node_execution: dict[str, "NodeExecutionProgress"],
running_node_evaluation: dict[str, Future],
@@ -991,10 +992,9 @@ class Executor:
clean_exec_files(graph_exec_id)
- @classmethod
@async_error_logged(swallow=True)
async def _process_node_output(
- cls,
+ self,
output: ExecutionOutputEntry,
node_id: str,
graph_exec: GraphExecutionEntry,
@@ -1027,9 +1027,8 @@ class Executor:
):
execution_queue.add(next_execution)
- @classmethod
def _handle_agent_run_notif(
- cls,
+ self,
db_client: "DatabaseManagerClient",
graph_exec: GraphExecutionEntry,
exec_stats: GraphExecutionStats,
@@ -1065,9 +1064,8 @@ class Executor:
)
)
- @classmethod
def _handle_low_balance_notif(
- cls,
+ self,
db_client: "DatabaseManagerClient",
user_id: str,
graph_id: str,
@@ -1132,11 +1130,11 @@ class ExecutionManager(AppProcess):
return self._stop_consuming
@property
- def executor(self) -> ProcessPoolExecutor:
+ def executor(self) -> ThreadPoolExecutor:
if self._executor is None:
- self._executor = ProcessPoolExecutor(
+ self._executor = ThreadPoolExecutor(
max_workers=self.pool_size,
- initializer=Executor.on_graph_executor_start,
+ initializer=init_worker,
)
return self._executor
@@ -1313,11 +1311,9 @@ class ExecutionManager(AppProcess):
_ack_message(reject=True)
return
- cancel_event = multiprocessing.Manager().Event()
+ cancel_event = threading.Event()
- future = self.executor.submit(
- Executor.on_graph_execution, graph_exec_entry, cancel_event
- )
+ future = self.executor.submit(execute_graph, graph_exec_entry, cancel_event)
self.active_graph_runs[graph_exec_id] = (future, cancel_event)
self._update_prompt_metrics()
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index cde3e2144c..a67e27973c 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -1,6 +1,7 @@
import asyncio
import logging
import os
+import threading
from enum import Enum
from typing import Optional
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
@@ -11,7 +12,6 @@ from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.triggers.cron import CronTrigger
-from autogpt_libs.utils.cache import thread_cached
from dotenv import load_dotenv
from pydantic import BaseModel, Field, ValidationError
from sqlalchemy import MetaData, create_engine
@@ -30,6 +30,7 @@ from backend.monitoring import (
from backend.util.cloud_storage import cleanup_expired_files_async
from backend.util.exceptions import NotAuthorizedError, NotFoundError
from backend.util.logging import PrefixFilter
+from backend.util.retry import func_retry
from backend.util.service import AppService, AppServiceClient, endpoint_to_async, expose
from backend.util.settings import Config
@@ -69,13 +70,23 @@ def job_listener(event):
logger.info(f"Job {event.job_id} completed successfully.")
-@thread_cached
+_event_loop: asyncio.AbstractEventLoop | None = None
+
+
+@func_retry
def get_event_loop():
- return asyncio.new_event_loop()
+ """Get the shared event loop."""
+ if _event_loop is None:
+ raise RuntimeError("Event loop not initialized. Scheduler not started.")
+ return _event_loop
def execute_graph(**kwargs):
- get_event_loop().run_until_complete(_execute_graph(**kwargs))
+ """Execute graph in the shared event loop and wait for completion."""
+ loop = get_event_loop()
+ future = asyncio.run_coroutine_threadsafe(_execute_graph(**kwargs), loop)
+ # Wait for completion to ensure job doesn't exit prematurely
+ future.result(timeout=300) # 5 minute timeout for graph execution
async def _execute_graph(**kwargs):
@@ -99,7 +110,10 @@ async def _execute_graph(**kwargs):
def cleanup_expired_files():
"""Clean up expired files from cloud storage."""
- get_event_loop().run_until_complete(cleanup_expired_files_async())
+ loop = get_event_loop()
+ future = asyncio.run_coroutine_threadsafe(cleanup_expired_files_async(), loop)
+ # Wait for completion
+ future.result(timeout=300) # 5 minute timeout for cleanup
# Monitoring functions are now imported from monitoring module
@@ -175,6 +189,17 @@ class Scheduler(AppService):
def run_service(self):
load_dotenv()
+
+ # Initialize the event loop for async jobs
+ global _event_loop
+ _event_loop = asyncio.new_event_loop()
+
+ # Use daemon thread since it should die with the main service
+ event_loop_thread = threading.Thread(
+ target=_event_loop.run_forever, daemon=True, name="SchedulerEventLoop"
+ )
+ event_loop_thread.start()
+
db_schema, db_url = _extract_schema_from_url(os.getenv("DIRECT_URL"))
self.scheduler = BlockingScheduler(
jobstores={
diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.py
index 67327d2985..052fc453fd 100644
--- a/autogpt_platform/backend/backend/notifications/notifications.py
+++ b/autogpt_platform/backend/backend/notifications/notifications.py
@@ -1,8 +1,7 @@
import asyncio
import logging
-from concurrent.futures import ProcessPoolExecutor
from datetime import datetime, timedelta, timezone
-from typing import Callable
+from typing import Awaitable, Callable
import aio_pika
from prisma.enums import NotificationType
@@ -28,7 +27,7 @@ from backend.data.notifications import (
from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
from backend.data.user import generate_unsubscribe_link
from backend.notifications.email import EmailSender
-from backend.util.clients import get_database_manager_client
+from backend.util.clients import get_database_manager_async_client
from backend.util.logging import TruncatedLogger
from backend.util.metrics import discord_send_alert
from backend.util.retry import continuous_retry
@@ -43,8 +42,6 @@ NOTIFICATION_EXCHANGE = Exchange(name="notifications", type=ExchangeType.TOPIC)
DEAD_LETTER_EXCHANGE = Exchange(name="dead_letter", type=ExchangeType.TOPIC)
EXCHANGES = [NOTIFICATION_EXCHANGE, DEAD_LETTER_EXCHANGE]
-background_executor = ProcessPoolExecutor(max_workers=2)
-
def create_notification_config() -> RabbitMQConfig:
"""Create RabbitMQ configuration for notifications"""
@@ -202,7 +199,8 @@ class NotificationManager(AppService):
@expose
def queue_weekly_summary(self):
- background_executor.submit(lambda: asyncio.run(self._queue_weekly_summary()))
+ # Use the existing event loop instead of creating a new one with asyncio.run()
+ asyncio.create_task(self._queue_weekly_summary())
async def _queue_weekly_summary(self):
"""Process weekly summary for specified notification types"""
@@ -211,7 +209,7 @@ class NotificationManager(AppService):
processed_count = 0
current_time = datetime.now(tz=timezone.utc)
start_time = current_time - timedelta(days=7)
- users = get_database_manager_client().get_active_user_ids_in_timerange(
+ users = await get_database_manager_async_client().get_active_user_ids_in_timerange(
end_time=current_time.isoformat(),
start_time=start_time.isoformat(),
)
@@ -235,9 +233,12 @@ class NotificationManager(AppService):
@expose
def process_existing_batches(self, notification_types: list[NotificationType]):
- background_executor.submit(self._process_existing_batches, notification_types)
+ # Use the existing event loop instead of creating a new process
+ asyncio.create_task(self._process_existing_batches(notification_types))
- def _process_existing_batches(self, notification_types: list[NotificationType]):
+ async def _process_existing_batches(
+ self, notification_types: list[NotificationType]
+ ):
"""Process existing batches for specified notification types"""
try:
processed_count = 0
@@ -245,13 +246,15 @@ class NotificationManager(AppService):
for notification_type in notification_types:
# Get all batches for this notification type
- batches = get_database_manager_client().get_all_batches_by_type(
- notification_type
+ batches = (
+ await get_database_manager_async_client().get_all_batches_by_type(
+ notification_type
+ )
)
for batch in batches:
# Check if batch has aged out
- oldest_message = get_database_manager_client().get_user_notification_oldest_message_in_batch(
+ oldest_message = await get_database_manager_async_client().get_user_notification_oldest_message_in_batch(
batch.user_id, notification_type
)
@@ -266,10 +269,8 @@ class NotificationManager(AppService):
# If batch has aged out, process it
if oldest_message.created_at + max_delay < current_time:
- recipient_email = (
- get_database_manager_client().get_user_email_by_id(
- batch.user_id
- )
+ recipient_email = await get_database_manager_async_client().get_user_email_by_id(
+ batch.user_id
)
if not recipient_email:
@@ -278,7 +279,7 @@ class NotificationManager(AppService):
)
continue
- should_send = self._should_email_user_based_on_preference(
+ should_send = await self._should_email_user_based_on_preference(
batch.user_id, notification_type
)
@@ -287,15 +288,13 @@ class NotificationManager(AppService):
f"User {batch.user_id} does not want to receive {notification_type} notifications"
)
# Clear the batch
- get_database_manager_client().empty_user_notification_batch(
+ await get_database_manager_async_client().empty_user_notification_batch(
batch.user_id, notification_type
)
continue
- batch_data = (
- get_database_manager_client().get_user_notification_batch(
- batch.user_id, notification_type
- )
+ batch_data = await get_database_manager_async_client().get_user_notification_batch(
+ batch.user_id, notification_type
)
if not batch_data or not batch_data.notifications:
@@ -303,7 +302,7 @@ class NotificationManager(AppService):
f"Batch data not found for user {batch.user_id}"
)
# Clear the batch
- get_database_manager_client().empty_user_notification_batch(
+ await get_database_manager_async_client().empty_user_notification_batch(
batch.user_id, notification_type
)
continue
@@ -339,7 +338,7 @@ class NotificationManager(AppService):
)
# Clear the batch
- get_database_manager_client().empty_user_notification_batch(
+ await get_database_manager_async_client().empty_user_notification_batch(
batch.user_id, notification_type
)
@@ -384,18 +383,20 @@ class NotificationManager(AppService):
except Exception as e:
logger.exception(f"Error queueing notification: {e}")
- def _should_email_user_based_on_preference(
+ async def _should_email_user_based_on_preference(
self, user_id: str, event_type: NotificationType
) -> bool:
"""Check if a user wants to receive a notification based on their preferences and email verification status"""
- validated_email = get_database_manager_client().get_user_email_verification(
- user_id
+ validated_email = (
+ await get_database_manager_async_client().get_user_email_verification(
+ user_id
+ )
)
preference = (
- get_database_manager_client()
- .get_user_notification_preference(user_id)
- .preferences.get(event_type, True)
- )
+ await get_database_manager_async_client().get_user_notification_preference(
+ user_id
+ )
+ ).preferences.get(event_type, True)
# only if both are true, should we email this person
return validated_email and preference
@@ -479,18 +480,16 @@ class NotificationManager(AppService):
else:
raise ValueError("Invalid event type or params")
- def _should_batch(
+ async def _should_batch(
self, user_id: str, event_type: NotificationType, event: NotificationEventModel
) -> bool:
- get_database_manager_client().create_or_add_to_user_notification_batch(
+ await get_database_manager_async_client().create_or_add_to_user_notification_batch(
user_id, event_type, event
)
- oldest_message = (
- get_database_manager_client().get_user_notification_oldest_message_in_batch(
- user_id, event_type
- )
+ oldest_message = await get_database_manager_async_client().get_user_notification_oldest_message_in_batch(
+ user_id, event_type
)
if not oldest_message:
logger.error(
@@ -519,7 +518,7 @@ class NotificationManager(AppService):
logger.error(f"Error parsing message due to non matching schema {e}")
return None
- def _process_admin_message(self, message: str) -> bool:
+ async def _process_admin_message(self, message: str) -> bool:
"""Process a single notification, sending to an admin, returning whether to put into the failed queue"""
try:
event = self._parse_message(message)
@@ -533,7 +532,7 @@ class NotificationManager(AppService):
logger.exception(f"Error processing notification for admin queue: {e}")
return False
- def _process_immediate(self, message: str) -> bool:
+ async def _process_immediate(self, message: str) -> bool:
"""Process a single notification immediately, returning whether to put into the failed queue"""
try:
event = self._parse_message(message)
@@ -541,14 +540,16 @@ class NotificationManager(AppService):
return False
logger.debug(f"Processing immediate notification: {event}")
- recipient_email = get_database_manager_client().get_user_email_by_id(
- event.user_id
+ recipient_email = (
+ await get_database_manager_async_client().get_user_email_by_id(
+ event.user_id
+ )
)
if not recipient_email:
logger.error(f"User email not found for user {event.user_id}")
return False
- should_send = self._should_email_user_based_on_preference(
+ should_send = await self._should_email_user_based_on_preference(
event.user_id, event.type
)
if not should_send:
@@ -570,7 +571,7 @@ class NotificationManager(AppService):
logger.exception(f"Error processing notification for immediate queue: {e}")
return False
- def _process_batch(self, message: str) -> bool:
+ async def _process_batch(self, message: str) -> bool:
"""Process a single notification with a batching strategy, returning whether to put into the failed queue"""
try:
event = self._parse_message(message)
@@ -578,14 +579,16 @@ class NotificationManager(AppService):
return False
logger.info(f"Processing batch notification: {event}")
- recipient_email = get_database_manager_client().get_user_email_by_id(
- event.user_id
+ recipient_email = (
+ await get_database_manager_async_client().get_user_email_by_id(
+ event.user_id
+ )
)
if not recipient_email:
logger.error(f"User email not found for user {event.user_id}")
return False
- should_send = self._should_email_user_based_on_preference(
+ should_send = await self._should_email_user_based_on_preference(
event.user_id, event.type
)
if not should_send:
@@ -594,13 +597,15 @@ class NotificationManager(AppService):
)
return True
- should_send = self._should_batch(event.user_id, event.type, event)
+ should_send = await self._should_batch(event.user_id, event.type, event)
if not should_send:
logger.info("Batch not old enough to send")
return False
- batch = get_database_manager_client().get_user_notification_batch(
- event.user_id, event.type
+ batch = (
+ await get_database_manager_async_client().get_user_notification_batch(
+ event.user_id, event.type
+ )
)
if not batch or not batch.notifications:
logger.error(f"Batch not found for user {event.user_id}")
@@ -702,7 +707,7 @@ class NotificationManager(AppService):
logger.info(
f"Successfully sent all {successfully_sent_count} notifications, clearing batch"
)
- get_database_manager_client().empty_user_notification_batch(
+ await get_database_manager_async_client().empty_user_notification_batch(
event.user_id, event.type
)
else:
@@ -715,7 +720,7 @@ class NotificationManager(AppService):
logger.exception(f"Error processing notification for batch queue: {e}")
return False
- def _process_summary(self, message: str) -> bool:
+ async def _process_summary(self, message: str) -> bool:
"""Process a single notification with a summary strategy, returning whether to put into the failed queue"""
try:
logger.info(f"Processing summary notification: {message}")
@@ -726,13 +731,15 @@ class NotificationManager(AppService):
logger.info(f"Processing summary notification: {model}")
- recipient_email = get_database_manager_client().get_user_email_by_id(
- event.user_id
+ recipient_email = (
+ await get_database_manager_async_client().get_user_email_by_id(
+ event.user_id
+ )
)
if not recipient_email:
logger.error(f"User email not found for user {event.user_id}")
return False
- should_send = self._should_email_user_based_on_preference(
+ should_send = await self._should_email_user_based_on_preference(
event.user_id, event.type
)
if not should_send:
@@ -767,7 +774,7 @@ class NotificationManager(AppService):
async def _consume_queue(
self,
queue: aio_pika.abc.AbstractQueue,
- process_func: Callable[[str], bool],
+ process_func: Callable[[str], Awaitable[bool]],
queue_name: str,
):
"""Continuously consume messages from a queue using async iteration"""
@@ -781,7 +788,7 @@ class NotificationManager(AppService):
try:
async with message.process():
- result = process_func(message.body.decode())
+ result = await process_func(message.body.decode())
if not result:
# Message will be rejected when exiting context without exception
raise aio_pika.exceptions.MessageProcessError(
diff --git a/autogpt_platform/backend/backend/util/metrics.py b/autogpt_platform/backend/backend/util/metrics.py
index f169887307..3752ded608 100644
--- a/autogpt_platform/backend/backend/util/metrics.py
+++ b/autogpt_platform/backend/backend/util/metrics.py
@@ -7,14 +7,16 @@ from sentry_sdk.integrations.logging import LoggingIntegration
from backend.util.settings import Settings
+settings = Settings()
+
def sentry_init():
- sentry_dsn = Settings().secrets.sentry_dsn
+ sentry_dsn = settings.secrets.sentry_dsn
sentry_sdk.init(
dsn=sentry_dsn,
traces_sample_rate=1.0,
profiles_sample_rate=1.0,
- environment=f"app:{Settings().config.app_env.value}-behave:{Settings().config.behave_as.value}",
+ environment=f"app:{settings.config.app_env.value}-behave:{settings.config.behave_as.value}",
_experiments={"enable_logs": True},
integrations=[
LoggingIntegration(sentry_logs_level=logging.INFO),
@@ -33,9 +35,7 @@ def sentry_capture_error(error: Exception):
async def discord_send_alert(content: str):
from backend.blocks.discord import SendDiscordMessageBlock
from backend.data.model import APIKeyCredentials, CredentialsMetaInput, ProviderName
- from backend.util.settings import Settings
- settings = Settings()
creds = APIKeyCredentials(
provider="discord",
api_key=SecretStr(settings.secrets.discord_bot_token),