AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): resolve scheduler deadlock and improve health checks (#10589)
Zamil Majdyd4b5508ed14d
diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex c27f95c957..f02a050a63 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -79,10 +79,10 @@ class DatabaseManager(AppService):logger.info(f"[{self.service_name}] ⏳ Disconnecting Database...")self.run_and_wait(db.disconnect())- def health_check(self) -> str:+ async def health_check(self) -> str:if not db.is_connected():raise UnhealthyServiceError("Database is not connected")- return super().health_check()+ return await super().health_check()@classmethoddef get_port(cls) -> int:diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex 06a4ad5dea..50b165045c 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -15,7 +15,7 @@ from apscheduler.events import (from apscheduler.job import Job as JobObjfrom apscheduler.jobstores.memory import MemoryJobStorefrom apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore-from apscheduler.schedulers.blocking import BlockingScheduler+from apscheduler.schedulers.background import BackgroundSchedulerfrom apscheduler.triggers.cron import CronTriggerfrom dotenv import load_dotenvfrom pydantic import BaseModel, Field, ValidationError@@ -134,6 +134,7 @@ def execute_graph(**kwargs):async def _execute_graph(**kwargs):args = GraphExecutionJobArgs(**kwargs)+ start_time = asyncio.get_event_loop().time()try:logger.info(f"Executing recurring job for graph #{args.graph_id}")graph_exec: GraphExecutionWithNodes = await execution_utils.add_graph_execution(@@ -143,12 +144,22 @@ async def _execute_graph(**kwargs):inputs=args.input_data,graph_credentials_inputs=args.input_credentials,)+ elapsed = asyncio.get_event_loop().time() - start_timelogger.info(- f"Graph execution started with ID {graph_exec.id} for graph {args.graph_id}"+ f"Graph execution started with ID {graph_exec.id} for graph {args.graph_id} "+ f"(took {elapsed:.2f}s to create and publish)")+ if elapsed > 10:+ logger.warning(+ f"Graph execution {graph_exec.id} took {elapsed:.2f}s to create/publish - "+ f"this is unusually slow and may indicate resource contention"+ )except Exception as e:- # TODO: We need to communicate this error to the user somehow.- logger.error(f"Error executing graph {args.graph_id}: {e}")+ elapsed = asyncio.get_event_loop().time() - start_time+ logger.error(+ f"Error executing graph {args.graph_id} after {elapsed:.2f}s: "+ f"{type(e).__name__}: {e}"+ )def cleanup_expired_files():@@ -210,7 +221,7 @@ class NotificationJobInfo(NotificationJobArgs):class Scheduler(AppService):- scheduler: BlockingScheduler+ scheduler: BackgroundSchedulerdef __init__(self, register_system_tasks: bool = True):self.register_system_tasks = register_system_tasks@@ -223,20 +234,20 @@ class Scheduler(AppService):def db_pool_size(cls) -> int:return config.scheduler_db_pool_size- def health_check(self) -> str:+ async def health_check(self) -> str:# Thread-safe health check with proper initialization handlingif not hasattr(self, "scheduler"):raise UnhealthyServiceError("Scheduler is still initializing")# Check if we're in the middle of cleanupif self.cleaned_up:- return super().health_check()+ return await super().health_check()# Normal operation - check if scheduler is runningif not self.scheduler.running:raise UnhealthyServiceError("Scheduler is not running")- return super().health_check()+ return await super().health_check()def run_service(self):load_dotenv()@@ -256,7 +267,7 @@ class Scheduler(AppService):# Configure executors to limit concurrency without skipping jobsfrom apscheduler.executors.pool import ThreadPoolExecutor- self.scheduler = BlockingScheduler(+ self.scheduler = BackgroundScheduler(executors={"default": ThreadPoolExecutor(max_workers=10), # Max 10 concurrent jobs},@@ -348,6 +359,9 @@ class Scheduler(AppService):self.scheduler.add_listener(job_max_instances_listener, EVENT_JOB_MAX_INSTANCES)self.scheduler.start()+ # Keep the service running since BackgroundScheduler doesn't block+ super().run_service()+def cleanup(self):super().cleanup()if self.scheduler:diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex afdcacbce9..2036891cda 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -199,13 +199,13 @@ class NotificationManager(AppService):raise UnhealthyServiceError("RabbitMQ not configured for this service")return self.rabbitmq_config- def health_check(self) -> str:+ async def health_check(self) -> str:# Service is unhealthy if RabbitMQ is not readyif not hasattr(self, "rabbitmq_service") or not self.rabbitmq_service:raise UnhealthyServiceError("RabbitMQ not configured for this service")if not self.rabbitmq_service.is_ready:raise UnhealthyServiceError("RabbitMQ channel is not ready")- return super().health_check()+ return await super().health_check()@classmethoddef get_port(cls) -> int:diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 4dc5976835..0346d35944 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -222,7 +222,7 @@ class AppService(BaseAppService, ABC):)self.shared_event_loop.run_until_complete(server.serve())- def health_check(self) -> str:+ async def health_check(self) -> str:"""A method to check the health of the process."""