AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): standardize service health checks with UnhealthyServiceError (#10584)
Zamil Majdye8f897ead1f9
diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex 00b0dbbd83..c27f95c957 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -41,7 +41,13 @@ from backend.data.user import (get_user_notification_preference,update_user_integrations,)-from backend.util.service import AppService, AppServiceClient, endpoint_to_sync, expose+from backend.util.service import (+ AppService,+ AppServiceClient,+ UnhealthyServiceError,+ endpoint_to_sync,+ expose,+)from backend.util.settings import Configconfig = Config()@@ -75,7 +81,7 @@ class DatabaseManager(AppService):def health_check(self) -> str:if not db.is_connected():- raise RuntimeError("Database is not connected")+ raise UnhealthyServiceError("Database is not connected")return super().health_check()@classmethoddiff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex 3c01e2ce91..5d657e7f49 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -31,7 +31,13 @@ from backend.util.cloud_storage import cleanup_expired_files_asyncfrom backend.util.exceptions import NotAuthorizedError, NotFoundErrorfrom backend.util.logging import PrefixFilterfrom backend.util.retry import func_retry-from backend.util.service import AppService, AppServiceClient, endpoint_to_async, expose+from backend.util.service import (+ AppService,+ AppServiceClient,+ UnhealthyServiceError,+ endpoint_to_async,+ expose,+)from backend.util.settings import Config@@ -192,7 +198,7 @@ class Scheduler(AppService):def health_check(self) -> str:# Thread-safe health check with proper initialization handlingif not hasattr(self, "scheduler"):- raise RuntimeError("Scheduler is still initializing")+ raise UnhealthyServiceError("Scheduler is still initializing")# Check if we're in the middle of cleanupif self.cleaned_up:@@ -200,8 +206,7 @@ class Scheduler(AppService):# Normal operation - check if scheduler is runningif not self.scheduler.running:- logger.error(f"{self.service_name} the scheduler is not running!")- raise RuntimeError("Scheduler is not running")+ raise UnhealthyServiceError("Scheduler is not running")return super().health_check()diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex 76120377de..afdcacbce9 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -31,7 +31,13 @@ from backend.util.clients import get_database_manager_async_clientfrom backend.util.logging import TruncatedLoggerfrom backend.util.metrics import discord_send_alertfrom backend.util.retry import continuous_retry-from backend.util.service import AppService, AppServiceClient, endpoint_to_sync, expose+from backend.util.service import (+ AppService,+ AppServiceClient,+ UnhealthyServiceError,+ endpoint_to_sync,+ expose,+)from backend.util.settings import Settingslogger = TruncatedLogger(logging.getLogger(__name__), "[NotificationManager]")@@ -183,16 +189,24 @@ class NotificationManager(AppService):def rabbit(self) -> rabbitmq.AsyncRabbitMQ:"""Access the RabbitMQ service. Will raise if not configured."""if not hasattr(self, "rabbitmq_service") or not self.rabbitmq_service:- raise RuntimeError("RabbitMQ not configured for this service")+ raise UnhealthyServiceError("RabbitMQ not configured for this service")return self.rabbitmq_service@propertydef rabbit_config(self) -> rabbitmq.RabbitMQConfig:"""Access the RabbitMQ config. Will raise if not configured."""if not self.rabbitmq_config:- raise RuntimeError("RabbitMQ not configured for this service")+ raise UnhealthyServiceError("RabbitMQ not configured for this service")return self.rabbitmq_config+ def health_check(self) -> str:+ # Service is unhealthy if RabbitMQ is not ready+ if not hasattr(self, "rabbitmq_service") or not self.rabbitmq_service:+ raise UnhealthyServiceError("RabbitMQ not configured for this service")+ if not self.rabbitmq_service.is_ready:+ raise UnhealthyServiceError("RabbitMQ channel is not ready")+ return super().health_check()+@classmethoddef get_port(cls) -> int:return settings.config.notification_service_portdiff --git a/autogpt_platform/backend/backend/server/rest_api.py b/autogpt_platform/backend/backend/server/rest_api.pyindex cdf0ea38b3..40ff268042 100644--- a/autogpt_platform/backend/backend/server/rest_api.py+++ b/autogpt_platform/backend/backend/server/rest_api.py@@ -41,6 +41,7 @@ from backend.server.external.api import external_appfrom backend.server.middleware.security import SecurityHeadersMiddlewarefrom backend.util import jsonfrom backend.util.cloud_storage import shutdown_cloud_storage_handler+from backend.util.service import UnhealthyServiceErrorsettings = backend.util.settings.Settings()logger = logging.getLogger(__name__)@@ -232,7 +233,7 @@ app.mount("/external-api", external_app)@app.get(path="/health", tags=["health"], dependencies=[])async def health():if not backend.data.db.is_connected():- raise RuntimeError("Database is not connected")+ raise UnhealthyServiceError("Database is not connected")return {"status": "healthy"}diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 2f8f8410af..bc91dcc7de 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -97,6 +97,18 @@ class RemoteCallError(BaseModel):args: Optional[Tuple[Any, ...]] = None+class UnhealthyServiceError(ValueError):+ def __init__(self, message: str = "Service is unhealthy", log: bool = True):+ msg = f"[{get_service_name()}] - please check the service health."+ super().__init__(msg)+ self.message = msg+ if log:+ logger.error(self.message)++ def __str__(self):+ return self.message++EXCEPTION_MAPPING = {e.__name__: efor e in [@@ -104,6 +116,7 @@ EXCEPTION_MAPPING = {RuntimeError,TimeoutError,ConnectionError,+ UnhealthyServiceError,*[ErrorTypefor _, ErrorType in inspect.getmembers(exceptions)