AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Fix structured logging for cloud environments (#11227)
Reinier van der Leer04df98111572
diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py b/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.pyindex 93a4030bcc..af958d65c3 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py@@ -94,42 +94,36 @@ def configure_logging(force_cloud_logging: bool = False) -> None:config = LoggingConfig()log_handlers: list[logging.Handler] = []+ structured_logging = config.enable_cloud_logging or force_cloud_logging+# Console output handlers- stdout = logging.StreamHandler(stream=sys.stdout)- stdout.setLevel(config.level)- stdout.addFilter(BelowLevelFilter(logging.WARNING))- if config.level == logging.DEBUG:- stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))- else:- stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))+ if not structured_logging:+ stdout = logging.StreamHandler(stream=sys.stdout)+ stdout.setLevel(config.level)+ stdout.addFilter(BelowLevelFilter(logging.WARNING))+ if config.level == logging.DEBUG:+ stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))+ else:+ stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))- stderr = logging.StreamHandler()- stderr.setLevel(logging.WARNING)- if config.level == logging.DEBUG:- stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))- else:- stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))+ stderr = logging.StreamHandler()+ stderr.setLevel(logging.WARNING)+ if config.level == logging.DEBUG:+ stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))+ else:+ stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))- log_handlers += [stdout, stderr]+ log_handlers += [stdout, stderr]# Cloud logging setup- if config.enable_cloud_logging or force_cloud_logging:- import google.cloud.logging- from google.cloud.logging.handlers import CloudLoggingHandler- from google.cloud.logging_v2.handlers.transports import (- BackgroundThreadTransport,- )+ else:+ # Use Google Cloud Structured Log Handler. Log entries are printed to stdout+ # in a JSON format which is automatically picked up by Google Cloud Logging.+ from google.cloud.logging.handlers import StructuredLogHandler- client = google.cloud.logging.Client()- # Use BackgroundThreadTransport to prevent blocking the main thread- # and deadlocks when gRPC calls to Google Cloud Logging hang- cloud_handler = CloudLoggingHandler(- client,- name="autogpt_logs",- transport=BackgroundThreadTransport,- )- cloud_handler.setLevel(config.level)- log_handlers.append(cloud_handler)+ structured_log_handler = StructuredLogHandler(stream=sys.stdout)+ structured_log_handler.setLevel(config.level)+ log_handlers.append(structured_log_handler)# File logging setupif config.enable_file_logging:@@ -185,7 +179,13 @@ def configure_logging(force_cloud_logging: bool = False) -> None:# Configure the root loggerlogging.basicConfig(- format=DEBUG_LOG_FORMAT if config.level == logging.DEBUG else SIMPLE_LOG_FORMAT,+ format=(+ "%(levelname)s %(message)s"+ if structured_logging+ else (+ DEBUG_LOG_FORMAT if config.level == logging.DEBUG else SIMPLE_LOG_FORMAT+ )+ ),level=config.level,handlers=log_handlers,)diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 0747c19541..a2a6d429f1 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -42,7 +42,7 @@ from backend.util.clients import (get_integration_credentials_store,)from backend.util.exceptions import GraphValidationError, NotFoundError-from backend.util.logging import TruncatedLogger+from backend.util.logging import TruncatedLogger, is_structured_logging_enabledfrom backend.util.settings import Configfrom backend.util.type import convert@@ -99,7 +99,11 @@ class LogMetadata(TruncatedLogger):"node_id": node_id,"block_name": block_name,}- prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]"+ prefix = (+ "[ExecutionManager]"+ if is_structured_logging_enabled()+ else f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]" # noqa+ )super().__init__(logger,max_length=max_length,diff --git a/autogpt_platform/backend/backend/util/logging.py b/autogpt_platform/backend/backend/util/logging.pyindex f07d154dab..61280d93f1 100644--- a/autogpt_platform/backend/backend/util/logging.py+++ b/autogpt_platform/backend/backend/util/logging.py@@ -8,10 +8,7 @@ settings = Settings()def configure_logging():import autogpt_libs.logging.config- if (- settings.config.behave_as == BehaveAs.LOCAL- or settings.config.app_env == AppEnvironment.LOCAL- ):+ if not is_structured_logging_enabled():autogpt_libs.logging.config.configure_logging(force_cloud_logging=False)else:autogpt_libs.logging.config.configure_logging(force_cloud_logging=True)@@ -20,6 +17,14 @@ def configure_logging():logging.getLogger("httpx").setLevel(logging.WARNING)+def is_structured_logging_enabled() -> bool:+ """Check if structured logging (cloud logging) is enabled."""+ return not (+ settings.config.behave_as == BehaveAs.LOCAL+ or settings.config.app_env == AppEnvironment.LOCAL+ )++class TruncatedLogger:def __init__(self,