Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): Fix structured logging for cloud environments (#11227)

Reinier van der Leer04df98111572
diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py b/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py
index 93a4030bcc..af958d65c3 100644
--- a/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py
+++ b/autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py
@@ -94,42 +94,36 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
config = LoggingConfig()
log_handlers: list[logging.Handler] = []
+ structured_logging = config.enable_cloud_logging or force_cloud_logging
+
# Console output handlers
- stdout = logging.StreamHandler(stream=sys.stdout)
- stdout.setLevel(config.level)
- stdout.addFilter(BelowLevelFilter(logging.WARNING))
- if config.level == logging.DEBUG:
- stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
- else:
- stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
+ if not structured_logging:
+ stdout = logging.StreamHandler(stream=sys.stdout)
+ stdout.setLevel(config.level)
+ stdout.addFilter(BelowLevelFilter(logging.WARNING))
+ if config.level == logging.DEBUG:
+ stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
+ else:
+ stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
- stderr = logging.StreamHandler()
- stderr.setLevel(logging.WARNING)
- if config.level == logging.DEBUG:
- stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
- else:
- stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
+ stderr = logging.StreamHandler()
+ stderr.setLevel(logging.WARNING)
+ if config.level == logging.DEBUG:
+ stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
+ else:
+ stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
- log_handlers += [stdout, stderr]
+ log_handlers += [stdout, stderr]
# Cloud logging setup
- if config.enable_cloud_logging or force_cloud_logging:
- import google.cloud.logging
- from google.cloud.logging.handlers import CloudLoggingHandler
- from google.cloud.logging_v2.handlers.transports import (
- BackgroundThreadTransport,
- )
+ else:
+ # Use Google Cloud Structured Log Handler. Log entries are printed to stdout
+ # in a JSON format which is automatically picked up by Google Cloud Logging.
+ from google.cloud.logging.handlers import StructuredLogHandler
- client = google.cloud.logging.Client()
- # Use BackgroundThreadTransport to prevent blocking the main thread
- # and deadlocks when gRPC calls to Google Cloud Logging hang
- cloud_handler = CloudLoggingHandler(
- client,
- name="autogpt_logs",
- transport=BackgroundThreadTransport,
- )
- cloud_handler.setLevel(config.level)
- log_handlers.append(cloud_handler)
+ structured_log_handler = StructuredLogHandler(stream=sys.stdout)
+ structured_log_handler.setLevel(config.level)
+ log_handlers.append(structured_log_handler)
# File logging setup
if config.enable_file_logging:
@@ -185,7 +179,13 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
# Configure the root logger
logging.basicConfig(
- format=DEBUG_LOG_FORMAT if config.level == logging.DEBUG else SIMPLE_LOG_FORMAT,
+ format=(
+ "%(levelname)s %(message)s"
+ if structured_logging
+ else (
+ DEBUG_LOG_FORMAT if config.level == logging.DEBUG else SIMPLE_LOG_FORMAT
+ )
+ ),
level=config.level,
handlers=log_handlers,
)
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index 0747c19541..a2a6d429f1 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -42,7 +42,7 @@ from backend.util.clients import (
get_integration_credentials_store,
)
from backend.util.exceptions import GraphValidationError, NotFoundError
-from backend.util.logging import TruncatedLogger
+from backend.util.logging import TruncatedLogger, is_structured_logging_enabled
from backend.util.settings import Config
from backend.util.type import convert
@@ -99,7 +99,11 @@ class LogMetadata(TruncatedLogger):
"node_id": node_id,
"block_name": block_name,
}
- prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]"
+ prefix = (
+ "[ExecutionManager]"
+ if is_structured_logging_enabled()
+ else f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]" # noqa
+ )
super().__init__(
logger,
max_length=max_length,
diff --git a/autogpt_platform/backend/backend/util/logging.py b/autogpt_platform/backend/backend/util/logging.py
index f07d154dab..61280d93f1 100644
--- a/autogpt_platform/backend/backend/util/logging.py
+++ b/autogpt_platform/backend/backend/util/logging.py
@@ -8,10 +8,7 @@ settings = Settings()
def configure_logging():
import autogpt_libs.logging.config
- if (
- settings.config.behave_as == BehaveAs.LOCAL
- or settings.config.app_env == AppEnvironment.LOCAL
- ):
+ if not is_structured_logging_enabled():
autogpt_libs.logging.config.configure_logging(force_cloud_logging=False)
else:
autogpt_libs.logging.config.configure_logging(force_cloud_logging=True)
@@ -20,6 +17,14 @@ def configure_logging():
logging.getLogger("httpx").setLevel(logging.WARNING)
+def is_structured_logging_enabled() -> bool:
+ """Check if structured logging (cloud logging) is enabled."""
+ return not (
+ settings.config.behave_as == BehaveAs.LOCAL
+ or settings.config.app_env == AppEnvironment.LOCAL
+ )
+
+
class TruncatedLogger:
def __init__(
self,