Skip to content

Automate browser based workflows with AI

fix(SKY-12701): disable OTLP trace export on Hetzner temporal workers and quiet exporter logs (#7552)

LawyZhengd3ba7efb9c91
diff --git a/skyvern/config.py b/skyvern/config.py
index 94847d884..df7423fe5 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -778,6 +778,10 @@ class Settings(BaseSettings):
OTEL_METRICS_ENABLED: bool = True
OTEL_LOGS_ENABLED: bool = True
OTEL_EXPORTER_INSECURE: bool = True
+ # Log level for the OTLP gRPC exporter's own logger. Raise above WARNING (e.g.
+ # "CRITICAL") to drop its retry/failure records where the OTLP endpoint is
+ # intentionally unavailable; the default keeps export failures visible.
+ OTEL_EXPORTER_LOG_LEVEL: str = "WARNING"
# script generation settings
WORKFLOW_START_BLOCK_LABEL: str = "__start_block__"
diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.py
index e9bfca280..7a31d0542 100644
--- a/skyvern/forge/sdk/forge_log.py
+++ b/skyvern/forge/sdk/forge_log.py
@@ -518,6 +518,13 @@ def setup_logger() -> None:
logging.getLogger("LiteLLM Router").setLevel(logging.CRITICAL)
logging.getLogger("LiteLLM Proxy").setLevel(logging.CRITICAL)
+ # The OTLP gRPC exporter logs a WARNING per retry and an ERROR per dropped batch when its
+ # endpoint is unreachable; raise its threshold via OTEL_EXPORTER_LOG_LEVEL (default WARNING) to
+ # drop that spam where the endpoint is intentionally unavailable, keeping it visible elsewhere.
+ logging.getLogger("opentelemetry.exporter.otlp.proto.grpc.exporter").setLevel(
+ LOGGING_LEVEL_MAP.get(settings.OTEL_EXPORTER_LOG_LEVEL.upper(), logging.WARNING)
+ )
+
# Drop asyncio's orphaned-future noise from torn-down Playwright driver pipes (logged at
# ERROR but non-actionable). setup_logger may run more than once (uvicorn reload), so keep
# exactly one instance instead of stacking duplicates.
diff --git a/tests/unit/test_forge_log_uvicorn_levels.py b/tests/unit/test_forge_log_uvicorn_levels.py
index 2aaf9cf5e..5768a663c 100644
--- a/tests/unit/test_forge_log_uvicorn_levels.py
+++ b/tests/unit/test_forge_log_uvicorn_levels.py
@@ -24,8 +24,11 @@ from types import ModuleType
import pytest
+from skyvern.config import settings
from skyvern.forge.sdk.forge_log import setup_logger
+_OTLP_EXPORTER_LOGGER = "opentelemetry.exporter.otlp.proto.grpc.exporter"
+
_TOUCHED_LOGGERS = (
"uvicorn.error",
"uvicorn.access",
@@ -35,6 +38,7 @@ _TOUCHED_LOGGERS = (
"websockets.client",
"websockets.legacy",
"websockets.legacy.server",
+ _OTLP_EXPORTER_LOGGER,
)
@@ -57,6 +61,37 @@ def test_setup_logger_silences_uvicorn_and_websockets(_restore_logger_levels: No
assert logging.getLogger("websockets.legacy.server").level == logging.WARNING
+def test_setup_logger_defaults_otlp_exporter_logger_to_warning(_restore_logger_levels: None) -> None:
+ setup_logger()
+ assert logging.getLogger(_OTLP_EXPORTER_LOGGER).level == logging.WARNING
+
+
+def test_setup_logger_suppresses_otlp_exporter_logs_at_critical(
+ _restore_logger_levels: None, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(settings, "OTEL_EXPORTER_LOG_LEVEL", "CRITICAL")
+
+ setup_logger()
+
+ exporter_logger = logging.getLogger(_OTLP_EXPORTER_LOGGER)
+ assert exporter_logger.level == logging.CRITICAL
+ # The exporter emits its failure spam at WARNING ("Transient error ... retrying")
+ # and ERROR ("Failed to export ..."); both records must be dropped so they never
+ # reach stdout/log ingestion. A mere severity relabel would still emit them.
+ assert exporter_logger.isEnabledFor(logging.WARNING) is False
+ assert exporter_logger.isEnabledFor(logging.ERROR) is False
+
+
+def test_setup_logger_falls_back_to_warning_for_unknown_exporter_level(
+ _restore_logger_levels: None, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(settings, "OTEL_EXPORTER_LOG_LEVEL", "NOT_A_LEVEL")
+
+ setup_logger()
+
+ assert logging.getLogger(_OTLP_EXPORTER_LOGGER).level == logging.WARNING
+
+
@pytest.mark.parametrize(
"module_path",
[