Skip to content

Self-improving AI Employees for everyone

fix(observability): stop Temporal spans flooding the log at INFO

trohitgb625c6b3b2b0
diff --git a/server/core/config.py b/server/core/config.py
index 3c6a4dc8..4550bbb5 100644
--- a/server/core/config.py
+++ b/server/core/config.py
@@ -171,7 +171,7 @@ class Settings(BaseSettings):
le=65535,
)
# Web UI port. ``temporal server start-dev`` defaults this to
- # ``--port + 1000``; declared explicitly via ``--ui-port`` (5683 in
+ # ``--port + 1000``; declared explicitly via ``--ui-port`` (5680 in
# the serial 5678-block) so the binding is intentional and surfaces
# in status snapshots.
temporal_ui_port: int = Field(
@@ -311,6 +311,17 @@ class Settings(BaseSettings):
env="LOG_FILE_BACKUP_COUNT",
ge=0,
)
+ # OpenTelemetry console span export. Off unless explicitly enabled or
+ # LOG_LEVEL=DEBUG (see ``tracing_console_spans_enabled``): the Temporal
+ # SDK TracingInterceptor emits a span per Signal / Query / Update /
+ # Activity on both the caller and handler side, which floods the
+ # operator log at INFO with per-RPC lines.
+ tracing_console_spans: bool = Field(default=False, env="TRACING_CONSOLE_SPANS")
+
+ @property
+ def tracing_console_spans_enabled(self) -> bool:
+ """Console spans are debug output; DEBUG turns them on implicitly."""
+ return self.tracing_console_spans or self.log_level.upper() == "DEBUG"
# Rate Limiting
rate_limit_enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED")
diff --git a/server/core/tracing.py b/server/core/tracing.py
index 0a030f7a..fcbea676 100644
--- a/server/core/tracing.py
+++ b/server/core/tracing.py
@@ -10,8 +10,17 @@ want to emit spans do::
with tracer.start_as_current_span("namespace.operation"):
...
-In dev, ``ConsoleSpanExporter`` is wired with a compact one-line
-``formatter`` so each completed span shows up as::
+Span export to the console is **opt-in**. Spans are always recorded (so
+an OTLP exporter can be attached without touching call sites), but the
+``ConsoleSpanExporter`` is only wired when ``console_spans=True`` — set
+by ``LOG_LEVEL=DEBUG`` or ``TRACING_CONSOLE_SPANS=true``. At INFO the
+Temporal SDK's ``TracingInterceptor`` alone emits a span per
+Signal/Query/Update/Activity on both the caller and handler side, which
+buries the operator log in per-RPC lines that carry no operational
+signal.
+
+When enabled, a compact one-line ``formatter`` renders each completed
+span as::
[span] broadcaster.refresh_telegram 4012ms path=auto_reconnect connected=False
@@ -58,9 +67,21 @@ def _compact_span_formatter(span: ReadableSpan) -> str:
return " ".join(parts) + "\n"
-def init_tracing(service_name: str = "opencompany-backend") -> None:
+def init_tracing(
+ service_name: str = "opencompany-backend",
+ *,
+ console_spans: bool = False,
+) -> None:
"""Configure the global TracerProvider.
+ Args:
+ service_name: ``service.name`` resource attribute.
+ console_spans: Attach the console exporter. Off by default —
+ span-per-RPC output is a debugging tool, not operator
+ signal. The provider is still installed either way, so
+ ``trace.get_tracer(__name__)`` call sites keep working and
+ an OTLP exporter can be added without code changes.
+
Idempotent — safe to call multiple times. Subsequent calls are
no-ops so tests / reloads do not accumulate processors.
"""
@@ -69,8 +90,9 @@ def init_tracing(service_name: str = "opencompany-backend") -> None:
return
provider = TracerProvider(resource=Resource.create({SERVICE_NAME: service_name}))
- # Compact one-line console output instead of the default 30-line
- # JSON dump per span — keeps interactive logs readable.
- provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter(formatter=_compact_span_formatter)))
+ if console_spans:
+ # Compact one-line console output instead of the default 30-line
+ # JSON dump per span — keeps interactive logs readable.
+ provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter(formatter=_compact_span_formatter)))
trace.set_tracer_provider(provider)
_INITIALIZED = True
diff --git a/server/main.py b/server/main.py
index 28c7446f..8fb8f4f0 100644
--- a/server/main.py
+++ b/server/main.py
@@ -52,7 +52,7 @@ from core.tracing import init_tracing
settings = Settings()
configure_logging(settings)
-init_tracing()
+init_tracing(console_spans=settings.tracing_console_spans_enabled)
logger = get_logger(__name__)
_startup_log("Importing DI container + all services...")
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index ca294ff4..3b0e65a0 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -238,7 +238,11 @@ class TemporalWorkerManager:
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(self.task_queue),
- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
+ # ``TracingInterceptor`` is registered once on the shared client;
+ # the SDK prepends client interceptors to every worker, so
+ # repeating it here double-instruments each Signal / Query /
+ # Update / Activity and doubles the emitted spans.
+ interceptors=[ObservabilityWorkerInterceptor()],
# Wave 18.2: sticky cache sized by deployment mode so
# cached workflows skip Event-History replay without
# blowing laptop RAM.
@@ -498,7 +502,11 @@ class TemporalWorkerPool:
activities=activities,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
identity=_worker_identity(queue),
- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
+ # ``TracingInterceptor`` is registered once on the shared client;
+ # the SDK prepends client interceptors to every worker, so
+ # repeating it here double-instruments each Signal / Query /
+ # Update / Activity and doubles the emitted spans.
+ interceptors=[ObservabilityWorkerInterceptor()],
# Wave 18.1: per-queue activities/second ceiling.
max_activities_per_second=rate_limit,
# Wave 18.3: activity-only workers need a small
@@ -545,7 +553,7 @@ class TemporalWorkerPool:
async def run_standalone_worker(
- server_address: str = "localhost:5682",
+ server_address: str | None = None,
namespace: str = "default",
task_queue: str = "machina-tasks",
pool_size: int = 100,
@@ -560,11 +568,19 @@ async def run_standalone_worker(
python -m services.temporal.worker
Args:
- server_address: Temporal server address
+ server_address: Temporal server address. Defaults to
+ ``TEMPORAL_SERVER_ADDRESS`` (canonical value in ``.env.template``
+ via ``core.env_defaults``) so a bare
+ ``python -m services.temporal.worker`` dials the same server
+ as the backend.
namespace: Temporal namespace
task_queue: Task queue to poll
pool_size: Connection pool size
"""
+ if server_address is None:
+ from core.env_defaults import env_value
+
+ server_address = env_value("TEMPORAL_SERVER_ADDRESS")
logger.info(
"Starting standalone Temporal worker",
server_address=server_address,
@@ -634,7 +650,11 @@ async def run_standalone_worker(
max_concurrent_activities=pool_size,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
+ # ``TracingInterceptor`` is registered once on the shared client;
+ # the SDK prepends client interceptors to every worker, so
+ # repeating it here double-instruments each Signal / Query /
+ # Update / Activity and doubles the emitted spans.
+ interceptors=[ObservabilityWorkerInterceptor()],
)
logger.info("Worker running. Press Ctrl+C to stop.")
@@ -694,7 +714,11 @@ async def create_worker(
max_concurrent_activities=100,
max_concurrent_workflow_tasks=10,
graceful_shutdown_timeout=_graceful_shutdown_timeout(),
- interceptors=[TracingInterceptor(), ObservabilityWorkerInterceptor()],
+ # ``TracingInterceptor`` is registered once on the shared client;
+ # the SDK prepends client interceptors to every worker, so
+ # repeating it here double-instruments each Signal / Query /
+ # Update / Activity and doubles the emitted spans.
+ interceptors=[ObservabilityWorkerInterceptor()],
)